From 5d7c7654223affe695989a95abcbaf5aefa97bb3 Mon Sep 17 00:00:00 2001
From: Piotr Sarna <sarna@scylladb.com>
Date: Wed, 8 Sep 2021 10:48:33 +0200
Subject: [PATCH] db,view: split stopping view builder to drain+stop

In order to be able to avoid a deadlock when CQL server cannot be started,
the view builder shutdown procedure is now split to two parts -
- drain and stop. Drain is performed before storage proxy shutdown,
but stop() will be called even before drain is scheduled.
The deadlock is as follows:
 - view builder creates a reader permit in order to be able
   to read from system tables
 - CQL server fails to start, shutdown procedure begins
 - view builder stop() is not called (because it was not scheduled
   yet), so it holds onto its reader permit
 - database shutdown procedure waits for all permits to be destroyed,
   and it hangs indefinitely because view builder keeps holding
   its permit.
---
 db/view/view.cc         | 12 ++++++++++--
 db/view/view_builder.hh |  5 +++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/db/view/view.cc b/db/view/view.cc
index 605066e4c9..239f778f9b 100644
--- a/db/view/view.cc
+++ b/db/view/view.cc
@@ -1473,8 +1473,11 @@ future<> view_builder::start(service::migration_manager& mm) {
     return make_ready_future<>();
 }
 
-future<> view_builder::stop() {
-    vlogger.info("Stopping view builder");
+future<> view_builder::drain() {
+    if (_as.abort_requested()) {
+        return make_ready_future();
+    }
+    vlogger.info("Draining view builder");
     _as.request_abort();
     return _started.then([this] {
         return _mnotifier.unregister_listener(this).then([this] {
@@ -1494,6 +1497,11 @@ future<> view_builder::stop() {
     });
 }
 
+future<> view_builder::stop() {
+    vlogger.info("Stopping view builder");
+    return drain();
+}
+
 view_builder::build_step& view_builder::get_or_create_build_step(utils::UUID base_id) {
     auto it = _base_to_build_step.find(base_id);
     if (it == _base_to_build_step.end()) {
diff --git a/db/view/view_builder.hh b/db/view/view_builder.hh
index dd4f6721b6..23922d38f5 100644
--- a/db/view/view_builder.hh
+++ b/db/view/view_builder.hh
@@ -205,6 +205,11 @@ public:
      */
     future<> start(service::migration_manager&);
 
+    /**
+     * Drains view building in order to prepare it for shutdown.
+     */
+    future<> drain();
+
     /**
      * Stops the view building process.
      */