Merge "sstables, gdb: Centralize tracking of sstable instances" from Tomasz

"
Currently, GDB scripts locate sstables by scanning the heap for
bag_sstable_set containers. That has disadvatanges:

  - not all containers are considered

  - it's extremely slow on large heaps

  - fragile, new containers can be added, and we won't even know

This series fixes all above by adding a per-shard sstable tracker
which tracks sstable objects in a linked-list.
"

* 'sstable-tracker' of github.com:tgrabiec/scylla:
  gdb: Use sstable tracker to get the list of sstables
  gdb: Make intrusive_list recognize member_hook links
  sstables: Track whether sstable was already open or not
  sstables: Track all instances of sstable objects
  sstables: Make sstable object not movable
  sstables: Move constructor out of line

scylla-gdb.py

@@ -50,6 +50,8 @@ class intrusive_list:
             # Some boost versions have this instead
             self.root = rps['m_header']
         member_hook = get_template_arg_with_prefix(list_type, "boost::intrusive::member_hook")
+        if not member_hook:
+            member_hook = get_template_arg_with_prefix(list_type, "struct boost::intrusive::member_hook")
         if member_hook:
             self.link_offset = member_hook.template_argument(2).cast(self.size_t)
         else:
@@ -2280,16 +2282,8 @@ class scylla_cache(gdb.Command):
 
 def find_sstables():
     """A generator which yields pointers to all live sstable objects on current shard."""
-    visited = set()
-    # FIXME: Add support for other sstable sets. Also, we should change Scylla to make this easier
-    for sst_set in find_instances('sstables::bag_sstable_set'):
-        sstables = std_vector(sst_set['_sstables'])
-        for sst_ptr in sstables:
-            sst = seastar_lw_shared_ptr(sst_ptr).get()
-            if not int(sst) in visited:
-                visited.add(int(sst))
-                yield sst
-
+    for sst in intrusive_list(gdb.parse_and_eval('sstables::tracker._sstables')):
+        yield sst.address
 
 class scylla_sstables(gdb.Command):
     """Lists all sstable objects on currents shard together with useful information like on-disk and in-memory size."""
@@ -2305,6 +2299,8 @@ class scylla_sstables(gdb.Command):
         count = 0
 
         for sst in find_sstables():
+            if not sst['_open']:
+                continue
             count += 1
             size = 0
 

sstables/sstables.cc

@@ -84,6 +84,20 @@ namespace sstables {
 
 logging::logger sstlog("sstable");
 
+namespace bi = boost::intrusive;
+
+class sstable_tracker {
+    bi::list<sstable,
+        bi::member_hook<sstable, sstable::tracker_link_type, &sstable::_tracker_link>,
+        bi::constant_time_size<false>> _sstables;
+public:
+    void add(sstable& sst) {
+        _sstables.push_back(sst);
+    }
+};
+
+static thread_local sstable_tracker tracker;
+
 // Because this is a noop and won't hold any state, it is better to use a global than a
 // thread_local. It will be faster, specially on non-x86.
 static noop_write_monitor default_noop_write_monitor;
@@ -1277,6 +1291,7 @@ future<> sstable::open_data() {
         }
         auto* sm = _components->scylla_metadata->data.get<scylla_metadata_type::Sharding, sharding_metadata>();
         if (!sm) {
+            _open = true;
             return make_ready_future<>();
         }
         auto c = &sm->token_ranges.elements;
@@ -1286,6 +1301,7 @@ future<> sstable::open_data() {
             return make_ready_future<>();
         }).then([this, c] () mutable {
             c = {};
+            _open = true;
             return make_ready_future<>();
         });
     });
@@ -3349,6 +3365,29 @@ mutation_source sstable::as_mutation_source() {
     });
 }
 
+sstable::sstable(schema_ptr schema,
+        sstring dir,
+        int64_t generation,
+        version_types v,
+        format_types f,
+        db::large_data_handler& large_data_handler,
+        gc_clock::time_point now,
+        io_error_handler_gen error_handler_gen,
+        size_t buffer_size)
+    : sstable_buffer_size(buffer_size)
+    , _schema(std::move(schema))
+    , _dir(std::move(dir))
+    , _generation(generation)
+    , _version(v)
+    , _format(f)
+    , _now(now)
+    , _read_error_handler(error_handler_gen(sstable_read_error))
+    , _write_error_handler(error_handler_gen(sstable_write_error))
+    , _large_data_handler(large_data_handler)
+{
+    tracker.add(*this);
+}
+
 bool supports_correct_non_compound_range_tombstones() {
     return service::get_local_storage_service().cluster_supports_reading_correctly_serialized_range_tombstones();
 }

sstables/sstables.hh

@@ -64,6 +64,7 @@
 #include "sstables/shareable_components.hh"
 
 #include <seastar/util/optimized_optional.hh>
+#include <boost/intrusive/list.hpp>
 
 class sstable_assertions;
 
@@ -123,11 +124,15 @@ struct sstable_writer_config {
     utils::UUID run_identifier = utils::make_random_uuid();
 };
 
+class sstable_tracker;
+
 class sstable : public enable_lw_shared_from_this<sstable> {
     friend ::sstable_assertions;
+    friend sstable_tracker;
 public:
     using version_types = sstable_version_types;
     using format_types = sstable_format_types;
+    using tracker_link_type = bi::list_member_hook<bi::link_mode<bi::auto_unlink>>;
 public:
     sstable(schema_ptr schema,
             sstring dir,
@@ -137,21 +142,10 @@ public:
             db::large_data_handler& large_data_handler,
             gc_clock::time_point now,
             io_error_handler_gen error_handler_gen,
-            size_t buffer_size)
-        : sstable_buffer_size(buffer_size)
-        , _schema(std::move(schema))
-        , _dir(std::move(dir))
-        , _generation(generation)
-        , _version(v)
-        , _format(f)
-        , _now(now)
-        , _read_error_handler(error_handler_gen(sstable_read_error))
-        , _write_error_handler(error_handler_gen(sstable_write_error))
-        , _large_data_handler(large_data_handler)
-    { }
+            size_t buffer_size);
     sstable& operator=(const sstable&) = delete;
     sstable(const sstable&) = delete;
-    sstable(sstable&&) = default;
+    sstable(sstable&&) = delete;
 
     ~sstable();
 
@@ -472,6 +466,7 @@ private:
     foreign_ptr<lw_shared_ptr<shareable_components>> _components = make_foreign(make_lw_shared<shareable_components>());
     column_translation _column_translation;
     bool _shared = true;  // across shards; safe default
+    bool _open = false;
     // NOTE: _collector and _c_stats are used to generation of statistics file
     // when writing a new sstable.
     metadata_collector _collector;
@@ -546,6 +541,7 @@ private:
     db::large_data_handler& _large_data_handler;
 
     sstables_stats _stats;
+    tracker_link_type _tracker_link;
 
 public:
     const bool has_component(component_type f) const;