db: refresh row cache's underlying data source after compaction

Underlying data source in row cache holds a reference to sstable set
prior to compaction which isn't released until a memtable flush, which
means file descriptors of deleted sstables remains opened, wasting
disk space.
The fix is to refresh underlying data source in row cache.

Fixes #2570.

Signed-off-by: Raphael S. Carvalho <raphaelsc@scylladb.com>

database.cc

@@ -170,7 +170,6 @@ column_family::sstables_as_mutation_source() {
 snapshot_source
 column_family::sstables_as_snapshot_source() {
     return snapshot_source([this] () {
-        // FIXME: Will keep sstables on disk until next memtable flush. Make compaction force cache refresh.
         auto sst_set = _sstables;
         return mutation_source([this, sst_set = std::move(sst_set)] (schema_ptr s,
                 const dht::partition_range& r,
@@ -1289,6 +1288,10 @@ column_family::rebuild_sstable_list(const std::vector<sstables::shared_sstable>&
             } catch (sstables::atomic_deletion_cancelled& adc) {
                 dblog.debug("Failed to delete sstables after compaction: {}", adc);
             }
+        }).then([this] {
+            // refresh underlying data source in row cache to prevent it from holding reference
+            // to sstables files which were previously deleted.
+            _cache.refresh_snapshot();
         });
     });
 }

row_cache.cc

@@ -860,6 +860,10 @@ future<> row_cache::update_invalidating(memtable& m) {
     });
 }
 
+void row_cache::refresh_snapshot() {
+    _underlying = _snapshot_source();
+}
+
 void row_cache::touch(const dht::decorated_key& dk) {
  _read_section(_tracker.region(), [&] {
   with_linearized_managed_bytes([&] {

row_cache.hh

@@ -442,6 +442,10 @@ public:
     // as few elements as possible.
     future<> update_invalidating(memtable&);
 
+    // Refreshes snapshot. Must only be used if logical state in the underlying data
+    // source hasn't changed.
+    void refresh_snapshot();
+
     // Moves given partition to the front of LRU if present in cache.
     void touch(const dht::decorated_key&);