mirror of
https://github.com/scylladb/scylladb.git
synced 2026-05-28 10:41:12 +00:00
5ab4932f34
Previously, the progress of download_task_impl launched by the "restore" API was not tracked. Since restore operations can involve large data transfers, this makes it difficult for users to monitor progress. The restore process happens in two sequential steps: 1. Open specified SSTables from object storage 2. Download and stream mutation fragments from the opened SSTables to mapped destinations While both steps contribute to overall progress, they use different units of measurement, making a unified progress metric challenging. Because the load-and-stream step (step 2) is the largest time-consuming part of the restore. This change implements progress tracking for this step as an initial improvement to provide users with partial visibility into the restore operation. Fixes scylladb/scylladb#21427 Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>
98 lines
3.2 KiB
C++
98 lines
3.2 KiB
C++
/*
|
|
* Copyright (C) 2021-present ScyllaDB
|
|
*/
|
|
|
|
/*
|
|
* SPDX-License-Identifier: AGPL-3.0-or-later
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <vector>
|
|
#include <seastar/core/sharded.hh>
|
|
#include "schema/schema_fwd.hh"
|
|
#include "sstables/shared_sstable.hh"
|
|
#include "tasks/task_manager.hh"
|
|
|
|
using namespace seastar;
|
|
|
|
namespace replica {
|
|
class database;
|
|
}
|
|
|
|
namespace sstables { class storage_manager; }
|
|
|
|
namespace netw { class messaging_service; }
|
|
namespace db {
|
|
namespace view {
|
|
class view_builder;
|
|
}
|
|
}
|
|
|
|
// The handler of the 'storage_service/load_new_ss_tables' endpoint which, in
|
|
// turn, is the target of the 'nodetool refresh' command.
|
|
// Gets sstables from the upload directory and makes them available in the
|
|
// system. Built on top of the distributed_loader functionality.
|
|
class sstables_loader : public seastar::peering_sharded_service<sstables_loader> {
|
|
public:
|
|
class task_manager_module : public tasks::task_manager::module {
|
|
public:
|
|
task_manager_module(tasks::task_manager& tm) noexcept : tasks::task_manager::module(tm, "sstables_loader") {}
|
|
};
|
|
|
|
private:
|
|
sharded<replica::database>& _db;
|
|
netw::messaging_service& _messaging;
|
|
sharded<db::view::view_builder>& _view_builder;
|
|
shared_ptr<task_manager_module> _task_manager_module;
|
|
sstables::storage_manager& _storage_manager;
|
|
seastar::scheduling_group _sched_group;
|
|
|
|
// Note that this is obviously only valid for the current shard. Users of
|
|
// this facility should elect a shard to be the coordinator based on any
|
|
// given objective criteria
|
|
//
|
|
// It shouldn't be impossible to actively serialize two callers if the need
|
|
// ever arise.
|
|
bool _loading_new_sstables = false;
|
|
|
|
future<> load_and_stream(sstring ks_name, sstring cf_name,
|
|
table_id, std::vector<sstables::shared_sstable> sstables,
|
|
bool primary_replica_only, bool unlink_sstables,
|
|
std::function<void(unsigned)> on_streamed);
|
|
|
|
public:
|
|
sstables_loader(sharded<replica::database>& db,
|
|
netw::messaging_service& messaging,
|
|
sharded<db::view::view_builder>& vb,
|
|
tasks::task_manager& tm,
|
|
sstables::storage_manager& sstm,
|
|
seastar::scheduling_group sg);
|
|
|
|
future<> stop();
|
|
|
|
/**
|
|
* Load new SSTables not currently tracked by the system
|
|
*
|
|
* This can be called, for instance, after copying a batch of SSTables to a CF directory.
|
|
*
|
|
* This should not be called in parallel for the same keyspace / column family, and doing
|
|
* so will throw an std::runtime_exception.
|
|
*
|
|
* @param ks_name the keyspace in which to search for new SSTables.
|
|
* @param cf_name the column family in which to search for new SSTables.
|
|
* @return a future<> when the operation finishes.
|
|
*/
|
|
future<> load_new_sstables(sstring ks_name, sstring cf_name,
|
|
bool load_and_stream, bool primary_replica_only);
|
|
|
|
/**
|
|
* Download new SSTables not currently tracked by the system from object store
|
|
*/
|
|
future<tasks::task_id> download_new_sstables(sstring ks_name, sstring cf_name,
|
|
sstring prefix, std::vector<sstring> sstables,
|
|
sstring endpoint, sstring bucket);
|
|
|
|
class download_task_impl;
|
|
};
|