mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "api: support table auto compaction control"

Browse Source 
This reverts commit 1c444b7e1e. The test
it adds sometimes fails as follows:

  test/boost/sstable_datafile_test.cc(1076): fatal error: in "autocompaction_control_test":
  critical check cm->get_stats().pending_tasks == 1 || cm->get_stats().active_tasks == 1 has failed

Ivan is working on a fix, but let's revert this commit to avoid blocking
next promotion failing from time to time.
This commit is contained in:
Pekka Enberg
2020-04-11 17:56:02 +03:00
parent 679fb5887a
commit c8247aced6
7 changed files with 17 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
api/api-doc/column_family.json
View File

@@ -380,54 +380,16 @@
"operations":[
{
"method":"GET",
"summary":"check if the auto_compaction property is enabled for a given table",
"summary":"check if the auto compaction disabled",
"type":"boolean",
"nickname":"get_auto_compaction",
"nickname":"is_auto_compaction_disabled",
"produces":[
"application/json"
],
"parameters":[
{
"name":"name",
"description":"The table name in keyspace:name format",
"required":true,
"allowMultiple":false,
"type":"string",
"paramType":"path"
}
]
},
{
"method":"POST",
"summary":"Enable table auto compaction",
"type":"void",
"nickname":"enable_auto_compaction",
"produces":[
"application/json"
],
"parameters":[
{
"name":"name",
"description":"The table name in keyspace:name format",
"required":true,
"allowMultiple":false,
"type":"string",
"paramType":"path"
}
]
},
{
"method":"DELETE",
"summary":"Disable table auto compaction",
"type":"void",
"nickname":"disable_auto_compaction",
"produces":[
"application/json"
],
"parameters":[
{
"name":"name",
"description":"The table name in keyspace:name format",
"description":"The column family name in keyspace:name format",
"required":true,
"allowMultiple":false,
"type":"string",

25
api/column_family.cc
View File

@@ -839,26 +839,11 @@ void set_column_family(http_context& ctx, routes& r) {
return make_ready_future<json::json_return_type>(res);
});
cf::get_auto_compaction.set(r, [&ctx] (const_req req) {
const utils::UUID& uuid = get_uuid(req.param["name"], ctx.db.local());
column_family& cf = ctx.db.local().find_column_family(uuid);
return !cf.is_auto_compaction_disabled_by_user();
});
cf::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
return foreach_column_family(ctx, req->param["name"], [](column_family &cf) {
cf.enable_auto_compaction();
}).then([] {
return make_ready_future<json::json_return_type>(json_void());
});
});
cf::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
return foreach_column_family(ctx, req->param["name"], [](column_family &cf) {
cf.disable_auto_compaction();
}).then([] {
return make_ready_future<json::json_return_type>(json_void());
});
cf::is_auto_compaction_disabled.set(r, [] (const_req req) {
// FIXME
// currently auto compaction is disable
// it should be changed when it would have an API
return true;
});
cf::get_built_indexes.set(r, [&ctx](std::unique_ptr<request> req) {

33
api/storage_service.cc
View File

@@ -89,23 +89,6 @@ static auto wrap_ks_cf(http_context &ctx, ks_cf_func f) {
};
}
future<> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
if (tables.empty()) {
tables = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
}
return ctx.db.invoke_on_all([keyspace, tables, enabled] (database& db) {
return parallel_for_each(tables, [&db, keyspace, enabled](const sstring& table) mutable {
column_family& cf = db.find_column_family(keyspace, table);
if (enabled) {
cf.enable_auto_compaction();
} else {
cf.disable_auto_compaction();
}
return make_ready_future<>();
});
});
}
void set_storage_service(http_context& ctx, routes& r) {
ss::local_hostid.set(r, [](std::unique_ptr<request> req) {
return db::system_keyspace::get_local_host_id().then([](const utils::UUID& id) {
@@ -720,19 +703,19 @@ void set_storage_service(http_context& ctx, routes& r) {
});
ss::enable_auto_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
//TBD
unimplemented();
auto keyspace = validate_keyspace(ctx, req->param);
auto tables = split_cf(req->get_query_param("cf"));
return set_tables_autocompaction(ctx, keyspace, tables, true).then([]{
return make_ready_future<json::json_return_type>(json_void());
});
auto column_family = req->get_query_param("cf");
return make_ready_future<json::json_return_type>(json_void());
});
ss::disable_auto_compaction.set(r, [&ctx](std::unique_ptr<request> req) {
//TBD
unimplemented();
auto keyspace = validate_keyspace(ctx, req->param);
auto tables = split_cf(req->get_query_param("cf"));
return set_tables_autocompaction(ctx, keyspace, tables, false).then([]{
return make_ready_future<json::json_return_type>(json_void());
});
auto column_family = req->get_query_param("cf");
return make_ready_future<json::json_return_type>(json_void());
});
ss::deliver_hints.set(r, [](std::unique_ptr<request> req) {

7
database.hh
View File

@@ -504,7 +504,6 @@ private:
compaction_manager& _compaction_manager;
secondary_index::secondary_index_manager _index_manager;
int _compaction_disabled = 0;
bool _compaction_disabled_by_user = false;
utils::phased_barrier _flush_barrier;
seastar::gate _streaming_flush_gate;
std::vector<view_ptr> _views;
@@ -956,12 +955,6 @@ public:
future<> run_with_compaction_disabled(std::function<future<> ()> func);
void enable_auto_compaction();
void disable_auto_compaction();
bool is_auto_compaction_disabled_by_user() const {
return _compaction_disabled_by_user;
}
utils::phased_barrier::operation write_in_progress() {
return _pending_writes_phaser.start();
}

6
sstables/compaction_manager.cc
View File

@@ -515,10 +515,6 @@ inline bool compaction_manager::maybe_stop_on_error(future<> f, stop_iteration w
}
void compaction_manager::submit(column_family* cf) {
if (cf->is_auto_compaction_disabled_by_user()) {
return;
}
auto task = make_lw_shared<compaction_manager::task>();
task->compacting_cf = cf;
_tasks.push_back(task);
@@ -536,7 +532,7 @@ void compaction_manager::submit(column_family* cf) {
sstables::compaction_descriptor descriptor = cs.get_sstables_for_compaction(cf, get_candidates(cf));
int weight = trim_to_compact(&cf, descriptor);
if (descriptor.sstables.empty() || !can_proceed(task) || cf.is_auto_compaction_disabled_by_user()) {
if (descriptor.sstables.empty() || !can_proceed(task)) {
_stats.pending_tasks--;
return make_ready_future<stop_iteration>(stop_iteration::yes);
}

38
table.cc
View File

@@ -2434,44 +2434,6 @@ table::run_with_compaction_disabled(std::function<future<> ()> func) {
});
}
void
table::enable_auto_compaction() {
// XXX: unmute backlog. turn table backlog back on.
// see table::disable_auto_compaction() notes.
_compaction_disabled_by_user = false;
}
void
table::disable_auto_compaction() {
// XXX: mute backlog. When we disable background compactions
// for the table, we must also disable current backlog of the
// table compaction strategy that contributes to the scheduling
// group resources prioritization.
//
// There are 2 possibilities possible:
// - there are no ongoing background compaction, and we can freely
// mute table backlog.
// - there are compactions happening. than we must decide either
// we want to allow them to finish not allowing submitting new
// compactions tasks, or we may "suspend" them until the bg
// compactions will be enabled back. This is not a worst option
// because it will allow bg compactions to finish if there are
// unused resourced, it will not lose any writers/readers stats.
//
// Besides that:
// - there are major compactions that additionally uses constant
// size backlog of shares,
// - sstables rewrites tasks that do the same.
//
// Setting NullCompactionStrategy is not an option due to the
// following reasons:
// - it will 0 backlog if suspending current compactions is not an
// option
// - it will break computation of major compaction descriptor
// for new submissions
_compaction_disabled_by_user = true;
}
flat_mutation_reader
table::make_reader_excluding_sstable(schema_ptr s,
sstables::shared_sstable sst,

45
test/boost/sstable_datafile_test.cc
View File

@@ -1036,51 +1036,6 @@ static flat_mutation_reader make_normalizing_sstable_reader(shared_sstable sst,
return make_normalizing_sstable_reader(sst, s, query::full_partition_range);
}
SEASTAR_TEST_CASE(autocompaction_control_test) {
return test_env::do_with_async([] (test_env& env) {
storage_service_for_tests ssft;
BOOST_REQUIRE(smp::count == 1);
auto s = make_lw_shared(schema({}, some_keyspace, some_column_family, {{"p1", utf8_type}}, {{"c1", utf8_type}}, {}, {}, utf8_type));
auto cm = make_lw_shared<compaction_manager>();
cm->start();
auto tmp = tmpdir();
column_family::config cfg = column_family_test_config();
cfg.datadir = tmp.path().string();
cfg.enable_commitlog = false;
cfg.enable_incremental_backups = false;
auto cl_stats = make_lw_shared<cell_locker_stats>();
auto tracker = make_lw_shared<cache_tracker>();
auto cf = make_lw_shared<column_family>(s, cfg, column_family::no_commitlog(), *cm, *cl_stats, *tracker);
cf->start();
cf->set_compaction_strategy(sstables::compaction_strategy_type::null);
// auto compaction is enabled by default
BOOST_REQUIRE(!cf->is_auto_compaction_disabled_by_user());
// disable auto compaction by user
cf->disable_auto_compaction();
// check it is disabled
BOOST_REQUIRE(cf->is_auto_compaction_disabled_by_user());
// check CompactionManager does not receive background compaction submissions
cf->trigger_compaction();
BOOST_REQUIRE(cm->get_stats().pending_tasks == 0 && cm->get_stats().active_tasks == 0);
cf->get_compaction_manager().submit(cf.get());
BOOST_REQUIRE(cm->get_stats().pending_tasks == 0 && cm->get_stats().active_tasks == 0);
// enable auto compaction
cf->enable_auto_compaction();
// check enabled
BOOST_REQUIRE(!cf->is_auto_compaction_disabled_by_user());
// trigger background compaction
cf->trigger_compaction();
BOOST_REQUIRE(cm->get_stats().pending_tasks == 1 || cm->get_stats().active_tasks == 1);
// XXX: test backlog state
cm->stop().wait();
});
}
SEASTAR_TEST_CASE(compaction_manager_test) {
return test_env::do_with_async([] (test_env& env) {
storage_service_for_tests ssft;