From b0edfa6d70f8f4751975f2eb606be1095c4bcba9 Mon Sep 17 00:00:00 2001 From: Calle Wilund Date: Wed, 11 Aug 2021 14:13:12 +0000 Subject: [PATCH] commitlog/config: Make hard size enforcement false by default + add config opt Refs #9053 Flips default for commitlog disk footprint hard limit enforcement to off due to observed latency stalls with stress runs. Instead adds an optional flag "commitlog_use_hard_size_limit" which can be turned on to in fact do enforce it. Sort of tape and string fix until we can properly tweak the balance between cl & sstable flush rate. Closes #9195 (cherry picked from commit 3633c077bed2662a2e66e6e2034856cc6df74923) --- db/commitlog/commitlog.cc | 1 + db/commitlog/commitlog.hh | 2 +- db/config.cc | 2 ++ db/config.hh | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/commitlog/commitlog.cc b/db/commitlog/commitlog.cc index 1bbb7b1050..559f3345b4 100644 --- a/db/commitlog/commitlog.cc +++ b/db/commitlog/commitlog.cc @@ -136,6 +136,7 @@ db::commitlog::config db::commitlog::config::from_db_config(const db::config& cf c.extensions = &cfg.extensions(); c.reuse_segments = cfg.commitlog_reuse_segments(); c.use_o_dsync = cfg.commitlog_use_o_dsync(); + c.allow_going_over_size_limit = !cfg.commitlog_use_hard_size_limit(); return c; } diff --git a/db/commitlog/commitlog.hh b/db/commitlog/commitlog.hh index f309d2d371..fba4932898 100644 --- a/db/commitlog/commitlog.hh +++ b/db/commitlog/commitlog.hh @@ -140,7 +140,7 @@ public: bool reuse_segments = true; bool use_o_dsync = false; bool warn_about_segments_left_on_disk_after_shutdown = true; - bool allow_going_over_size_limit = false; + bool allow_going_over_size_limit = true; const db::extensions * extensions = nullptr; }; diff --git a/db/config.cc b/db/config.cc index 30704b1f0b..87a652deb1 100644 --- a/db/config.cc +++ b/db/config.cc @@ -371,6 +371,8 @@ db::config::config(std::shared_ptr exts) "Whether or not to re-use commitlog segments when finished instead of deleting them. Can improve commitlog latency on some file systems.\n") , commitlog_use_o_dsync(this, "commitlog_use_o_dsync", value_status::Used, true, "Whether or not to use O_DSYNC mode for commitlog segments IO. Can improve commitlog latency on some file systems.\n") + , commitlog_use_hard_size_limit(this, "commitlog_use_hard_size_limit", value_status::Used, false, + "Whether or not to use a hard size limit for commitlog disk usage. Default is false. Enabling this can cause latency spikes, whereas the default can lead to occasional disk usage peaks.\n") /* Compaction settings */ /* Related information: Configuring compaction */ , compaction_preheat_key_cache(this, "compaction_preheat_key_cache", value_status::Unused, true, diff --git a/db/config.hh b/db/config.hh index c9df160fe5..b78ca933cd 100644 --- a/db/config.hh +++ b/db/config.hh @@ -163,6 +163,7 @@ public: named_value commitlog_total_space_in_mb; named_value commitlog_reuse_segments; named_value commitlog_use_o_dsync; + named_value commitlog_use_hard_size_limit; named_value compaction_preheat_key_cache; named_value concurrent_compactors; named_value in_memory_compaction_limit_in_mb;