diff --git a/ent/encryption/encryption.cc b/ent/encryption/encryption.cc index 2fd27a505c..42afe7c888 100644 --- a/ent/encryption/encryption.cc +++ b/ent/encryption/encryption.cc @@ -475,6 +475,14 @@ public: for (auto&& [id, h] : _per_thread_kmip_host_cache[this_shard_id()]) { co_await h->disconnect(); } + static auto stop_all = [](auto&& cache) -> future<> { + for (auto& [k, host] : cache) { + co_await host->stop(); + } + }; + co_await stop_all(_per_thread_kms_host_cache[this_shard_id()]); + co_await stop_all(_per_thread_gcp_host_cache[this_shard_id()]); + _per_thread_provider_cache[this_shard_id()].clear(); _per_thread_system_key_cache[this_shard_id()].clear(); _per_thread_kmip_host_cache[this_shard_id()].clear(); diff --git a/ent/encryption/gcp_host.cc b/ent/encryption/gcp_host.cc index 90d53c0bba..ca7641f191 100644 --- a/ent/encryption/gcp_host.cc +++ b/ent/encryption/gcp_host.cc @@ -102,6 +102,7 @@ public: ~impl() = default; future<> init(); + future<> stop(); const host_options& options() const { return _options; } @@ -874,6 +875,11 @@ future<> encryption::gcp_host::impl::init() { _initialized = true; } +future<> encryption::gcp_host::impl::stop() { + co_await _attr_cache.stop(); + co_await _id_cache.stop(); +} + std::tuple encryption::gcp_host::impl::parse_key(std::string_view spec) { auto i = spec.find_last_of('/'); if (i == std::string_view::npos) { @@ -1036,6 +1042,10 @@ future<> encryption::gcp_host::init() { return _impl->init(); } +future<> encryption::gcp_host::stop() { + return _impl->stop(); +} + const encryption::gcp_host::host_options& encryption::gcp_host::options() const { return _impl->options(); } diff --git a/ent/encryption/gcp_host.hh b/ent/encryption/gcp_host.hh index 8764c0d24a..8821a9d0f6 100644 --- a/ent/encryption/gcp_host.hh +++ b/ent/encryption/gcp_host.hh @@ -65,6 +65,8 @@ public: ~gcp_host(); future<> init(); + future<> stop(); + const host_options& options() const; struct option_override : public t_credentials_source> { diff --git a/ent/encryption/kmip_host.cc b/ent/encryption/kmip_host.cc index c39fce964b..910c4b5178 100644 --- a/ent/encryption/kmip_host.cc +++ b/ent/encryption/kmip_host.cc @@ -724,9 +724,11 @@ future<> kmip_host::impl::connect() { } future<> kmip_host::impl::disconnect() { - return do_for_each(_options.hosts, [this](const sstring& host) { + co_await do_for_each(_options.hosts, [this](const sstring& host) { return clear_connections(host); }); + co_await _attr_cache.stop(); + co_await _id_cache.stop(); } static unsigned from_str(unsigned (*f)(char*, int, int*), const sstring& s, const sstring& what) { diff --git a/ent/encryption/kms_host.cc b/ent/encryption/kms_host.cc index bc0920d018..b88d507a4d 100644 --- a/ent/encryption/kms_host.cc +++ b/ent/encryption/kms_host.cc @@ -160,6 +160,8 @@ public: ~impl() = default; future<> init(); + future<> stop(); + const host_options& options() const { return _options; } @@ -992,6 +994,11 @@ future<> encryption::kms_host::impl::init() { _initialized = true; } +future<> encryption::kms_host::impl::stop() { + co_await _attr_cache.stop(); + co_await _id_cache.stop(); +} + future encryption::kms_host::impl::create_key(const attr_cache_key& k) { auto& master_key = k.master_key; auto& aws_assume_role_arn = k.aws_assume_role_arn; @@ -1154,6 +1161,10 @@ future<> encryption::kms_host::init() { return _impl->init(); } +future<> encryption::kms_host::stop() { + return _impl->stop(); +} + const encryption::kms_host::host_options& encryption::kms_host::options() const { return _impl->options(); } diff --git a/ent/encryption/kms_host.hh b/ent/encryption/kms_host.hh index ac9fd1de41..2594578018 100644 --- a/ent/encryption/kms_host.hh +++ b/ent/encryption/kms_host.hh @@ -63,6 +63,8 @@ public: ~kms_host(); future<> init(); + future<> stop(); + const host_options& options() const; struct option_override { diff --git a/test/boost/encryption_at_rest_test.cc b/test/boost/encryption_at_rest_test.cc index 8084e401df..164a140ebe 100644 --- a/test/boost/encryption_at_rest_test.cc +++ b/test/boost/encryption_at_rest_test.cc @@ -1078,41 +1078,49 @@ static future<> test_broken_encrypted_commitlog(const test_provider_args& args, */ static future<> network_error_test_helper(const tmpdir& tmp, const std::string& host, std::function(const fake_proxy&)> make_opts) { fake_proxy proxy(host); + std::exception_ptr p; + try { + auto [scopts, yaml] = make_opts(proxy); - auto [scopts, yaml] = make_opts(proxy); + test_provider_args args{ + .tmp = tmp, + .extra_yaml = yaml, + .n_tables = 10, + .before_create_table = [&](auto& env) { + // turn off proxy. all key resolution after this should fail + proxy.enable(false); + // wait for key cache expiry. + seastar::sleep(10ms).get(); + // ensure commitlog will create a new segment on write -> eventual write failure + env.db().invoke_on_all([](replica::database& db) { + return db.commitlog()->force_new_active_segment(); + }).get(); + }, + .on_insert_exception = [&](auto&&) { + // once we get the exception we have to enable key resolution again, + // otherwise we can't shut down cql test env. + proxy.enable(true); + }, + .timeout = timeout_config{ + // set really low write timeouts so we get a failure (timeout) + // when we fail to write to commitlog + 100ms, 100ms, 100ms, 100ms, 100ms, 100ms, 100ms + }, + }; - test_provider_args args{ - .tmp = tmp, - .extra_yaml = yaml, - .n_tables = 10, - .before_create_table = [&](auto& env) { - // turn off proxy. all key resolution after this should fail - proxy.enable(false); - // wait for key cache expiry. - seastar::sleep(10ms).get(); - // ensure commitlog will create a new segment on write -> eventual write failure - env.db().invoke_on_all([](replica::database& db) { - return db.commitlog()->force_new_active_segment(); - }).get(); - }, - .on_insert_exception = [&](auto&&) { - // once we get the exception we have to enable key resolution again, - // otherwise we can't shut down cql test env. - proxy.enable(true); - }, - .timeout = timeout_config{ - // set really low write timeouts so we get a failure (timeout) - // when we fail to write to commitlog - 100ms, 100ms, 100ms, 100ms, 100ms, 100ms, 100ms - }, - }; - - BOOST_REQUIRE_THROW( - co_await test_broken_encrypted_commitlog(args, scopts); - , std::exception - ); + BOOST_REQUIRE_THROW( + co_await test_broken_encrypted_commitlog(args, scopts); + , exceptions::mutation_write_timeout_exception + ); + } catch (...) { + p = std::current_exception(); + } co_await proxy.stop(); + + if (p) { + std::rethrow_exception(p); + } } SEASTAR_TEST_CASE(test_kms_network_error, *check_run_test_decorator("ENABLE_KMS_TEST")) {