/* * Copyright (C) 2020-present ScyllaDB */ /* * SPDX-License-Identifier: AGPL-3.0-or-later */ #include "test/lib/cql_test_env.hh" #include #include #include #include "utils/error_injection.hh" #include "db/timeout_clock.hh" #include "test/lib/cql_assertions.hh" #include "types/list.hh" #include "log.hh" #include "cql3/query_options.hh" #include using namespace std::literals::chrono_literals; static logging::logger flogger("error_injection_test"); using milliseconds = std::chrono::milliseconds; using minutes = std::chrono::minutes; using steady_clock = std::chrono::steady_clock; constexpr milliseconds sleep_msec(10); // Injection time sleep 10 msec constexpr minutes future_mins(10); // Far in future 10 mins SEASTAR_TEST_CASE(test_inject_noop) { utils::error_injection errinj; BOOST_REQUIRE_NO_THROW(errinj.inject("noop1", [] () { throw std::runtime_error("shouldn't happen"); })); errinj.enable("error"); BOOST_ASSERT(errinj.enabled_injections().empty()); BOOST_ASSERT(errinj.enter("error") == false); auto start_time = steady_clock::now(); return errinj.inject("noop2", sleep_msec).then([start_time] { auto wait_time = std::chrono::duration_cast(steady_clock::now() - start_time); BOOST_REQUIRE_LT(wait_time.count(), sleep_msec.count()); return make_ready_future<>(); }); } SEASTAR_TEST_CASE(test_is_enabled) { utils::error_injection errinj; // Test enable and disable errinj.enable("is_enabled_test", false); errinj.disable("is_enabled_test"); BOOST_ASSERT(errinj.enabled_injections().size() == 0); // Test enable with one_shot=true and enter errinj.enable("is_enabled_test", true); BOOST_ASSERT(errinj.enabled_injections().size() == 1); BOOST_ASSERT(errinj.enter("is_enabled_test")); BOOST_ASSERT(errinj.enabled_injections().size() == 0); return make_ready_future<>(); } SEASTAR_TEST_CASE(test_inject_lambda) { utils::error_injection errinj; errinj.enable("lambda"); BOOST_REQUIRE_THROW(errinj.inject("lambda", [] () -> void { throw std::runtime_error("test"); }), std::runtime_error); errinj.disable("lambda"); BOOST_REQUIRE_NO_THROW(errinj.inject("lambda", [] () -> void { throw std::runtime_error("test"); })); errinj.enable("lambda"); BOOST_REQUIRE_THROW(errinj.inject("lambda", [] () -> void { throw std::runtime_error("test"); }), std::runtime_error); return make_ready_future<>(); } SEASTAR_TEST_CASE(test_inject_sleep_duration) { utils::error_injection errinj; auto start_time = steady_clock::now(); errinj.enable("future_sleep"); return errinj.inject("future_sleep", sleep_msec).then([start_time] { auto wait_time = std::chrono::duration_cast(steady_clock::now() - start_time); BOOST_REQUIRE_GE(wait_time.count(), sleep_msec.count()); return make_ready_future<>(); }); } SEASTAR_TEST_CASE(test_inject_sleep_deadline_steady_clock) { return do_with_cql_env_thread([] (cql_test_env& e) { utils::error_injection errinj; // Inject sleep, deadline short-circuit auto deadline = steady_clock::now() + sleep_msec; errinj.enable("future_deadline"); errinj.inject("future_deadline", deadline).then([deadline] { BOOST_REQUIRE_GE(std::chrono::duration_cast(steady_clock::now() - deadline).count(), 0); return make_ready_future<>(); }).get(); }); } SEASTAR_TEST_CASE(test_inject_sleep_deadline_manual_clock) { return do_with_cql_env_thread([] (cql_test_env& e) { utils::error_injection errinj; // Inject sleep, deadline short-circuit auto deadline = seastar::manual_clock::now() + sleep_msec; errinj.enable("future_deadline"); auto f = errinj.inject("future_deadline", deadline).then([deadline] { BOOST_REQUIRE_GE(std::chrono::duration_cast(seastar::manual_clock::now() - deadline).count(), 0); return make_ready_future<>(); }); manual_clock::advance(sleep_msec); f.get(); }); } SEASTAR_TEST_CASE(test_inject_sleep_deadline_manual_clock_lambda) { return do_with_cql_env_thread([] (cql_test_env& e) { utils::error_injection errinj; // Inject sleep, deadline short-circuit auto deadline = seastar::manual_clock::now() + sleep_msec; errinj.enable("future_deadline"); auto f = errinj.inject("future_deadline", deadline, [deadline] { BOOST_REQUIRE_GE(std::chrono::duration_cast(seastar::manual_clock::now() - deadline).count(), 0); return make_ready_future<>(); }); manual_clock::advance(sleep_msec); f.get(); }); } SEASTAR_TEST_CASE(test_inject_sleep_deadline_db_clock) { return do_with_cql_env_thread([] (cql_test_env& e) { utils::error_injection errinj; // Inject sleep, deadline short-circuit auto deadline = db::timeout_clock::now() + sleep_msec; errinj.enable("future_deadline"); errinj.inject("future_deadline", deadline).then([deadline] { BOOST_REQUIRE_GE(std::chrono::duration_cast(db::timeout_clock::now() - deadline).count(), 0); return make_ready_future<>(); }).get(); }); } SEASTAR_TEST_CASE(test_inject_future_disabled) { utils::error_injection errinj; auto start_time = steady_clock::now(); return errinj.inject("futid", sleep_msec).then([start_time] { auto wait_time = std::chrono::duration_cast(steady_clock::now() - start_time); BOOST_REQUIRE_LT(wait_time.count(), sleep_msec.count()); return make_ready_future<>(); }); } SEASTAR_TEST_CASE(test_error_exceptions) { auto exc = std::make_exception_ptr(utils::injected_error("test")); BOOST_TEST(!is_timeout_exception(exc)); return make_ready_future<>(); } SEASTAR_TEST_CASE(test_is_timeout_exception) { for (auto ep : { std::make_exception_ptr(seastar::rpc::timeout_error()), std::make_exception_ptr(seastar::semaphore_timed_out()), std::make_exception_ptr(seastar::timed_out_error()), }) { BOOST_TEST(is_timeout_exception(ep)); try { std::rethrow_exception(ep); } catch (...) { try { std::throw_with_nested(std::runtime_error("Hello")); } catch (...) { BOOST_TEST(is_timeout_exception(std::current_exception())); } } } return make_ready_future<>(); } SEASTAR_TEST_CASE(test_inject_exception) { utils::error_injection errinj; errinj.enable("exc"); return errinj.inject("exc", [] () -> std::exception_ptr { return std::make_exception_ptr(std::runtime_error("test")); }).then_wrapped([] (auto f) { BOOST_REQUIRE_THROW(f.get(), std::runtime_error); return make_ready_future<>(); }); } SEASTAR_TEST_CASE(test_inject_two) { utils::error_injection errinj; auto f = make_ready_future<>(); errinj.enable("one"); errinj.enable("two"); std::vector expected = { "one", "two" }; auto enabled_injections = errinj.enabled_injections(); std::sort(enabled_injections.begin(), enabled_injections.end()); BOOST_TEST(enabled_injections == expected); return make_ready_future<>(); } SEASTAR_TEST_CASE(test_disable_all) { utils::error_injection errinj; auto f = make_ready_future<>(); errinj.enable("one"); errinj.enable("two"); errinj.disable_all(); auto enabled_injections = errinj.enabled_injections(); BOOST_TEST(enabled_injections == std::vector()); return make_ready_future<>(); } SEASTAR_TEST_CASE(test_inject_once) { utils::error_injection errinj; errinj.enable("first", true); std::vector expected1 = { "first" }; auto enabled_injections1 = errinj.enabled_injections(); BOOST_TEST(enabled_injections1 == expected1); BOOST_REQUIRE_THROW(errinj.inject("first", [] { throw std::runtime_error("test"); }), std::runtime_error); std::vector expected_empty; auto enabled_injections2 = errinj.enabled_injections(); BOOST_TEST(enabled_injections2 == expected_empty); BOOST_REQUIRE_NO_THROW(errinj.inject("first", [] { throw std::runtime_error("test"); })); return make_ready_future<>(); } // Test error injection CQL API // NOTE: currently since functions can't get terminals an auxiliary table // with error injection names and one shot parameters SEASTAR_TEST_CASE(test_inject_cql) { return do_with_cql_env([](cql_test_env& e) { return seastar::async([&e] { // Type of returned list of error injections cql3/functions/error_injcetion_fcts.cc const auto my_list_type = list_type_impl::get_instance(ascii_type, false); #ifdef SCYLLA_ENABLE_ERROR_INJECTION auto row_empty = my_list_type->decompose(make_list_value(my_list_type, list_type_impl::native_type{{}})); auto row_test1 = my_list_type->decompose(make_list_value(my_list_type, list_type_impl::native_type{{"test1"}})); auto row_test2 = my_list_type->decompose(make_list_value(my_list_type, list_type_impl::native_type{{"test2"}})); #else auto row_empty = my_list_type->decompose(make_list_value(my_list_type, list_type_impl::native_type{{}})); auto row_test1 = row_empty; auto row_test2 = row_empty; #endif // Auxiliary table with terminals cquery_nofail(e, "create table error_name (name ascii primary key, one_shot ascii)"); // Enable (test1,one_shot=true) cquery_nofail(e, "insert into error_name (name, one_shot) values ('test1', 'true')"); // Check no error injections before injecting auto ret0 = e.execute_cql("select enabled_injections() from error_name limit 1").get0(); assert_that(ret0).is_rows().with_rows({ {row_empty} }); cquery_nofail(e, "select enable_injection(name, one_shot) from error_name where name = 'test1'"); // enabled_injections() returns a list all injections in one call, so limit 1 auto ret1 = e.execute_cql("select enabled_injections() from error_name limit 1").get0(); assert_that(ret1).is_rows().with_rows({ {row_test1} }); utils::get_local_injector().inject("test1", [] {}); // Noop one-shot injection auto ret2 = e.execute_cql("select enabled_injections() from error_name limit 1").get0(); assert_that(ret2).is_rows().with_rows({ // Empty list after one shot executed {row_empty} }); // Again (test1,one_shot=true) but disable with CQL API cquery_nofail(e, "select enable_injection(name, one_shot) from error_name where name = 'test1'"); // enabled_injections() returns a list all injections in one call, so limit 1 auto ret3 = e.execute_cql("select enabled_injections() from error_name limit 1").get0(); assert_that(ret3).is_rows().with_rows({ {row_test1} }); // Disable cquery_nofail(e, "select disable_injection(name) from error_name where name = 'test1'"); auto ret4 = e.execute_cql("select enabled_injections() from error_name limit 1").get0(); assert_that(ret4).is_rows().with_rows({ // Empty list after one shot disabled {row_empty} }); cquery_nofail(e, "insert into error_name (name, one_shot) values ('test2', 'false')"); cquery_nofail(e, "select enable_injection(name, one_shot) from error_name where name = 'test2'"); utils::get_local_injector().inject("test2", [] {}); // Noop injection, doesn't disable auto ret5 = e.execute_cql("select enabled_injections() from error_name limit 1").get0(); assert_that(ret5).is_rows().with_rows({ {row_test2} }); }); }); }