scylladb/test/boost/error_injection_test.cc

/*
 * Copyright (C) 2020 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "test/lib/cql_test_env.hh"
#include <seastar/testing/test_case.hh>
#include "utils/error_injection.hh"
#include "log.hh"
#include <chrono>

using namespace std::literals::chrono_literals;

static logging::logger flogger("error_injection_test");

using milliseconds = std::chrono::milliseconds;
using minutes = std::chrono::minutes;
using steady_clock = std::chrono::steady_clock;

constexpr milliseconds sleep_msec(10); // Injection time sleep 10 msec
constexpr minutes future_mins(10);     // Far in future        10 mins

SEASTAR_TEST_CASE(test_inject_noop) {
    utils::error_injection<false> errinj;

    BOOST_REQUIRE_NO_THROW(errinj.inject("noop1",
            [] () { throw std::runtime_error("shouldn't happen"); }));

    BOOST_ASSERT(errinj.enabled_injections().empty());

    auto f = make_ready_future<>();
    auto start_time = steady_clock::now();
    errinj.inject("noop2", sleep_msec, f);
    return f.then([start_time] {
        auto wait_time = std::chrono::duration_cast<milliseconds>(steady_clock::now() - start_time);
        BOOST_REQUIRE_LT(wait_time.count(), sleep_msec.count());
        return make_ready_future<>();
    });
}

SEASTAR_TEST_CASE(test_inject_lambda) {
    utils::error_injection<true> errinj;

    errinj.enable("lambda");
    BOOST_REQUIRE_THROW(errinj.inject("lambda",
            [] () { throw std::runtime_error("test"); }),
            std::runtime_error);
    errinj.disable("lambda");
    BOOST_REQUIRE_NO_THROW(errinj.inject("lambda",
            [] () { throw std::runtime_error("test"); }));
    errinj.enable("lambda");
    BOOST_REQUIRE_THROW(errinj.inject("lambda",
            [] () { throw std::runtime_error("test"); }),
            std::runtime_error);
    return make_ready_future<>();
}

SEASTAR_TEST_CASE(test_inject_lambda_timeout) {
    utils::error_injection<true> errinj;

    errinj.enable("lambda_timeout1");
    auto past_time = steady_clock::now();
    BOOST_REQUIRE_NO_THROW(errinj.inject("lambda_timeout1",
            [] () { throw std::runtime_error("test"); }, std::make_optional(past_time)));

    errinj.enable("lambda_timeout2");
    auto future_time = steady_clock::now() + minutes(future_mins);
    BOOST_REQUIRE_THROW(errinj.inject("lambda_timeout2",
            [] () { throw std::runtime_error("test"); }, std::make_optional(future_time)),
            std::runtime_error);
    return make_ready_future<>();
}

SEASTAR_TEST_CASE(test_inject_future_sleep) {
    utils::error_injection<true> errinj;

    auto f = make_ready_future<>();
    auto start_time = steady_clock::now();
    errinj.enable("futi");
    errinj.inject("futi", sleep_msec, f);
    return f.then([start_time] {
        auto wait_time = std::chrono::duration_cast<milliseconds>(steady_clock::now() - start_time);
        BOOST_REQUIRE_GE(wait_time.count(), sleep_msec.count());
        return make_ready_future<>();
    });
}

SEASTAR_TEST_CASE(test_inject_future_sleep_timeout) {
    return do_with_cql_env_thread([] (cql_test_env& e) {
        utils::error_injection<true> errinj;

        // Inject sleep, deadline short-circuit
        auto f1 = make_ready_future<>();
        auto start_time = steady_clock::now();
        auto past_time = steady_clock::now();
        errinj.enable("futi_timeout1");
        errinj.inject("futi_timeout1", sleep_msec, f1, past_time);
        auto wait1 = f1.then([start_time] {
            auto wait_time = std::chrono::duration_cast<milliseconds>(steady_clock::now() - start_time);
            return make_ready_future<milliseconds>(wait_time);
        }).get0();
        BOOST_REQUIRE_LT(wait1.count(), sleep_msec.count());

        // Inject sleep, before cutoff deadline
        auto f2 = make_ready_future<>();
        start_time = steady_clock::now();
        auto future_time = steady_clock::now() + minutes(future_mins);
        errinj.enable("futi_timeout2");
        errinj.inject("futi_timeout2", sleep_msec, f2, future_time);
        auto wait2 = f2.then([start_time] {
            auto wait_time = std::chrono::duration_cast<milliseconds>(steady_clock::now() - start_time);
            return make_ready_future<milliseconds>(wait_time);
        }).get0();
        BOOST_REQUIRE_GE(wait2.count(), sleep_msec.count());
    });
}

SEASTAR_TEST_CASE(test_inject_future_sleep_timeout_short) {
    return do_with_cql_env_thread([] (cql_test_env& e) {
        utils::error_injection<true> errinj;

        // Inject sleep, deadline before sleep ms, should truncate sleep
        auto f = make_ready_future<>();
        auto start_time = steady_clock::now();
        auto actual_sleep = sleep_msec/2;
        auto future_time = start_time + actual_sleep;
        errinj.enable("futi_timeout_trunc");
        errinj.inject("futi_timeout_trunc", sleep_msec, f, future_time);
        auto wait = f.then([start_time] {
            auto wait_time = std::chrono::duration_cast<milliseconds>(steady_clock::now() - start_time);
            return make_ready_future<milliseconds>(wait_time);
        }).get0();
        BOOST_REQUIRE_LE(wait.count(), actual_sleep.count());
    });
}

SEASTAR_TEST_CASE(test_inject_future_disabled) {
    utils::error_injection<true> errinj;

    auto f = make_ready_future<>();
    auto start_time = steady_clock::now();
    errinj.inject("futid", sleep_msec, f);
    return f.then([start_time] {
        auto wait_time = std::chrono::duration_cast<milliseconds>(steady_clock::now() - start_time);
        BOOST_REQUIRE_LT(wait_time.count(), sleep_msec.count());
        return make_ready_future<>();
    });
}

SEASTAR_TEST_CASE(test_inject_exception) {
    utils::error_injection<true> errinj;

    auto f = make_ready_future<>();
    errinj.enable("exc");
    errinj.inject("exc", [] () { return std::make_exception_ptr(std::runtime_error("test")); }, f);
    return f.then_wrapped([] (auto f) {
        BOOST_REQUIRE_THROW(f.get(), std::runtime_error);
        return make_ready_future<>();
    });
}

SEASTAR_TEST_CASE(test_inject_exception_timeout) {
    return do_with_cql_env_thread([] (cql_test_env& e) {
        utils::error_injection<true> errinj;

        // Inject exception, should not throw, deadline short-circuit
        auto f1 = make_ready_future<>();
        auto past_time = steady_clock::now();
        errinj.enable("exc");
        errinj.inject("exc", [] () { return std::make_exception_ptr(std::runtime_error("test")); },
                f1, past_time);
        f1.then_wrapped([] (auto f1) {
            BOOST_REQUIRE_NO_THROW(f1.get());
            return make_ready_future<>();
        }).get();

        // Inject exception, should throw before cutoff deadline
        auto f2 = make_ready_future<>();
        auto future_time = steady_clock::now() + minutes(future_mins);
        errinj.enable("exc");
        errinj.inject("exc", [] () { return std::make_exception_ptr(std::runtime_error("test")); },
                f2, future_time);
        f2.then_wrapped([] (auto f2) {
            BOOST_REQUIRE_THROW(f2.get(), std::runtime_error);
            return make_ready_future<>();
        }).get();
    });
}

SEASTAR_TEST_CASE(test_inject_two) {
    utils::error_injection<true> errinj;

    auto f = make_ready_future<>();
    errinj.enable("one");
    errinj.enable("two");

    std::vector<sstring> expected = { "one", "two" };
    auto enabled_injections = errinj.enabled_injections();
    std::sort(enabled_injections.begin(), enabled_injections.end());
    BOOST_TEST(enabled_injections == expected);

    return make_ready_future<>();
}

SEASTAR_TEST_CASE(test_disable_all) {
    utils::error_injection<true> errinj;

    auto f = make_ready_future<>();
    errinj.enable("one");
    errinj.enable("two");
    errinj.disable_all();
    auto enabled_injections = errinj.enabled_injections();
    BOOST_TEST(enabled_injections == std::vector<sstring>());

    return make_ready_future<>();
}

SEASTAR_TEST_CASE(test_inject_once) {
    utils::error_injection<true> errinj;

    errinj.enable("first", true);

    std::vector<sstring> expected1 = { "first" };
    auto enabled_injections1 = errinj.enabled_injections();
    BOOST_TEST(enabled_injections1 == expected1);
    BOOST_REQUIRE_THROW(errinj.inject("first", [] { throw std::runtime_error("test"); }),
        std::runtime_error);

    std::vector<sstring> expected_empty;
    auto enabled_injections2 = errinj.enabled_injections();
    BOOST_TEST(enabled_injections2 == expected_empty);
    BOOST_REQUIRE_NO_THROW(errinj.inject("first", [] { throw std::runtime_error("test"); }));

    return make_ready_future<>();
}