Merge 'test.py: add support for CQL tests' from Kostja

This patch set adds support for CQL tests to test.py,
as well as many other improvements:

* --name is now a positional argument
* test output is preserved in testlog/${mode}
* concise output format
* better color support
* arbitrary number of test suites
* per-suite yaml-based configuration
* options --jenkins and --xunit are removed and xml
  files are generated for all runs

A simple driver is written in C++ to read CQL for
standard input, execute in embedded mode and produce output.

The patch is checked with BYO.

Reviewed-by: Dejan Mircevski <dejan@scylladb.com>
* 'test.py' of github.com:/scylladb/scylla-dev: (39 commits)
  test.py: introduce BoostTest and virtualize custom boost arguments
  test.py: sort tests within a suite, and sort suites
  test.py: add a basic CQL test
  test.py: add CQL .reject files to gitignore
  test.py: print a colored unidiff in case of test failure
  test.py: add CqlTestSuite to run CQL tests
  test.py: initial import of CQL test driver, cql_repl
  test.py: remove custom colors and define a color palette
  test.py: split test output per test mode
  test.py: remove tests_to_run
  test.py: virtualize Test.run(), to introduce CqlTest.Run next
  test.py: virtualize test search pattern per TestSuite
  test.py: virtualize write_xunit_report()
  test.py: ensure print_summary() is agnostic of test type
  test.py: tidy up print_summary()
  test.py: introduce base class Test for CQL and Unit tests
  test.py: move the default arguments handling to UnitTestSuite
  test.py: move custom unit test command line arguments to suite.yaml
  test.py: move command line argument processing to UnitTestSuite
  test.py: introduce add_test(), which is suite-specific
  ...

test/boost/suite.yaml

@@ -0,0 +1,11 @@
+type: boost
+# Custom command line arguments for some of the tests
+custom_args:
+    mutation_reader_test:
+        - '-c3 -m2G'
+    sstable_test:
+        - '-c1 -m2G'
+    sstable_datafile_test:
+        - '-c1 -m2G'
+    sstable_3_x_test:
+        - '-c1 -m2G'

test/cql/lwt_test.cql

@@ -0,0 +1,5 @@
+create table t1 (a int primary key);
+insert into t1 (a) values (1);
+insert into t1 (a) values (2);
+select * from t1 allow filtering;
+drop table t1;

test/cql/lwt_test.result

@@ -0,0 +1,28 @@
+create table t1 (a int primary key);
+{
+	"status" : "ok"
+}
+insert into t1 (a) values (1);
+{
+	"status" : "ok"
+}
+insert into t1 (a) values (2);
+{
+	"status" : "ok"
+}
+select * from t1 allow filtering;
+{
+	"rows" : 
+	[
+		{
+			"a" : "1"
+		},
+		{
+			"a" : "2"
+		}
+	]
+}
+drop table t1;
+{
+	"status" : "ok"
+}

test/cql/suite.yaml

@@ -0,0 +1 @@
+type: CQL

test/tools/cql_repl.cc

@@ -0,0 +1,218 @@
+/*
+ * Copyright (C) 2019 ScyllaDB
+ */
+
+/*
+ * This file is part of Scylla.
+ *
+ * Scylla is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Scylla is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#include <filesystem>
+#include <regex>
+
+#include "test/lib/cql_test_env.hh"
+#include "test/lib/cql_assertions.hh"
+
+#include <seastar/core/future-util.hh>
+#include <seastar/core/sleep.hh>
+#include <seastar/core/app-template.hh>
+#include <seastar/core/reactor.hh>
+#include "transport/messages/result_message.hh"
+#include "types/user.hh"
+#include "types/map.hh"
+#include "types/list.hh"
+#include "types/set.hh"
+#include "db/config.hh"
+#include "cql3/cql_config.hh"
+#include "cql3/type_json.hh"
+#include "test/lib/exception_utils.hh"
+
+static std::ofstream std_cout;
+
+//
+// A helper class to serialize result set output to a formatted JSON
+//
+class json_visitor final : public cql_transport::messages::result_message::visitor {
+    Json::Value& _root;
+public:
+    json_visitor(Json::Value& root)
+        : _root(root)
+    {
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::void_message&) override {
+        _root["status"] = "ok";
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::set_keyspace& m) override {
+        _root["status"] = "ok";
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::prepared::cql& m) override {
+        _root["status"] = "ok";
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::prepared::thrift& m) override {
+        assert(false);
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::schema_change& m) override {
+        _root["status"] = "ok";
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::bounce_to_shard& m) override {
+        assert(false);
+    }
+
+    virtual void visit(const cql_transport::messages::result_message::rows& m) override {
+        Json::Value& output_rows = _root["rows"];
+        const auto input_rows = m.rs().result_set().rows();
+        const auto& meta = m.rs().result_set().get_metadata().get_names();
+        for (auto&& in_row: input_rows) {
+            Json::Value out_row;
+            for (unsigned i = 0; i < meta.size(); ++i) {
+                const cql3::column_specification& col = *meta[i];
+                const bytes_opt& cell = in_row[i];
+                if (cell.has_value()) {
+                    out_row[col.name->text()] = fmt::format("{}", to_json_string(*col.type, cell));
+                }
+            }
+            output_rows.append(out_row);
+        }
+    }
+};
+
+// Prepare query_options with serial consistency
+std::unique_ptr<cql3::query_options> repl_options() {
+    const auto& so = cql3::query_options::specific_options::DEFAULT;
+    auto qo = std::make_unique<cql3::query_options>(
+            db::consistency_level::ONE,
+            infinite_timeout_config,
+            std::vector<cql3::raw_value>{},
+            // Ensure (optional) serial consistency is always specified.
+            cql3::query_options::specific_options{
+                so.page_size,
+                so.state,
+                db::consistency_level::SERIAL,
+                so.timestamp,
+            }
+    );
+    return qo;
+}
+
+// Read-evaluate-print-loop for CQL
+void repl(seastar::app_template& app) {
+    do_with_cql_env_thread([] (cql_test_env& e) {
+
+        // Comments allowed by CQL - -- and //
+        const std::regex comment_re("^[[:space:]]*((--|//).*)?$");
+        // A comment is not a delimiter even if ends with one
+        const std::regex delimiter_re("^(?![[:space:]]*(--|//)).*;[[:space:]]*$");
+
+        while (std::cin) {
+            std::string line;
+            std::ostringstream stmt;
+            if (!std::getline(std::cin, line)) {
+                break;
+            }
+            // Handle multiline input and comments
+            if (std::regex_match(line.begin(), line.end(), comment_re)) {
+                std_cout << line << std::endl;
+                continue;
+            }
+            stmt << line << std::endl;
+            while (!std::regex_match(line.begin(), line.end(), delimiter_re)) {
+                // Read the rest of input until delimiter or EOF
+                if (!std::getline(std::cin, line)) {
+                    break;
+                }
+                stmt << line << std::endl;
+            }
+            // Print the statement
+            std_cout << stmt.str();
+            Json::Value json;
+            try {
+                auto qo = repl_options();
+                auto msg = e.execute_cql(stmt.str(), std::move(qo)).get0();
+                json_visitor visitor(json);
+                msg->accept(visitor);
+            } catch (std::exception& e) {
+                json["status"] = "error";
+                json["message"] = fmt::format("{}", e);
+            }
+            std_cout << json << std::endl;
+        }
+    }).get0();
+}
+
+// Reset stdin/stdout/log streams to locations pointed
+// on the command line.
+void apply_configuration(const boost::program_options::variables_map& cfg) {
+
+    if (cfg.count("input")) {
+        static std::ifstream input(cfg["input"].as<std::string>());
+        std::cin.rdbuf(input.rdbuf());
+    }
+    static std::ofstream log(cfg["log"].as<std::string>());
+    // Seastar always logs to std::cout, hack this around
+    // by redirecting std::cout to a file and capturing
+    // the old std::cout in std_cout
+    auto save_filebuf = std::cout.rdbuf(log.rdbuf());
+    if (cfg.count("output")) {
+        std_cout.open(cfg["output"].as<std::string>());
+    } else  {
+        std_cout.std::ios::rdbuf(save_filebuf);
+    }
+}
+
+int main(int argc, char* argv[]) {
+
+    namespace bpo = boost::program_options;
+    namespace fs = std::filesystem;
+
+    seastar::app_template::config cfg;
+    cfg.name = fmt::format(R"({} - An embedded single-node version of Scylla.
+
+Runs read-evaluate-print loop, reading commands from stdin,
+evaluating them and printing output, formatted as JSON, to stdout.
+Creates a temporary database in /tmp and deletes it at exit.
+Pre-configures a default keyspace, naturally, with replication
+factor 1.
+
+Used in unit tests as a test driver for .test.cql files.
+
+Available )", argv[0]);
+
+    seastar::app_template app(cfg);
+
+    /* Define options for input, output and log file. */
+    app.add_options()
+        ("input", bpo::value<std::string>(),
+         "Input file with CQL, defaults to stdin")
+        ("output", bpo::value<std::string>(),
+         "Output file for data, defaults to stdout")
+        ("log", bpo::value<std::string>()->default_value(
+                    fmt::format("{}.log", fs::path(argv[0]).stem().string())),
+        "Output file for Scylla log");
+
+    return app.run(argc, argv, [&app] {
+
+        apply_configuration(app.configuration());
+
+        return seastar::async([&app] {
+            return repl(app);
+        });
+    });
+}
+

test/unit/suite.yaml

@@ -0,0 +1,20 @@
+# Suite test type. Supported types: unit, boost, cql
+type: unit
+# A list of long tests, these are only run in dev and release modes
+long:
+    - lsa_async_eviction_test
+    - lsa_sync_eviction_test
+    - row_cache_alloc_stress_test
+    - row_cache_stress_test
+# Custom command line arguments for some of the tests
+custom_args:
+    lsa_async_eviction_test:
+        - '-c1 -m200M --size 1024 --batch 3000 --count 2000000'
+    lsa_sync_eviction_test:
+        - '-c1 -m100M --count 10 --standard-object-size 3000000'
+        - '-c1 -m100M --count 24000 --standard-object-size 2048'
+        - '-c1 -m1G --count 4000000 --standard-object-size 128'
+    row_cache_alloc_stress_test:
+        - '-c1 -m2G'
+    row_cache_stress_test:
+        - '-c1 -m1G --seconds 10'