diff --git a/.github/workflows/codespell.yaml b/.github/workflows/codespell.yaml index 91f70628ae..12e9bb7422 100644 --- a/.github/workflows/codespell.yaml +++ b/.github/workflows/codespell.yaml @@ -14,4 +14,4 @@ jobs: with: only_warn: 1 ignore_words_list: "ans,datas,fo,ser,ue,crate,nd,reenable,strat,stap,te,raison" - skip: "./.git,./build,./tools,*.js,*.thrift,*.lock,./test,./licenses,./redis/lolwut.cc,*.svg" + skip: "./.git,./build,./tools,*.js,*.lock,./test,./licenses,./redis/lolwut.cc,*.svg" diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f3aa11489..cc417edec0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,7 +100,6 @@ find_package(libdeflate REQUIRED) find_package(libxcrypt REQUIRED) find_package(Snappy REQUIRED) find_package(RapidJSON REQUIRED) -find_package(Thrift REQUIRED) find_package(xxHash REQUIRED) set(scylla_gen_build_dir "${CMAKE_BINARY_DIR}/gen") @@ -198,7 +197,6 @@ add_subdirectory(dht) add_subdirectory(gms) add_subdirectory(idl) add_subdirectory(index) -add_subdirectory(interface) add_subdirectory(lang) add_subdirectory(locator) add_subdirectory(message) @@ -216,7 +214,6 @@ add_subdirectory(service) add_subdirectory(sstables) add_subdirectory(streaming) add_subdirectory(test) -add_subdirectory(thrift) add_subdirectory(tools) add_subdirectory(tracing) add_subdirectory(transport) @@ -257,7 +254,6 @@ target_link_libraries(scylla PRIVATE sstables streaming test-perf - thrift tools tracing transport diff --git a/README.md b/README.md index a9ec81a9dd..6555c9f3bd 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,8 @@ $ ./tools/toolchain/dbuild ./build/release/scylla --help See [test.py manual](docs/dev/testing.md). ## Scylla APIs and compatibility -By default, Scylla is compatible with Apache Cassandra and its APIs - CQL and -Thrift. There is also support for the API of Amazon DynamoDB™, +By default, Scylla is compatible with Apache Cassandra and its API - CQL. +There is also support for the API of Amazon DynamoDB™, which needs to be enabled and configured in order to be used. For more information on how to enable the DynamoDB™ API in Scylla, and the current compatibility of this feature as well as Scylla-specific extensions, see diff --git a/alternator/executor.cc b/alternator/executor.cc index 025610fb13..c37f4c6515 100644 --- a/alternator/executor.cc +++ b/alternator/executor.cc @@ -1251,7 +1251,7 @@ future executor::update_table(client_state& clien auto schema = builder.build(); - auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema, false, std::vector(), group0_guard.write_timestamp()); + auto m = co_await service::prepare_column_family_update_announcement(p.local(), schema, std::vector(), group0_guard.write_timestamp()); co_await mm.announce(std::move(m), std::move(group0_guard), format("alternator-executor: update {} table", tab->cf_name())); diff --git a/api/api-doc/collectd.json b/api/api-doc/collectd.json index 876cf9336d..a0f964bb06 100644 --- a/api/api-doc/collectd.json +++ b/api/api-doc/collectd.json @@ -67,7 +67,7 @@ "parameters":[ { "name":"pluginid", - "description":"The plugin ID, describe the component the metric belongs to. Examples are cache, thrift, etc'. Regex are supported.The plugin ID, describe the component the metric belong to. Examples are: cache, thrift etc'. regex are supported", + "description":"The plugin ID, describe the component the metric belongs to. Examples are cache and alternator, etc'. Regex are supported.", "required":true, "allowMultiple":false, "type":"string", @@ -199,4 +199,4 @@ } } } -} \ No newline at end of file +} diff --git a/api/api-doc/storage_service.json b/api/api-doc/storage_service.json index e63e26e47a..f0f11d68d3 100644 --- a/api/api-doc/storage_service.json +++ b/api/api-doc/storage_service.json @@ -1689,33 +1689,11 @@ { "path":"/storage_service/rpc_server", "operations":[ - { - "method":"DELETE", - "summary":"Allows a user to disable thrift", - "type":"void", - "nickname":"stop_rpc_server", - "produces":[ - "application/json" - ], - "parameters":[ - ] - }, - { - "method":"POST", - "summary":"allows a user to re-enable thrift", - "type":"void", - "nickname":"start_rpc_server", - "produces":[ - "application/json" - ], - "parameters":[ - ] - }, { "method":"GET", "summary":"Determine if thrift is running", "type":"boolean", - "nickname":"is_rpc_server_running", + "nickname":"is_thrift_server_running", "produces":[ "application/json" ], @@ -2070,7 +2048,7 @@ "operations":[ { "method":"POST", - "summary":"Enables/Disables tracing for the whole system. Only thrift requests can start tracing currently", + "summary":"Enables/Disables tracing for the whole system.", "type":"void", "nickname":"set_trace_probability", "produces":[ diff --git a/api/api.cc b/api/api.cc index 8d49681ed4..486b7c565e 100644 --- a/api/api.cc +++ b/api/api.cc @@ -100,12 +100,12 @@ future<> unset_transport_controller(http_context& ctx) { return ctx.http_server.set_routes([&ctx] (routes& r) { unset_transport_controller(ctx, r); }); } -future<> set_rpc_controller(http_context& ctx, thrift_controller& ctl) { - return ctx.http_server.set_routes([&ctx, &ctl] (routes& r) { set_rpc_controller(ctx, r, ctl); }); +future<> set_thrift_controller(http_context& ctx) { + return ctx.http_server.set_routes([&ctx] (routes& r) { set_thrift_controller(ctx, r); }); } -future<> unset_rpc_controller(http_context& ctx) { - return ctx.http_server.set_routes([&ctx] (routes& r) { unset_rpc_controller(ctx, r); }); +future<> unset_thrift_controller(http_context& ctx) { + return ctx.http_server.set_routes([&ctx] (routes& r) { unset_thrift_controller(ctx, r); }); } future<> set_server_storage_service(http_context& ctx, sharded& ss, service::raft_group0_client& group0_client) { diff --git a/api/api_init.hh b/api/api_init.hh index 51e3de7800..53fea716b6 100644 --- a/api/api_init.hh +++ b/api/api_init.hh @@ -46,7 +46,6 @@ class snitch_ptr; } // namespace locator namespace cql_transport { class controller; } -class thrift_controller; namespace db { class snapshot_ctl; class config; @@ -100,8 +99,8 @@ future<> set_server_repair(http_context& ctx, sharded& repair); future<> unset_server_repair(http_context& ctx); future<> set_transport_controller(http_context& ctx, cql_transport::controller& ctl); future<> unset_transport_controller(http_context& ctx); -future<> set_rpc_controller(http_context& ctx, thrift_controller& ctl); -future<> unset_rpc_controller(http_context& ctx); +future<> set_thrift_controller(http_context& ctx); +future<> unset_thrift_controller(http_context& ctx); future<> set_server_authorization_cache(http_context& ctx, sharded &auth_service); future<> unset_server_authorization_cache(http_context& ctx); future<> set_server_snapshot(http_context& ctx, sharded& snap_ctl); diff --git a/api/storage_service.cc b/api/storage_service.cc index 94b90cc693..05dde9a6e7 100644 --- a/api/storage_service.cc +++ b/api/storage_service.cc @@ -48,7 +48,6 @@ #include "db/extensions.hh" #include "db/snapshot-ctl.hh" #include "transport/controller.hh" -#include "thrift/controller.hh" #include "locator/token_metadata.hh" #include "cdc/generation_service.hh" #include "locator/abstract_replication_strategy.hh" @@ -337,36 +336,17 @@ void unset_transport_controller(http_context& ctx, routes& r) { ss::is_native_transport_running.unset(r); } -void set_rpc_controller(http_context& ctx, routes& r, thrift_controller& ctl) { - ss::stop_rpc_server.set(r, [&ctl](std::unique_ptr req) { - return smp::submit_to(0, [&] { - return ctl.request_stop_server(); - }).then([] { - return make_ready_future(json_void()); - }); - }); - - ss::start_rpc_server.set(r, [&ctl](std::unique_ptr req) { - return smp::submit_to(0, [&] { - return ctl.start_server(); - }).then([] { - return make_ready_future(json_void()); - }); - }); - - ss::is_rpc_server_running.set(r, [&ctl] (std::unique_ptr req) { - return smp::submit_to(0, [&] { - return !ctl.listen_addresses().empty(); - }).then([] (bool running) { - return make_ready_future(running); +// NOTE: preserved only for backward compatibility +void set_thrift_controller(http_context& ctx, routes& r) { + ss::is_thrift_server_running.set(r, [] (std::unique_ptr req) { + return smp::submit_to(0, [] { + return make_ready_future(false); }); }); } -void unset_rpc_controller(http_context& ctx, routes& r) { - ss::stop_rpc_server.unset(r); - ss::start_rpc_server.unset(r); - ss::is_rpc_server_running.unset(r); +void unset_thrift_controller(http_context& ctx, routes& r) { + ss::is_thrift_server_running.unset(r); } void set_repair(http_context& ctx, routes& r, sharded& repair) { diff --git a/api/storage_service.hh b/api/storage_service.hh index 3de65e4d0d..72ab6201ec 100644 --- a/api/storage_service.hh +++ b/api/storage_service.hh @@ -14,7 +14,6 @@ #include "db/data_listeners.hh" namespace cql_transport { class controller; } -class thrift_controller; namespace db { class snapshot_ctl; namespace view { @@ -80,8 +79,8 @@ void set_repair(http_context& ctx, httpd::routes& r, sharded& re void unset_repair(http_context& ctx, httpd::routes& r); void set_transport_controller(http_context& ctx, httpd::routes& r, cql_transport::controller& ctl); void unset_transport_controller(http_context& ctx, httpd::routes& r); -void set_rpc_controller(http_context& ctx, httpd::routes& r, thrift_controller& ctl); -void unset_rpc_controller(http_context& ctx, httpd::routes& r); +void set_thrift_controller(http_context& ctx, httpd::routes& r); +void unset_thrift_controller(http_context& ctx, httpd::routes& r); void set_snapshot(http_context& ctx, httpd::routes& r, sharded& snap_ctl); void unset_snapshot(http_context& ctx, httpd::routes& r); seastar::future run_toppartitions_query(db::toppartitions_query& q, http_context &ctx, bool legacy_request = false); diff --git a/cdc/log.cc b/cdc/log.cc index 3e23d2a052..d90fd396a8 100644 --- a/cdc/log.cc +++ b/cdc/log.cc @@ -207,7 +207,7 @@ public: auto new_log_schema = create_log_schema(new_schema, log_schema ? std::make_optional(log_schema->id()) : std::nullopt, log_schema); auto log_mut = log_schema - ? db::schema_tables::make_update_table_mutations(db, keyspace.metadata(), log_schema, new_log_schema, timestamp, false) + ? db::schema_tables::make_update_table_mutations(db, keyspace.metadata(), log_schema, new_log_schema, timestamp) : db::schema_tables::make_create_table_mutations(new_log_schema, timestamp) ; diff --git a/client_data.hh b/client_data.hh index 3134737b0c..36515ef3da 100644 --- a/client_data.hh +++ b/client_data.hh @@ -33,7 +33,7 @@ sstring to_string(client_connection_stage ct); struct client_data { net::inet_address ip; int32_t port; - client_type ct; + client_type ct = client_type::cql; client_connection_stage connection_stage = client_connection_stage::established; int32_t shard_id; /// ID of server-side shard which is processing the connection. diff --git a/cmake/FindThrift.cmake b/cmake/FindThrift.cmake deleted file mode 100644 index e3bda48b4c..0000000000 --- a/cmake/FindThrift.cmake +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright 2023-present ScyllaDB -# - -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -find_package(PkgConfig REQUIRED) - -pkg_check_modules(PC_thrift QUIET thrift) - -find_library(thrift_LIBRARY - NAMES thrift - HINTS - ${PC_thrift_LIBDIR} - ${PC_thrift_LIBRARY_DIRS}) - -find_path(thrift_INCLUDE_DIR - NAMES thrift/Thrift.h - HINTS - ${PC_thrift_INCLUDEDIR} - ${PC_thrift_INCLUDE_DIRS}) - -mark_as_advanced( - thrift_LIBRARY - thrift_INCLUDE_DIR) - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(Thrift - REQUIRED_VARS - thrift_LIBRARY - thrift_INCLUDE_DIR - VERSION_VAR PC_thrift_VERSION) - -if(Thrift_FOUND) - set(thrift_LIBRARIES ${thrift_LIBRARY}) - set(thrift_INCLUDE_DIRS ${thrift_INCLUDE_DIR}) - if(NOT(TARGET Thrift::thrift)) - add_library(Thrift::thrift UNKNOWN IMPORTED) - - set_target_properties(Thrift::thrift - PROPERTIES - IMPORTED_LOCATION ${thrift_LIBRARY} - INTERFACE_INCLUDE_DIRECTORIES ${thrift_INCLUDE_DIRS}) - endif() -endif() diff --git a/conf/scylla.yaml b/conf/scylla.yaml index 9feb353827..516da9e664 100644 --- a/conf/scylla.yaml +++ b/conf/scylla.yaml @@ -199,8 +199,7 @@ cas_contention_timeout_in_ms: 1000 # of the snitch, which will be assumed to be on your classpath. endpoint_snitch: SimpleSnitch -# The address or interface to bind the Thrift RPC service and native transport -# server to. +# The address or interface to bind the native transport server to. # # Set rpc_address OR rpc_interface, not both. Interfaces must correspond # to a single address, IP aliasing is not supported. @@ -221,9 +220,6 @@ rpc_address: localhost # rpc_interface: eth1 # rpc_interface_prefer_ipv6: false -# port for Thrift to listen for clients on -rpc_port: 9160 - # port for REST API server api_port: 10000 @@ -356,9 +352,6 @@ commitlog_total_space_in_mb: -1 # be rejected as invalid. The default is 256MB. # native_transport_max_frame_size_in_mb: 256 -# Whether to start the thrift rpc server. -# start_rpc: true - # enable or disable keepalive on rpc/native connections # rpc_keepalive: true diff --git a/configure.py b/configure.py index 6cbc0cf49f..29402dddec 100755 --- a/configure.py +++ b/configure.py @@ -203,18 +203,6 @@ class Source(object): def endswith(self, end): return self.source.endswith(end) -class Thrift(Source): - def __init__(self, source, service): - Source.__init__(self, source, '.h', '.cpp') - self.service = service - - def generated(self, gen_dir): - basename = os.path.splitext(os.path.basename(self.source))[0] - files = [basename + '_' + ext - for ext in ['types.cpp', 'types.h', 'constants.cpp', 'constants.h']] - files += [self.service + ext - for ext in ['.cpp', '.h']] - return [os.path.join(gen_dir, file) for file in files] def default_target_arch(): if platform.machine() in ['i386', 'i686', 'x86_64']: @@ -370,18 +358,6 @@ def check_for_lz4(cxx, cflags): sys.exit(1) -def thrift_uses_boost_share_ptr(): - # thrift version detection, see #4538 - proc_res = subprocess.run(["thrift", "-version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - proc_res_output = proc_res.stdout.decode("utf-8") - if proc_res.returncode != 0 and not re.search(r'^Thrift version', proc_res_output): - raise Exception("Thrift compiler must be missing: {}".format(proc_res_output)) - - thrift_version = proc_res_output.split(" ")[-1] - thrift_boost_versions = ["0.{}.".format(n) for n in range(1, 11)] - return any(filter(thrift_version.startswith, thrift_boost_versions)) - - def find_ninja(): ninja = which('ninja') or which('ninja-build') if ninja: @@ -742,8 +718,6 @@ arg_parser.add_argument('--optimization-level', action='append', dest='mode_o_le help=f'Override default compiler optimization level for mode (defaults: {" ".join([x+"="+modes[x]["optimization-level"] for x in modes])})') arg_parser.add_argument('--static-stdc++', dest='staticcxx', action='store_true', help='Link libgcc and libstdc++ statically') -arg_parser.add_argument('--static-thrift', dest='staticthrift', action='store_true', - help='Link libthrift statically') arg_parser.add_argument('--static-boost', dest='staticboost', action='store_true', help='Link boost statically') arg_parser.add_argument('--static-yaml-cpp', dest='staticyamlcpp', action='store_true', @@ -982,10 +956,6 @@ scylla_core = (['message/messaging_service.cc', 'cql3/ut_name.cc', 'cql3/role_name.cc', 'data_dictionary/data_dictionary.cc', - 'thrift/handler.cc', - 'thrift/server.cc', - 'thrift/controller.cc', - 'thrift/thrift_validation.cc', 'utils/runtime.cc', 'utils/murmur_hash.cc', 'utils/uuid.cc', @@ -1204,7 +1174,7 @@ scylla_core = (['message/messaging_service.cc', 'service/topology_mutation.cc', 'service/topology_coordinator.cc', 'node_ops/node_ops_ctl.cc' - ] + [Antlr3Grammar('cql3/Cql.g')] + [Thrift('interface/cassandra.thrift', 'Cassandra')] \ + ] + [Antlr3Grammar('cql3/Cql.g')] \ + scylla_raft_core ) @@ -1849,9 +1819,6 @@ libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-l if not args.staticboost: user_cflags += ' -DBOOST_ALL_DYN_LINK' -if thrift_uses_boost_share_ptr(): - user_cflags += ' -DTHRIFT_USES_BOOST' - for pkg in pkgs: user_cflags += ' ' + pkg_config(pkg, '--cflags') libs += ' ' + pkg_config(pkg, '--libs') @@ -2035,10 +2002,6 @@ def write_build_file(f, rule ar.{mode} command = rm -f $out; ar cr $out $in; ranlib $out description = AR $out - rule thrift.{mode} - command = thrift -gen cpp:cob_style -out $builddir/{mode}/gen $in - description = THRIFT $in - restat = 1 rule antlr3.{mode} # We replace many local `ExceptionBaseType* ex` variables with a single function-scope one. # Because we add such a variable to every function, and because `ExceptionBaseType` is not a global @@ -2082,7 +2045,6 @@ def write_build_file(f, compiles = {} swaggers = set() serializers = {} - thrifts = set() ragels = {} antlr3_grammars = set() rust_headers = {} @@ -2098,12 +2060,8 @@ def write_build_file(f, for src in srcs if src.endswith('.cc')] objs.append('$builddir/../utils/arch/powerpc/crc32-vpmsum/crc32.S') - has_thrift = False has_rust = False for dep in deps[binary]: - if isinstance(dep, Thrift): - has_thrift = True - objs += dep.objects('$builddir/' + mode + '/gen') if isinstance(dep, Antlr3Grammar): objs += dep.objects(f'$builddir/{mode}/gen') if isinstance(dep, Json2Code): @@ -2116,9 +2074,6 @@ def write_build_file(f, if has_rust: objs.append(f'$builddir/{mode}/rust-{mode}/librust_combined.a') local_libs = f'$seastar_libs_{mode} $libs' - if has_thrift: - local_libs += ' ' + maybe_static(args.staticthrift, '-lthrift') - local_libs += ' ' + maybe_static(args.staticboost, '-lboost_system') objs.extend([f'$builddir/{mode}/abseil/{lib}' for lib in abseil_libs]) if binary in tests: if binary in pure_boost_tests: @@ -2156,8 +2111,6 @@ def write_build_file(f, elif src.endswith('.rl'): hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh') ragels[hh] = src - elif src.endswith('.thrift'): - thrifts.add(src) elif src.endswith('.g'): antlr3_grammars.add(src) elif src.endswith('.rs'): @@ -2207,8 +2160,6 @@ def write_build_file(f, gen_dir = '$builddir/{}/gen'.format(mode) gen_headers = [] - for th in thrifts: - gen_headers += th.headers('$builddir/{}/gen'.format(mode)) for g in antlr3_grammars: gen_headers += g.headers('$builddir/{}/gen'.format(mode)) for g in swaggers: @@ -2247,12 +2198,6 @@ def write_build_file(f, f.write('build {}: cxxbridge_header\n'.format('$builddir/{}/gen/rust/cxx.h'.format(mode))) librust = '$builddir/{}/rust-{}/librust_combined'.format(mode, mode) f.write('build {}.a: rust_lib.{} rust/Cargo.lock\n depfile={}.d\n'.format(librust, mode, librust)) - for thrift in thrifts: - outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode))) - f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source)) - for cc in thrift.sources('$builddir/{}/gen'.format(mode)): - obj = cc.replace('.cpp', '.o') - f.write('build {}: cxx.{} {}\n'.format(obj, mode, cc)) for grammar in antlr3_grammars: outs = ' '.join(grammar.generated('$builddir/{}/gen'.format(mode))) f.write('build {}: antlr3.{} {}\n stem = {}\n'.format(outs, mode, grammar.source, diff --git a/cql3/authorized_prepared_statements_cache.hh b/cql3/authorized_prepared_statements_cache.hh index 71f6785a60..ef111abb33 100644 --- a/cql3/authorized_prepared_statements_cache.hh +++ b/cql3/authorized_prepared_statements_cache.hh @@ -59,7 +59,8 @@ public: bool operator==(const authorized_prepared_statements_cache_key&) const = default; static size_t hash(const auth::authenticated_user& user, const cql3::prepared_cache_key_type::cache_key_type& prep_cache_key) { - return utils::hash_combine(std::hash()(user), utils::tuple_hash()(prep_cache_key)); + return utils::hash_combine(std::hash()(user), + std::hash()(prep_cache_key)); } }; diff --git a/cql3/prepared_statements_cache.hh b/cql3/prepared_statements_cache.hh index 2a2fa85a83..ae84a54ff1 100644 --- a/cql3/prepared_statements_cache.hh +++ b/cql3/prepared_statements_cache.hh @@ -27,35 +27,30 @@ struct prepared_cache_entry_size { }; typedef bytes cql_prepared_id_type; -typedef int32_t thrift_prepared_id_type; /// \brief The key of the prepared statements cache /// -/// We are going to store the CQL and Thrift prepared statements in the same cache therefore we need generate the key -/// that is going to be unique in both cases. Thrift use int32_t as a prepared statement ID, CQL - MD5 digest. -/// -/// We are going to use an std::pair as a key. For CQL statements we will use {CQL_PREP_ID, std::numeric_limits::max()} as a key -/// and for Thrift - {CQL_PREP_ID_TYPE(0), THRIFT_PREP_ID}. This way CQL and Thrift keys' values will never collide. +/// TODO: consolidate prepared_cache_key_type and the nested cache_key_type +/// the latter was introduced for unifying the CQL and Thrift prepared +/// statements so that they can be stored in the same cache. class prepared_cache_key_type { public: - using cache_key_type = std::pair; + // derive from cql_prepared_id_type so we can customize the formatter of + // cache_key_type + struct cache_key_type : public cql_prepared_id_type {}; private: cache_key_type _key; public: prepared_cache_key_type() = default; - explicit prepared_cache_key_type(cql_prepared_id_type cql_id) : _key(std::move(cql_id), std::numeric_limits::max()) {} - explicit prepared_cache_key_type(thrift_prepared_id_type thrift_id) : _key(cql_prepared_id_type(), thrift_id) {} + explicit prepared_cache_key_type(cql_prepared_id_type cql_id) : _key(std::move(cql_id)) {} cache_key_type& key() { return _key; } const cache_key_type& key() const { return _key; } static const cql_prepared_id_type& cql_id(const prepared_cache_key_type& key) { - return key.key().first; - } - static thrift_prepared_id_type thrift_id(const prepared_cache_key_type& key) { - return key.key().second; + return key.key(); } bool operator==(const prepared_cache_key_type& other) const = default; @@ -98,7 +93,7 @@ private: // // Therefore a typical "pollution" (when a cache entry is used only once) would involve // 2 cache hits. - using cache_type = utils::loading_cache, prepared_cache_stats_updater, prepared_cache_stats_updater>; + using cache_type = utils::loading_cache, std::equal_to, prepared_cache_stats_updater, prepared_cache_stats_updater>; using cache_value_ptr = typename cache_type::value_ptr; using checked_weak_ptr = typename statements::prepared_statement::checked_weak_ptr; @@ -161,10 +156,18 @@ public: } namespace std { + +template<> +struct hash final { + size_t operator()(const cql3::prepared_cache_key_type::cache_key_type& k) const { + return std::hash()(k); + } +}; + template<> struct hash final { size_t operator()(const cql3::prepared_cache_key_type& k) const { - return utils::tuple_hash()(k.key()); + return std::hash()(k.key()); } }; } @@ -173,7 +176,7 @@ struct hash final { template <> struct fmt::formatter { constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } auto format(const cql3::prepared_cache_key_type::cache_key_type& p, fmt::format_context& ctx) const { - return fmt::format_to(ctx.out(), "{{cql_id: {}, thrift_id: {}}}", p.first, p.second); + return fmt::format_to(ctx.out(), "{{cql_id: {}}}", static_cast(p)); } }; diff --git a/cql3/query_processor.cc b/cql3/query_processor.cc index f5d9871c44..cb9a21a2e8 100644 --- a/cql3/query_processor.cc +++ b/cql3/query_processor.cc @@ -655,24 +655,17 @@ query_processor::process_authorized_statement(const ::shared_ptr future<::shared_ptr> query_processor::prepare(sstring query_string, service::query_state& query_state) { auto& client_state = query_state.get_client_state(); - return prepare(std::move(query_string), client_state, client_state.is_thrift()); + return prepare(std::move(query_string), client_state); } future<::shared_ptr> -query_processor::prepare(sstring query_string, const service::client_state& client_state, bool for_thrift) { +query_processor::prepare(sstring query_string, const service::client_state& client_state) { using namespace cql_transport::messages; - if (for_thrift) { - return prepare_one( - std::move(query_string), - client_state, - compute_thrift_id, prepared_cache_key_type::thrift_id); - } else { - return prepare_one( - std::move(query_string), - client_state, - compute_id, - prepared_cache_key_type::cql_id); - } + return prepare_one( + std::move(query_string), + client_state, + compute_id, + prepared_cache_key_type::cql_id); } static std::string hash_target(std::string_view query_string, std::string_view keyspace) { @@ -687,16 +680,6 @@ prepared_cache_key_type query_processor::compute_id( return prepared_cache_key_type(md5_hasher::calculate(hash_target(query_string, keyspace))); } -prepared_cache_key_type query_processor::compute_thrift_id( - const std::string_view& query_string, - const sstring& keyspace) { - uint32_t h = 0; - for (auto&& c : hash_target(query_string, keyspace)) { - h = 31*h + c; - } - return prepared_cache_key_type(static_cast(h)); -} - std::unique_ptr query_processor::get_statement(const sstring_view& query, const service::client_state& client_state) { std::unique_ptr statement = parse_statement(query); @@ -1069,22 +1052,6 @@ future<> query_processor::announce_schema_statement(const statements::schema_alt co_await remote_.get().mm.announce(std::move(m), std::move(guard), description); } -future -query_processor::execute_thrift_schema_command( - std::function>(data_dictionary::database, api::timestamp_type)> prepare_schema_mutations, - std::string_view description) { - assert(this_shard_id() == 0); - - auto [remote_, holder] = remote(); - auto& mm = remote_.get().mm; - auto group0_guard = co_await mm.start_group0_operation(); - auto ts = group0_guard.write_timestamp(); - - co_await mm.announce(co_await prepare_schema_mutations(db(), ts), std::move(group0_guard), description); - - co_return std::string(db().get_version().to_sstring()); -} - query_processor::migration_subscriber::migration_subscriber(query_processor* qp) : _qp{qp} { } diff --git a/cql3/query_processor.hh b/cql3/query_processor.hh index 5499e5263c..2a2e3a85e6 100644 --- a/cql3/query_processor.hh +++ b/cql3/query_processor.hh @@ -139,10 +139,6 @@ public: std::string_view query_string, std::string_view keyspace); - static prepared_cache_key_type compute_thrift_id( - const std::string_view& query_string, - const sstring& keyspace); - static std::unique_ptr parse_statement(const std::string_view& query); static std::vector> parse_statements(std::string_view queries); @@ -404,7 +400,7 @@ public: prepare(sstring query_string, service::query_state& query_state); future<::shared_ptr> - prepare(sstring query_string, const service::client_state& client_state, bool for_thrift); + prepare(sstring query_string, const service::client_state& client_state); future<> stop(); @@ -445,11 +441,6 @@ public: execute_schema_statement(const statements::schema_altering_statement&, service::query_state& state, const query_options& options, service::group0_batch& mc); future<> announce_schema_statement(const statements::schema_altering_statement&, service::group0_batch& mc); - future - execute_thrift_schema_command( - std::function>(data_dictionary::database, api::timestamp_type)> prepare_schema_mutations, - std::string_view description); - std::unique_ptr get_statement( const std::string_view& query, const service::client_state& client_state); @@ -520,10 +511,10 @@ private: ::shared_ptr statement, service::query_state& query_state, const query_options& options); /// - /// \tparam ResultMsgType type of the returned result message (CQL or Thrift) + /// \tparam ResultMsgType type of the returned result message (CQL) /// \tparam PreparedKeyGenerator a function that generates the prepared statement cache key for given query and /// keyspace - /// \tparam IdGetter a function that returns the corresponding prepared statement ID (CQL or Thrift) for a given + /// \tparam IdGetter a function that returns the corresponding prepared statement ID (CQL) for a given //// prepared statement cache key /// \param query_string /// \param client_state diff --git a/cql3/statements/alter_table_statement.cc b/cql3/statements/alter_table_statement.cc index 7934a62a8f..7f6648580b 100644 --- a/cql3/statements/alter_table_statement.cc +++ b/cql3/statements/alter_table_statement.cc @@ -402,7 +402,7 @@ future, std::vector alter_table_statement::prepare_schema_mutations(query_processor& qp, const query_options& options, api::timestamp_type ts) const { data_dictionary::database db = qp.db(); auto [cfm, view_updates] = prepare_schema_update(db, options); - auto m = co_await service::prepare_column_family_update_announcement(qp.proxy(), cfm.build(), false, std::move(view_updates), ts); + auto m = co_await service::prepare_column_family_update_announcement(qp.proxy(), cfm.build(), std::move(view_updates), ts); using namespace cql_transport; auto ret = ::make_shared( diff --git a/cql3/statements/alter_type_statement.cc b/cql3/statements/alter_type_statement.cc index 34fee325e9..4741f1a53e 100644 --- a/cql3/statements/alter_type_statement.cc +++ b/cql3/statements/alter_type_statement.cc @@ -88,7 +88,7 @@ future> alter_type_statement::prepare_announcement_mutatio auto res = co_await service::prepare_view_update_announcement(sp, view_ptr(cfm.build()), ts); std::move(res.begin(), res.end(), std::back_inserter(m)); } else { - auto res = co_await service::prepare_column_family_update_announcement(sp, cfm.build(), false, {}, ts); + auto res = co_await service::prepare_column_family_update_announcement(sp, cfm.build(), {}, ts); std::move(res.begin(), res.end(), std::back_inserter(m)); } } diff --git a/cql3/statements/alter_type_statement.hh b/cql3/statements/alter_type_statement.hh index 4153687993..0ad8a00a07 100644 --- a/cql3/statements/alter_type_statement.hh +++ b/cql3/statements/alter_type_statement.hh @@ -42,12 +42,6 @@ public: protected: virtual user_type make_updated_type(data_dictionary::database db, user_type to_update) const = 0; private: - struct base_visitor { - virtual future<> operator()(view_ptr view) = 0; - virtual future<> operator()(user_type type) = 0; - virtual future<> operator()(schema_ptr cfm, bool from_thrift, std::vector&& view_updates, std::optional ts_opt) = 0; - }; - future> prepare_announcement_mutations(service::storage_proxy& sp, api::timestamp_type) const; }; diff --git a/cql3/statements/batch_statement.hh b/cql3/statements/batch_statement.hh index b894f3073a..efee8aa354 100644 --- a/cql3/statements/batch_statement.hh +++ b/cql3/statements/batch_statement.hh @@ -70,8 +70,7 @@ private: cql_stats& _stats; public: /** - * Creates a new BatchStatement from a list of statements and a - * Thrift consistency level. + * Creates a new BatchStatement from a list of statements * * @param type type of the batch * @param statements a list of UpdateStatements diff --git a/cql3/statements/create_index_statement.cc b/cql3/statements/create_index_statement.cc index 4e2b9a4f59..2338c2b699 100644 --- a/cql3/statements/create_index_statement.cc +++ b/cql3/statements/create_index_statement.cc @@ -131,7 +131,7 @@ std::vector<::shared_ptr> create_index_statement::validate_while_e } // Origin TODO: we could lift that limitation - if ((schema->is_dense() || !schema->thrift().has_compound_comparator()) && cd->is_primary_key()) { + if ((schema->is_dense() || !schema->is_compound()) && cd->is_primary_key()) { throw exceptions::invalid_request_exception( "Secondary indexes are not supported on PRIMARY KEY columns in COMPACT STORAGE tables"); } @@ -382,7 +382,7 @@ create_index_statement::prepare_schema_mutations(query_processor& qp, const quer std::vector m; if (res) { - m = co_await service::prepare_column_family_update_announcement(qp.proxy(), std::move(res->schema), false, {}, ts); + m = co_await service::prepare_column_family_update_announcement(qp.proxy(), std::move(res->schema), {}, ts); ret = ::make_shared( event::schema_change::change_type::UPDATED, diff --git a/cql3/statements/drop_index_statement.cc b/cql3/statements/drop_index_statement.cc index 6b9d256d3f..e5b9312992 100644 --- a/cql3/statements/drop_index_statement.cc +++ b/cql3/statements/drop_index_statement.cc @@ -78,7 +78,7 @@ drop_index_statement::prepare_schema_mutations(query_processor& qp, const query_ auto cfm = make_drop_idex_schema(qp); if (cfm) { - m = co_await service::prepare_column_family_update_announcement(qp.proxy(), cfm, false, {}, ts); + m = co_await service::prepare_column_family_update_announcement(qp.proxy(), cfm, {}, ts); using namespace cql_transport; ret = ::make_shared(event::schema_change::change_type::UPDATED, diff --git a/cql3/statements/drop_keyspace_statement.cc b/cql3/statements/drop_keyspace_statement.cc index b75148c36e..3ad6cfa3f7 100644 --- a/cql3/statements/drop_keyspace_statement.cc +++ b/cql3/statements/drop_keyspace_statement.cc @@ -38,9 +38,6 @@ future<> drop_keyspace_statement::check_access(query_processor& qp, const servic void drop_keyspace_statement::validate(query_processor&, const service::client_state& state) const { warn(unimplemented::cause::VALIDATION); -#if 0 - ThriftValidation.validateKeyspaceNotSystem(keyspace); -#endif } const sstring& drop_keyspace_statement::keyspace() const diff --git a/cql3/statements/truncate_statement.cc b/cql3/statements/truncate_statement.cc index 9d723fa8ca..bb5d9d0148 100644 --- a/cql3/statements/truncate_statement.cc +++ b/cql3/statements/truncate_statement.cc @@ -84,9 +84,6 @@ future<> truncate_statement::check_access(query_processor& qp, const service::cl void truncate_statement::validate(query_processor&, const service::client_state& state) const { warn(unimplemented::cause::VALIDATION); -#if 0 - ThriftValidation.validateColumnFamily(keyspace(), columnFamily()); -#endif } future<::shared_ptr> diff --git a/db/config.cc b/db/config.cc index 9f652814e6..65e99a256f 100644 --- a/db/config.cc +++ b/db/config.cc @@ -416,16 +416,16 @@ db::config::config(std::shared_ptr exts) */ , commit_failure_policy(this, "commit_failure_policy", value_status::Unused, "stop", "Policy for commit disk failures:\n" - "* die Shut down gossip and Thrift and kill the JVM, so the node can be replaced.\n" - "* stop Shut down gossip and Thrift, leaving the node effectively dead, but can be inspected using JMX.\n" + "* die Shut down gossip, so the node can be replaced.\n" + "* stop Shut down gossip, leaving the node effectively dead, but can be inspected using the RESTful APIs.\n" "* stop_commit Shut down the commit log, letting writes collect but continuing to service reads (as in pre-2.0.5 Cassandra).\n" "* ignore Ignore fatal errors and let the batches fail." , {"die", "stop", "stop_commit", "ignore"}) , disk_failure_policy(this, "disk_failure_policy", value_status::Unused, "stop", "Sets how Scylla responds to disk failure. Recommend settings are stop or best_effort.\n" - "* die Shut down gossip and Thrift and kill the JVM for any file system errors or single SSTable errors, so the node can be replaced.\n" - "* stop_paranoid Shut down gossip and Thrift even for single SSTable errors.\n" - "* stop Shut down gossip and Thrift, leaving the node effectively dead, but available for inspection using JMX.\n" + "* die Shut down gossip for any file system errors or single SSTable errors, so the node can be replaced.\n" + "* stop_paranoid Shut down gossip even for single SSTable errors.\n" + "* stop Shut down gossip, leaving the node effectively dead, but available for inspection using the RESTful APIs.\n" "* best_effort Stop using the failed disk and respond to requests based on the remaining available SSTables. This means you will see obsolete data at consistency level of ONE.\n" "* ignore Ignores fatal errors and lets the requests fail; all file system errors are logged but otherwise ignored. Scylla acts as in versions prior to Cassandra 1.2.\n" "Related information: Handling Disk Failures In Cassandra 1.2 blog and Recovering from a single disk failure using JBOD.\n" @@ -442,7 +442,7 @@ db::config::config(std::shared_ptr exts) "\n" "Related information: Snitches\n") , rpc_address(this, "rpc_address", value_status::Used, "localhost", - "The listen address for client connections (Thrift RPC service and native transport).Valid values are:\n" + "The listen address for client connections (native transport).Valid values are:\n" "* unset: Resolves the address using the hostname configuration of the node. If left unset, the hostname must resolve to the IP address of this node using /etc/hostname, /etc/hosts, or DNS.\n" "* 0.0.0.0: Listens on all configured interfaces, but you must set the broadcast_rpc_address to a value other than 0.0.0.0.\n" "* IP address\n" @@ -799,26 +799,12 @@ db::config::config(std::shared_ptr exts) */ , broadcast_rpc_address(this, "broadcast_rpc_address", value_status::Used, {/* unset */}, "RPC address to broadcast to drivers and other Scylla nodes. This cannot be set to 0.0.0.0. If blank, it is set to the value of the rpc_address or rpc_interface. If rpc_address or rpc_interfaceis set to 0.0.0.0, this property must be set.\n") - , rpc_port(this, "rpc_port", "thrift_port", value_status::Used, 9160, + , rpc_port(this, "rpc_port", "thrift_port", value_status::Unused, 0, "Thrift port for client connections.") - , start_rpc(this, "start_rpc", value_status::Used, false, + , start_rpc(this, "start_rpc", value_status::Unused, false, "Starts the Thrift RPC server") , rpc_keepalive(this, "rpc_keepalive", value_status::Used, true, - "Enable or disable keepalive on client connections (RPC or native).") - , rpc_max_threads(this, "rpc_max_threads", value_status::Invalid, 0, - "Regardless of your choice of RPC server (rpc_server_type), the number of maximum requests in the RPC thread pool dictates how many concurrent requests are possible. However, if you are using the parameter sync in the rpc_server_type, it also dictates the number of clients that can be connected. For a large number of client connections, this could cause excessive memory usage for the thread stack. Connection pooling on the client side is highly recommended. Setting a maximum thread pool size acts as a safeguard against misbehaved clients. If the maximum is reached, Cassandra blocks additional connections until a client disconnects.") - , rpc_min_threads(this, "rpc_min_threads", value_status::Invalid, 16, - "Sets the minimum thread pool size for remote procedure calls.") - , rpc_recv_buff_size_in_bytes(this, "rpc_recv_buff_size_in_bytes", value_status::Unused, 0, - "Sets the receiving socket buffer size for remote procedure calls.") - , rpc_send_buff_size_in_bytes(this, "rpc_send_buff_size_in_bytes", value_status::Unused, 0, - "Sets the sending socket buffer size in bytes for remote procedure calls.") - , rpc_server_type(this, "rpc_server_type", value_status::Unused, "sync", - "Cassandra provides three options for the RPC server. On Windows, sync is about 30% slower than hsha. On Linux, sync and hsha performance is about the same, but hsha uses less memory.\n" - "* sync (Default One thread per Thrift connection.) For a very large number of clients, memory is the limiting factor. On a 64-bit JVM, 180KB is the minimum stack size per thread and corresponds to your use of virtual memory. Physical memory may be limited depending on use of stack space.\n" - "* hsh Half synchronous, half asynchronous. All Thrift clients are handled asynchronously using a small number of threads that does not vary with the number of clients and thus scales well to many clients. The RPC requests are synchronous (one thread per active request).\n" - "* Note: When selecting this option, you must change the default value (unlimited) of rpc_max_threads.\n" - "* Your own RPC server: You must provide a fully-qualified class name of an o.a.c.t.TServerFactory that can create a server instance.") + "Enable or disable keepalive on client connections (CQL native, Redis and the maintenance socket).") , cache_hit_rate_read_balancing(this, "cache_hit_rate_read_balancing", value_status::Used, true, "This boolean controls whether the replicas for read query will be chosen based on cache hit ratio.") /** @@ -865,14 +851,6 @@ db::config::config(std::shared_ptr exts) "* default_weight: (Default: 1 **) How many requests are handled during each turn of the RoundRobin.\n" "* weights: (Default: Keyspace: 1) Takes a list of keyspaces. It sets how many requests are handled during each turn of the RoundRobin, based on the request_scheduler_id.") /** - * @Group Thrift interface properties - * @GroupDescription Legacy API for older clients. CQL is a simpler and better API for Scylla. - */ - , thrift_framed_transport_size_in_mb(this, "thrift_framed_transport_size_in_mb", value_status::Unused, 15, - "Frame size (maximum field length) for Thrift. The frame is the row or part of the row the application is inserting.") - , thrift_max_message_length_in_mb(this, "thrift_max_message_length_in_mb", value_status::Used, 16, - "The maximum length of a Thrift message in megabytes, including all fields and internal Thrift overhead (1 byte of overhead for each frame). Message length is usually used in conjunction with batches. A frame length greater than or equal to 24 accommodates a batch with four inserts, each of which is 24 bytes. The required message length is greater than or equal to 24+24+24+24+4 (number of frames).") - /** * @Group Security properties * @GroupDescription Server and client security settings. */ diff --git a/db/config.hh b/db/config.hh index 347405f149..2e4d5fc376 100644 --- a/db/config.hh +++ b/db/config.hh @@ -293,11 +293,6 @@ public: named_value rpc_port; named_value start_rpc; named_value rpc_keepalive; - named_value rpc_max_threads; - named_value rpc_min_threads; - named_value rpc_recv_buff_size_in_bytes; - named_value rpc_send_buff_size_in_bytes; - named_value rpc_server_type; named_value cache_hit_rate_read_balancing; named_value dynamic_snitch_badness_threshold; named_value dynamic_snitch_reset_interval_in_ms; @@ -311,8 +306,6 @@ public: named_value request_scheduler; named_value request_scheduler_id; named_value request_scheduler_options; - named_value thrift_framed_transport_size_in_mb; - named_value thrift_max_message_length_in_mb; named_value authenticator; named_value internode_authenticator; named_value authorizer; diff --git a/db/schema_tables.cc b/db/schema_tables.cc index fc9e191ac5..dccc92db96 100644 --- a/db/schema_tables.cc +++ b/db/schema_tables.cc @@ -2802,7 +2802,6 @@ static void add_drop_column_to_mutations(schema_ptr table, const sstring& name, static void make_update_columns_mutations(schema_ptr old_table, schema_ptr new_table, api::timestamp_type timestamp, - bool from_thrift, std::vector& mutations) { mutation columns_mutation(columns(), partition_key::from_singular(*columns(), old_table->ks_name())); mutation view_virtual_columns_mutation(view_virtual_columns(), partition_key::from_singular(*columns(), old_table->ks_name())); @@ -2815,9 +2814,6 @@ static void make_update_columns_mutations(schema_ptr old_table, // Thrift only knows about the REGULAR ColumnDefinition type, so don't consider other type // are being deleted just because they are not here. const column_definition& column = *old_table->v3().columns_by_name().at(name); - if (from_thrift && !column.is_regular()) { - continue; - } if (column.is_view_virtual()) { drop_column_from_schema_mutation(view_virtual_columns(), old_table, column.name_as_text(), timestamp, mutations); } else { @@ -2859,13 +2855,12 @@ std::vector make_update_table_mutations(replica::database& db, lw_shared_ptr keyspace, schema_ptr old_table, schema_ptr new_table, - api::timestamp_type timestamp, - bool from_thrift) + api::timestamp_type timestamp) { std::vector mutations; add_table_or_view_to_schema_mutation(new_table, timestamp, false, mutations); make_update_indices_mutations(db, old_table, new_table, timestamp, mutations); - make_update_columns_mutations(std::move(old_table), std::move(new_table), timestamp, from_thrift, mutations); + make_update_columns_mutations(std::move(old_table), std::move(new_table), timestamp, mutations); warn(unimplemented::cause::TRIGGERS); #if 0 @@ -3550,7 +3545,7 @@ std::vector make_update_view_mutations(lw_shared_ptr make_update_table_mutations( lw_shared_ptr keyspace, schema_ptr old_table, schema_ptr new_table, - api::timestamp_type timestamp, - bool from_thrift); + api::timestamp_type timestamp); future> create_tables_from_tables_partition(distributed& proxy, const schema_result::mapped_type& result); diff --git a/db/system_distributed_keyspace.cc b/db/system_distributed_keyspace.cc index 6f0ed35d5a..9edaf039fb 100644 --- a/db/system_distributed_keyspace.cc +++ b/db/system_distributed_keyspace.cc @@ -301,7 +301,7 @@ future<> system_distributed_keyspace::start() { // The service_levels table exists. Update it if it lacks new columns. if (table->cf_name() == SERVICE_LEVELS && !get_current_service_levels(db)->equal_columns(*table)) { - auto update_mutations = co_await service::prepare_column_family_update_announcement(_sp, table, false, std::vector(), ts); + auto update_mutations = co_await service::prepare_column_family_update_announcement(_sp, table, std::vector(), ts); std::move(update_mutations.begin(), update_mutations.end(), std::back_inserter(mutations)); } }); diff --git a/db/system_keyspace.cc b/db/system_keyspace.cc index fa1eeede05..3f87b37124 100644 --- a/db/system_keyspace.cc +++ b/db/system_keyspace.cc @@ -18,7 +18,6 @@ #include #include "system_keyspace.hh" #include "cql3/untyped_result_set.hh" -#include "thrift/server.hh" #include "cql3/query_processor.hh" #include "partition_slice_builder.hh" #include "db/config.hh" @@ -461,6 +460,7 @@ schema_ptr system_keyspace::built_indexes() { builder.remove_column("scylla_cpu_sharding_algorithm"); builder.remove_column("scylla_nr_shards"); builder.remove_column("scylla_msb_ignore"); + builder.remove_column("thrift_version"); return builder.build(schema_builder::compact_storage::no); }(); return local; @@ -1544,7 +1544,7 @@ future system_keyspace::load_local_info() { future<> system_keyspace::save_local_info(local_info sysinfo, locator::endpoint_dc_rack location, gms::inet_address broadcast_address, gms::inet_address broadcast_rpc_address) { auto& cfg = _db.get_config(); - sstring req = fmt::format("INSERT INTO system.{} (key, host_id, cluster_name, release_version, cql_version, thrift_version, native_protocol_version, data_center, rack, partitioner, rpc_address, broadcast_address, listen_address) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + sstring req = fmt::format("INSERT INTO system.{} (key, host_id, cluster_name, release_version, cql_version, native_protocol_version, data_center, rack, partitioner, rpc_address, broadcast_address, listen_address) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" , db::system_keyspace::LOCAL); return execute_cql(req, sstring(db::system_keyspace::LOCAL), @@ -1552,7 +1552,6 @@ future<> system_keyspace::save_local_info(local_info sysinfo, locator::endpoint_ sysinfo.cluster_name, version::release(), cql3::query_processor::CQL_VERSION, - ::cassandra::thrift_version, to_sstring(unsigned(cql_serialization_format::latest().protocol_version())), location.dc, location.rack, diff --git a/db/tags/utils.cc b/db/tags/utils.cc index 851d065c26..36a2efa1a0 100644 --- a/db/tags/utils.cc +++ b/db/tags/utils.cc @@ -69,7 +69,7 @@ future<> modify_tags(service::migration_manager& mm, sstring ks, sstring cf, builder.add_extension(tags_extension::NAME, ::make_shared(tags)); auto m = co_await service::prepare_column_family_update_announcement(mm.get_storage_proxy(), - builder.build(), false, std::vector(), group0_guard.write_timestamp()); + builder.build(), std::vector(), group0_guard.write_timestamp()); co_await mm.announce(std::move(m), std::move(group0_guard), format("Modify tags for {} table", cf)); }); diff --git a/default.nix b/default.nix index 48aa5103b1..9e217e487d 100644 --- a/default.nix +++ b/default.nix @@ -150,7 +150,6 @@ in derive ({ rapidjson snappy systemd - thrift valgrind xorg.libpciaccess xxHash diff --git a/docs/dev/docker-hub.md b/docs/dev/docker-hub.md index 808b7f922a..e38f6c5996 100644 --- a/docs/dev/docker-hub.md +++ b/docs/dev/docker-hub.md @@ -118,7 +118,6 @@ INFO 2016-08-04 06:57:40,836 [shard 7] database - Setting compaction strategy o INFO 2016-08-04 06:57:40,837 [shard 6] database - Setting compaction strategy of system_traces.events to SizeTieredCompactionStrategy INFO 2016-08-04 06:57:40,839 [shard 0] database - Schema version changed to fea14d93-9c5a-34f5-9d0e-2e49dcfa747e INFO 2016-08-04 06:57:40,839 [shard 0] storage_service - Starting listening for CQL clients on 172.17.0.2:9042... -INFO 2016-08-04 06:57:40,840 [shard 0] storage_service - Thrift server listening on 172.17.0.2:9160 ... ``` ### Configuring data volume for storage diff --git a/docs/dev/protocols.md b/docs/dev/protocols.md index 084d0830db..32e1f42149 100644 --- a/docs/dev/protocols.md +++ b/docs/dev/protocols.md @@ -149,31 +149,25 @@ The CQL protocol support can be disabled altogether by setting the These option names were chosen for backward-compatibility with Cassandra configuration files: they refer to CQL as the "native transport", to -contrast with the older Thrift protocol (described below) which wasn't -native to Cassandra. +contrast with the older Thrift protocol which wasn't +native to Cassandra. The thrift protocol was once supported by Scylla, +but the support of this protocol was later deprecated and removed. There is also a `rpc_address` configuration option to set the IP address (and therefore network interface) on which Scylla should listen for the CQL protocol. This address defaults to `localhost`, but in any setup except -a one-node test, should be overridden. Note that the same option `rpc_address` -applies to both CQL and Thrift protocols. - -TODO: there is also `rpc_interface` option... Which wins? What's the default? +a one-node test, should be overridden. ## Thrift client protocol The Apache Thrift protocol was early Cassandra's client protocol, until it was superseded in Cassandra 1.2 with the binary CQL protocol. Thrift was still nominally supported by both Cassandra and Scylla for many years, -but was recently dropped in Cassandra (version 4.0) and is likely to be -dropped by Scylla in the future as well, so it is not recommended for new -applications. +but was recently dropped in Cassandra (version 4.0) and was also +dropped by Scylla in version 6.0 as well. -By default, Scylla does not enable the Thrift server. In order to use it, -it must be explicitly enabled by setting the `start_rpc` configuration option -to true. - -When Thrift is enabled, by default scylla listens to the Thrift protocol on port 9160, +When Thrift was enabled by Scylla versions earlier than 6.0, by default scylla +listens to the Thrift protocol on port 9160, which can be configured via the `rpc_port` configuration option. Again, this confusing name was used for backward-compatibility with Cassandra's configuration files. Cassandra used the term "rpc" because Apache Thrift is a remote procedure @@ -181,14 +175,11 @@ call (RPC) framework. In Scylla, this name is especially confusing, because as mentioned above, Scylla's internal communication protocol is based on Seastar's RPC, which has nothing to do with the "`rpc_port`" described here. -There is also a `rpc_address` configuration option to set the IP address -(and therefore network interface) on which Scylla should listen for the -Thrift protocol. This address defaults to `localhost`, but in any -setup except a one-node test, should be overridden. Note that the same -option `rpc_address` applies to both CQL and Thrift protocols. +This option is now marked `Unused`, and still stays with us for one more +release, because scylla need to be able to consume existing configurations, +and to work with toolings which might be still setting this option. TODO: there is also `rpc_interface` option... Which wins? What's the default? -TODO: is there an SSL version of Thrift? ## DynamoDB client protocol @@ -224,8 +215,8 @@ and/or `redis_ssl_port` configuration option. The traditional port used for Redis is 6379. Regular Redis does not support SSL, so there is no traditional choice of port for it. -The same `rpc_address` configuration option used by the CQL and Thrift -protocols to set the IP address (and therefore network interface) on which +The same `rpc_address` configuration option used by the CQL +protocol to set the IP address (and therefore network interface) on which Scylla should listen also applies to the Redis protocol. See [redis.md](redis.md) for more information about Scylla's diff --git a/docs/dev/system_keyspace.md b/docs/dev/system_keyspace.md index 539e1ada4d..aa6271040e 100644 --- a/docs/dev/system_keyspace.md +++ b/docs/dev/system_keyspace.md @@ -276,7 +276,7 @@ Implemented by `cluster_status_table` in `db/system_keyspace.cc`. ## system.protocol_servers The list of all the client-facing data-plane protocol servers and listen addresses (if running). -Equivalent of the `nodetool statusbinary` plus the `Thrift active` and `Native Transport active` fields from `nodetool info`. +Equivalent of the `nodetool statusbinary` plus the `Native Transport active` fields from `nodetool info`. TODO: include control-plane diagnostics-plane protocols here too. diff --git a/docs/getting-started/install-scylla/install-on-linux.rst b/docs/getting-started/install-scylla/install-on-linux.rst index a0f68b0d25..37b7429090 100644 --- a/docs/getting-started/install-scylla/install-on-linux.rst +++ b/docs/getting-started/install-scylla/install-on-linux.rst @@ -164,7 +164,7 @@ Configure and Run ScyllaDB * ``seeds`` - The IP address of the first node. Other nodes will use it as the first contact point to discover the cluster topology when joining the cluster. * ``listen_address`` - The IP address that ScyllaDB uses to connect to other nodes in the cluster. - * ``rpc_address`` - The IP address of the interface for client connections (Thrift, CQL). + * ``rpc_address`` - The IP address of the interface for CQL client connections. #. Run the ``scylla_setup`` script to tune the system settings and determine the optimal configuration. diff --git a/docs/operating-scylla/_common/networking-ports.rst b/docs/operating-scylla/_common/networking-ports.rst index 0c73a89767..f38b7c5b5e 100644 --- a/docs/operating-scylla/_common/networking-ports.rst +++ b/docs/operating-scylla/_common/networking-ports.rst @@ -22,8 +22,6 @@ Port Description Protocol ------ -------------------------------------------- -------- 9100 node_exporter (Optionally) TCP ------ -------------------------------------------- -------- -9160 Scylla client port (Thrift) TCP ------- -------------------------------------------- -------- 19042 Native shard-aware transport port TCP ------ -------------------------------------------- -------- 19142 Native shard-aware transport port (ssl) TCP diff --git a/docs/operating-scylla/admin-tools/cassandra-stress.rst b/docs/operating-scylla/admin-tools/cassandra-stress.rst index 1c31a3dede..f9ee9cf8b2 100644 --- a/docs/operating-scylla/admin-tools/cassandra-stress.rst +++ b/docs/operating-scylla/admin-tools/cassandra-stress.rst @@ -45,7 +45,7 @@ Primary Options: -rate: Thread count, rate limit or automatic mode (default is auto). --mode: Thrift or CQL with options. +-mode: CQL with options. -errors: How to handle errors when encountered during stress. diff --git a/docs/operating-scylla/admin.rst b/docs/operating-scylla/admin.rst index fa2ca43674..3bdc025fe1 100644 --- a/docs/operating-scylla/admin.rst +++ b/docs/operating-scylla/admin.rst @@ -67,7 +67,7 @@ The following addresses can be configured in scylla.yaml: * - listen_address - Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports. * - rpc_address - - Address on which Scylla is going to expect Thrift and CQL client connections. See rpc_port, native_transport_port and native_transport_port_ssl in the :ref:`Networking ` parameters. + - Address on which Scylla is going to expect CQL client connections. See rpc_port, native_transport_port and native_transport_port_ssl in the :ref:`Networking ` parameters. * - broadcast_address - Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above. * - broadcast_rpc_address @@ -167,7 +167,7 @@ Do not set any IP address to :code:`0.0.0.0`. - Address Scylla listens for connections from other nodes. See storage_port and ssl_storage_ports. - No default * - rpc_address (required) - - Address on which Scylla is going to expect Thrift and CQL clients connections. See rpc_port, native_transport_port and native_transport_port_ssl in the :ref:`Networking ` parameters. + - Address on which Scylla is going to expect CQL clients connections. See rpc_port, native_transport_port and native_transport_port_ssl in the :ref:`Networking ` parameters. - No default * - broadcast_address - Address that is broadcasted to tell other Scylla nodes to connect to. Related to listen_address above. diff --git a/docs/operating-scylla/nodetool-commands/info.rst b/docs/operating-scylla/nodetool-commands/info.rst index 410aff5083..00803d165e 100644 --- a/docs/operating-scylla/nodetool-commands/info.rst +++ b/docs/operating-scylla/nodetool-commands/info.rst @@ -17,7 +17,7 @@ Example output: ID : 2110829b-47f2-4a6b-b87e-a81bc3b5cb31 Gossip active : true - Thrift active : true + Thrift active : false Native Transport active: true Load : 294.44 MB Generation No : 1474434958 diff --git a/docs/operating-scylla/nodetool-commands/setlogginglevel.rst b/docs/operating-scylla/nodetool-commands/setlogginglevel.rst index 898b1e0df6..f58449d889 100644 --- a/docs/operating-scylla/nodetool-commands/setlogginglevel.rst +++ b/docs/operating-scylla/nodetool-commands/setlogginglevel.rst @@ -188,8 +188,6 @@ To display the log classes (output changes with each version so your display may tags task_manager testlog - thrift - thrift_controller token_group_based_splitting_mutation_writer token_metadata topology diff --git a/docs/operating-scylla/procedures/cluster-management/add-dc-to-existing-dc.rst b/docs/operating-scylla/procedures/cluster-management/add-dc-to-existing-dc.rst index d778bf3112..cf688a5f73 100644 --- a/docs/operating-scylla/procedures/cluster-management/add-dc-to-existing-dc.rst +++ b/docs/operating-scylla/procedures/cluster-management/add-dc-to-existing-dc.rst @@ -121,7 +121,7 @@ Add New DC * **seeds** - IP address of an existing node (or nodes). * **listen_address** - IP address that Scylla used to connect to the other Scylla nodes in the cluster. * **endpoint_snitch** - Set the selected snitch. - * **rpc_address** - Address for client connections (Thrift, CQL). + * **rpc_address** - Address for CQL client connections. The parameters ``seeds``, ``cluster_name`` and ``endpoint_snitch`` need to match the existing cluster. @@ -237,4 +237,4 @@ Additional Resources for Java Clients .. _add-dc-upgrade-info: -.. scylladb_include_flag:: upgrade-warning-add-new-node-or-dc.rst \ No newline at end of file +.. scylladb_include_flag:: upgrade-warning-add-new-node-or-dc.rst diff --git a/docs/operating-scylla/procedures/cluster-management/add-node-to-cluster.rst b/docs/operating-scylla/procedures/cluster-management/add-node-to-cluster.rst index 697060d862..d9373ac92f 100644 --- a/docs/operating-scylla/procedures/cluster-management/add-node-to-cluster.rst +++ b/docs/operating-scylla/procedures/cluster-management/add-node-to-cluster.rst @@ -47,7 +47,7 @@ Procedure * **endpoint_snitch** - Specifies the selected snitch. - * **rpc_address** - Specifies the address for client connections (Thrift, CQL). + * **rpc_address** - Specifies the address for CQL client connections. * **seeds** - Specifies the IP address of an existing node in the cluster. The new node will use this IP to connect to the cluster and learn the cluster topology and state. @@ -103,4 +103,4 @@ Procedure .. _add-new-node-upgrade-info: -.. scylladb_include_flag:: upgrade-warning-add-new-node-or-dc.rst \ No newline at end of file +.. scylladb_include_flag:: upgrade-warning-add-new-node-or-dc.rst diff --git a/docs/operating-scylla/procedures/cluster-management/create-cluster-multidc.rst b/docs/operating-scylla/procedures/cluster-management/create-cluster-multidc.rst index af49a6299c..bbc0b5d111 100644 --- a/docs/operating-scylla/procedures/cluster-management/create-cluster-multidc.rst +++ b/docs/operating-scylla/procedures/cluster-management/create-cluster-multidc.rst @@ -67,7 +67,7 @@ The file can be found under ``/etc/scylla/``. - **seeds** - Specify the IP of the node you chose to be a seed node. New nodes will use the IP of this seed node to connect to the cluster and learn the cluster topology and state. - **listen_address** - IP address that the Scylla use to connect to other Scylla nodes in the cluster - **endpoint_snitch** - Set the selected snitch -- **rpc_address** - Address for client connection (Thrift, CQLSH) +- **rpc_address** - Address for CQL client connection 3. In the ``cassandra-rackdc.properties`` file, edit the rack and data center information. The file can be found under ``/etc/scylla/``. diff --git a/docs/operating-scylla/procedures/cluster-management/create-cluster.rst b/docs/operating-scylla/procedures/cluster-management/create-cluster.rst index 323ba5da26..b873f30f81 100644 --- a/docs/operating-scylla/procedures/cluster-management/create-cluster.rst +++ b/docs/operating-scylla/procedures/cluster-management/create-cluster.rst @@ -23,7 +23,7 @@ The file can be found under ``/etc/scylla/``. - **seeds** - Specify the IP of the node you chose to be a seed node. New nodes will use the IP of this seed node to connect to the cluster and learn the cluster topology and state. - **listen_address** - IP address that ScyllaDB used to connect to other ScyllaDB nodes in the cluster - **endpoint_snitch** - Set the selected snitch -- **rpc_address** - Address for client connection (Thrift, CQL) +- **rpc_address** - Address for CQL client connection 3. This step needs to be done **only** if you are using the **GossipingPropertyFileSnitch**. If not, skip this step. In the ``cassandra-rackdc.properties`` file, edit the parameters listed below. diff --git a/docs/operating-scylla/procedures/cluster-management/ec2-dc.rst b/docs/operating-scylla/procedures/cluster-management/ec2-dc.rst index 1684c90798..8cac33c9b8 100644 --- a/docs/operating-scylla/procedures/cluster-management/ec2-dc.rst +++ b/docs/operating-scylla/procedures/cluster-management/ec2-dc.rst @@ -60,7 +60,7 @@ Procedure * **seeds** - Specify the IP of the node you chose to be a seed node. See :doc:`Scylla Seed Nodes ` for details. * **listen_address** - IP address that Scylla used to connect to other Scylla nodes in the cluster. * **endpoint_snitch** - Set the selected snitch. - * **rpc_address** - Address for client connection (Thrift, CQL). + * **rpc_address** - Address for CQL client connection. * **broadcast_address** - The IP address a node tells other nodes in the cluster to contact it by. * **broadcast_rpc_address** - Default: unset. The RPC address to broadcast to drivers and other Scylla nodes. It cannot be set to 0.0.0.0. If left blank, it will be set to the value of ``rpc_address``. If ``rpc_address`` is set to 0.0.0.0, ``broadcast_rpc_address`` must be explicitly configured. diff --git a/docs/operating-scylla/procedures/cluster-management/replace-dead-node.rst b/docs/operating-scylla/procedures/cluster-management/replace-dead-node.rst index 34f88b8467..73b41e3c2e 100644 --- a/docs/operating-scylla/procedures/cluster-management/replace-dead-node.rst +++ b/docs/operating-scylla/procedures/cluster-management/replace-dead-node.rst @@ -68,7 +68,7 @@ Procedure - **endpoint_snitch** - Set the selected snitch - - **rpc_address** - Address for client connection (Thrift, CQL) + - **rpc_address** - Address for CQL client connection #. Add the ``replace_node_first_boot`` parameter to the ``scylla.yaml`` config file on the new node. This line can be added to any place in the config file. After a successful node replacement, there is no need to remove it from the ``scylla.yaml`` file. (Note: The obsolete parameters "replace_address" and "replace_address_first_boot" are not supported and should not be used). The value of the ``replace_node_first_boot`` parameter should be the Host ID of the node to be replaced. diff --git a/docs/operating-scylla/procedures/tips/best-practices-scylla-on-docker.rst b/docs/operating-scylla/procedures/tips/best-practices-scylla-on-docker.rst index 5378a24b10..17cb2e948f 100644 --- a/docs/operating-scylla/procedures/tips/best-practices-scylla-on-docker.rst +++ b/docs/operating-scylla/procedures/tips/best-practices-scylla-on-docker.rst @@ -31,7 +31,7 @@ The ``docker run`` command starts a new Docker instance in the background named CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 616ee646cb9d scylladb/scylla "/docker-entrypoint.p" 4 seconds ago Up 4 seconds 7000-7001/tcp, 9042/tcp, 9160/tcp, 10000/tcp some-scylla -As seen from the ``docker ps`` output, the image exposes ports **7000-7001** (Inter-node RPC), **9042** (CQL), **9160** (Thrift), and **10000** (REST API). +As seen from the ``docker ps`` output, the image exposes ports **7000-7001** (Inter-node RPC), **9042** (CQL), and **10000** (REST API). Viewing ScyllaDB Server Logs diff --git a/docs/operating-scylla/scylla-yaml.inc b/docs/operating-scylla/scylla-yaml.inc index bb6e79e086..8c8c673af0 100644 --- a/docs/operating-scylla/scylla-yaml.inc +++ b/docs/operating-scylla/scylla-yaml.inc @@ -77,7 +77,7 @@ seeds IP address of an existing node in the cluster. It allows a new n -------------- -------------------------------------------------- listen_address IP address that the Scylla use to connect to other Scylla nodes in the cluster -------------- -------------------------------------------------- -rpc_address IP address of the interface for client connections (Thrift, CQL) +rpc_address IP address of the interface for CQL client connections ============== ================================================== .. _yaml_enabling_experimental_features: diff --git a/docs/using-scylla/cassandra-compatibility.rst b/docs/using-scylla/cassandra-compatibility.rst index 8aa23a8475..c606ed21fb 100644 --- a/docs/using-scylla/cassandra-compatibility.rst +++ b/docs/using-scylla/cassandra-compatibility.rst @@ -30,9 +30,10 @@ Interfaces - | Fully compatible with version 3.3.1, with additional features from later CQL versions (for example, :ref:`Duration type `). | Fully compatible with protocol v4, with additional features from v5. - More below - * - Thrift - - Deprecated in ScyllaDB and Cassandra - - Support for the Thrift protocol is deprecated and will be dropped in future releases of ScyllaDB. + * - Thrift + - Not supported anymore in ScyllaDB 6.0 + - | deprecated in Apache Cassandra and got dropped in 4.0 + | deprecated in ScyllaDB 5.2 and got dropped in 6.0 * - SSTable format (all versions) - 3.11(mc / md / me), 2.2(la), 2.1.8 (ka) - | ``me`` - supported in ScyllaDB Open Source 5.1 and ScyllaDB Enterprise 2022.2.0 (and later) diff --git a/docs/using-scylla/lwt.rst b/docs/using-scylla/lwt.rst index 813bf72d02..b3157d3dcd 100644 --- a/docs/using-scylla/lwt.rst +++ b/docs/using-scylla/lwt.rst @@ -534,7 +534,6 @@ Other limitations are more minor: * While a non-LWT batch can be UNLOGGED, a conditional batch cannot; * IF conditions must be a perfect conjunct (... AND ... AND ...); -* Unlike Cassandra, Scylla doesn't have LWT support in Thrift protocol and doesn't plan to add it; * Conditional batches are always logged in system.paxos table, so UNLOGGED keyword is silently ignored for them. Additional Information diff --git a/install-dependencies.sh b/install-dependencies.sh index 190e1a2f14..2c2cc8ef52 100755 --- a/install-dependencies.sh +++ b/install-dependencies.sh @@ -40,9 +40,7 @@ debian_base_packages=( libsnappy-dev libjsoncpp-dev rapidjson-dev - scylla-libthrift010-dev scylla-antlr35-c++-dev - thrift-compiler git pigz libunistring-dev @@ -61,7 +59,6 @@ fedora_packages=( gdb lua-devel yaml-cpp-devel - thrift-devel antlr3-tool antlr3-C++-devel jsoncpp-devel @@ -152,7 +149,6 @@ pip_symlinks=( centos_packages=( gdb yaml-cpp-devel - thrift-devel scylla-antlr35-tool scylla-antlr35-C++-devel jsoncpp-devel snappy-devel @@ -184,7 +180,6 @@ arch_packages=( python3 rapidjson snappy - thrift ) go_arch() { @@ -318,7 +313,7 @@ if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then else apt-get -y install libsystemd-dev antlr3 libyaml-cpp-dev fi - echo -e "Configure example:\n\t./configure.py --enable-dpdk --mode=release --static-thrift --static-boost --static-yaml-cpp --compiler=/opt/scylladb/bin/g++-7 --cflags=\"-I/opt/scylladb/include -L/opt/scylladb/lib/x86-linux-gnu/\" --ldflags=\"-Wl,-rpath=/opt/scylladb/lib\"" + echo -e "Configure example:\n\t./configure.py --enable-dpdk --mode=release --static-boost --static-yaml-cpp --compiler=/opt/scylladb/bin/g++-7 --cflags=\"-I/opt/scylladb/include -L/opt/scylladb/lib/x86-linux-gnu/\" --ldflags=\"-Wl,-rpath=/opt/scylladb/lib\"" elif [ "$ID" = "fedora" ]; then if rpm -q --quiet yum-utils; then echo diff --git a/interface/CMakeLists.txt b/interface/CMakeLists.txt deleted file mode 100644 index ced4db4ec6..0000000000 --- a/interface/CMakeLists.txt +++ /dev/null @@ -1,58 +0,0 @@ -# Generate C++ source files from thrift definitions -function(scylla_generate_thrift) - set(one_value_args TARGET VAR THRIFT_VERSION IN_FILE OUT_DIR SERVICE) - cmake_parse_arguments(args "" "${one_value_args}" "" ${ARGN}) - - get_filename_component(in_file_name ${args_IN_FILE} NAME_WE) - - set(aux_out_file_name ${args_OUT_DIR}/${in_file_name}) - set(outputs - ${aux_out_file_name}_types.cpp - ${aux_out_file_name}_types.h - ${aux_out_file_name}_constants.cpp - ${aux_out_file_name}_constants.h - ${args_OUT_DIR}/${args_SERVICE}.cpp - ${args_OUT_DIR}/${args_SERVICE}.h) - - find_program(THRIFT thrift - REQUIRED) - execute_process( - COMMAND "${THRIFT}" -version - OUTPUT_VARIABLE thrift_version_output - OUTPUT_STRIP_TRAILING_WHITESPACE) - string(REGEX MATCH "[0-9]+\.[0-9]+\.[0-9]+$" - thrift_version "${thrift_version_output}") - set(${args_THRIFT_VERSION} ${thrift_version} PARENT_SCOPE) - - add_custom_command( - DEPENDS ${args_IN_FILE} - OUTPUT ${outputs} - COMMAND ${CMAKE_COMMAND} -E make_directory ${args_OUT_DIR} - COMMAND ${THRIFT} -gen cpp:cob_style,no_skeleton -out "${args_OUT_DIR}" "${args_IN_FILE}") - - add_custom_target(${args_TARGET} - DEPENDS ${outputs}) - - set(${args_VAR} ${outputs} PARENT_SCOPE) -endfunction() - -scylla_generate_thrift( - TARGET scylla_thrift_gen_cassandra - VAR scylla_thrift_gen_cassandra_files - THRIFT_VERSION thrift_version - IN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cassandra.thrift" - OUT_DIR ${scylla_gen_build_dir} - SERVICE Cassandra) - -add_library(interface STATIC) -target_sources(interface - PRIVATE - ${scylla_thrift_gen_cassandra_files}) -target_include_directories(interface - PUBLIC - ${scylla_gen_build_dir}) -if(thrift_version VERSION_LESS 0.11.0) - target_compile_definitions(interface - PUBLIC - THRIFT_USES_BOOST) -endif() diff --git a/interface/cassandra.thrift b/interface/cassandra.thrift deleted file mode 100644 index 6f74b4153d..0000000000 --- a/interface/cassandra.thrift +++ /dev/null @@ -1,941 +0,0 @@ -#!/usr/local/bin/thrift --java --php --py -# SPDX-License-Identifier: Apache-2.0 - -# -# Copyright (C) 2014-present ScyllaDB -# - -# -# This file has been modified from the Apache distribution -# by ScyllaDB -# - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# *** PLEASE REMEMBER TO EDIT THE VERSION CONSTANT WHEN MAKING CHANGES *** -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# -# Interface definition for Cassandra Service -# - -namespace java org.apache.cassandra.thrift -namespace cpp cassandra -namespace netstd Apache.Cassandra -namespace py cassandra -namespace php cassandra -namespace perl Cassandra - -# Thrift.rb has a bug where top-level modules that include modules -# with the same name are not properly referenced, so we can't do -# Cassandra::Cassandra::Client. -namespace rb CassandraThrift - -# The API version (NOT the product version), composed as a dot delimited -# string with major, minor, and patch level components. -# -# - Major: Incremented for backward incompatible changes. An example would -# be changes to the number or disposition of method arguments. -# - Minor: Incremented for backward compatible changes. An example would -# be the addition of a new (optional) method. -# - Patch: Incremented for bug fixes. The patch level should be increased -# for every edit that doesn't result in a change to major/minor. -# -# See the Semantic Versioning Specification (SemVer) http://semver.org. -# -# Note that this backwards compatibility is from the perspective of the server, -# not the client. Cassandra should always be able to talk to older client -# software, but client software may not be able to talk to older Cassandra -# instances. -# -# An effort should be made not to break forward-client-compatibility either -# (e.g. one should avoid removing obsolete fields from the IDL), but no -# guarantees in this respect are made by the Cassandra project. -const string VERSION_ = "20.1.0" - - -# -# data structures -# - -/** Basic unit of data within a ColumnFamily. - * @param name, the name by which this column is set and retrieved. Maximum 64KB long. - * @param value. The data associated with the name. Maximum 2GB long, but in practice you should limit it to small numbers of MB (since Thrift must read the full value into memory to operate on it). - * @param timestamp. The timestamp is used for conflict detection/resolution when two columns with same name need to be compared. - * @param ttl. An optional, positive delay (in seconds) after which the column will be automatically deleted. - */ -struct Column { - 1: required binary name, - 2: optional binary value, - 3: optional i64 timestamp, - 4: optional i32 ttl, -} - -/** A named list of columns. - * @param name. see Column.name. - * @param columns. A collection of standard Columns. The columns within a super column are defined in an adhoc manner. - * Columns within a super column do not have to have matching structures (similarly named child columns). - */ -struct SuperColumn { - 1: required binary name, - 2: required list columns, -} - -struct CounterColumn { - 1: required binary name, - 2: required i64 value -} - -struct CounterSuperColumn { - 1: required binary name, - 2: required list columns -} - -/** - Methods for fetching rows/records from Cassandra will return either a single instance of ColumnOrSuperColumn or a list - of ColumnOrSuperColumns (get_slice()). If you're looking up a SuperColumn (or list of SuperColumns) then the resulting - instances of ColumnOrSuperColumn will have the requested SuperColumn in the attribute super_column. For queries resulting - in Columns, those values will be in the attribute column. This change was made between 0.3 and 0.4 to standardize on - single query methods that may return either a SuperColumn or Column. - - If the query was on a counter column family, you will either get a counter_column (instead of a column) or a - counter_super_column (instead of a super_column) - - @param column. The Column returned by get() or get_slice(). - @param super_column. The SuperColumn returned by get() or get_slice(). - @param counter_column. The Counterolumn returned by get() or get_slice(). - @param counter_super_column. The CounterSuperColumn returned by get() or get_slice(). - */ -struct ColumnOrSuperColumn { - 1: optional Column column, - 2: optional SuperColumn super_column, - 3: optional CounterColumn counter_column, - 4: optional CounterSuperColumn counter_super_column -} - - -# -# Exceptions -# (note that internal server errors will raise a TApplicationException, courtesy of Thrift) -# - -/** A specific column was requested that does not exist. */ -exception NotFoundException { -} - -/** Invalid request could mean keyspace or column family does not exist, required parameters are missing, or a parameter is malformed. - why contains an associated error message. -*/ -exception InvalidRequestException { - 1: required string why -} - -/** Not all the replicas required could be created and/or read. */ -exception UnavailableException { -} - -/** RPC timeout was exceeded. either a node failed mid-operation, or load was too high, or the requested op was too large. */ -exception TimedOutException { - /** - * if a write operation was acknowledged by some replicas but not by enough to - * satisfy the required ConsistencyLevel, the number of successful - * replies will be given here. In case of atomic_batch_mutate method this field - * will be set to -1 if the batch was written to the batchlog and to 0 if it wasn't. - */ - 1: optional i32 acknowledged_by - - /** - * in case of atomic_batch_mutate method this field tells if the batch - * was written to the batchlog. - */ - 2: optional bool acknowledged_by_batchlog - - /** - * for the CAS method, this field tells if we timed out during the paxos - * protocol, as opposed to during the commit of our update - */ - 3: optional bool paxos_in_progress -} - -/** invalid authentication request (invalid keyspace, user does not exist, or credentials invalid) */ -exception AuthenticationException { - 1: required string why -} - -/** invalid authorization request (user does not have access to keyspace) */ -exception AuthorizationException { - 1: required string why -} - -/** - * NOTE: This up outdated exception left for backward compatibility reasons, - * no actual schema agreement validation is done starting from Cassandra 1.2 - * - * schemas are not in agreement across all nodes - */ -exception SchemaDisagreementException { -} - - -# -# service api -# -/** - * The ConsistencyLevel is an enum that controls both read and write - * behavior based on the ReplicationFactor of the keyspace. The - * different consistency levels have different meanings, depending on - * if you're doing a write or read operation. - * - * If W + R > ReplicationFactor, where W is the number of nodes to - * block for on write, and R the number to block for on reads, you - * will have strongly consistent behavior; that is, readers will - * always see the most recent write. Of these, the most interesting is - * to do QUORUM reads and writes, which gives you consistency while - * still allowing availability in the face of node failures up to half - * of . Of course if latency is more important than - * consistency then you can use lower values for either or both. - * - * Some ConsistencyLevels (ONE, TWO, THREE) refer to a specific number - * of replicas rather than a logical concept that adjusts - * automatically with the replication factor. Of these, only ONE is - * commonly used; TWO and (even more rarely) THREE are only useful - * when you care more about guaranteeing a certain level of - * durability, than consistency. - * - * Write consistency levels make the following guarantees before reporting success to the client: - * ANY Ensure that the write has been written once somewhere, including possibly being hinted in a non-target node. - * ONE Ensure that the write has been written to at least 1 node's commit log and memory table - * TWO Ensure that the write has been written to at least 2 node's commit log and memory table - * THREE Ensure that the write has been written to at least 3 node's commit log and memory table - * QUORUM Ensure that the write has been written to / 2 + 1 nodes - * LOCAL_ONE Ensure that the write has been written to 1 node within the local datacenter (requires NetworkTopologyStrategy) - * LOCAL_QUORUM Ensure that the write has been written to / 2 + 1 nodes, within the local datacenter (requires NetworkTopologyStrategy) - * EACH_QUORUM Ensure that the write has been written to / 2 + 1 nodes in each datacenter (requires NetworkTopologyStrategy) - * ALL Ensure that the write is written to <ReplicationFactor> nodes before responding to the client. - * - * Read consistency levels make the following guarantees before returning successful results to the client: - * ANY Not supported. You probably want ONE instead. - * ONE Returns the record obtained from a single replica. - * TWO Returns the record with the most recent timestamp once two replicas have replied. - * THREE Returns the record with the most recent timestamp once three replicas have replied. - * QUORUM Returns the record with the most recent timestamp once a majority of replicas have replied. - * LOCAL_ONE Returns the record with the most recent timestamp once a single replica within the local datacenter have replied. - * LOCAL_QUORUM Returns the record with the most recent timestamp once a majority of replicas within the local datacenter have replied. - * EACH_QUORUM Returns the record with the most recent timestamp once a majority of replicas within each datacenter have replied. - * ALL Returns the record with the most recent timestamp once all replicas have replied (implies no replica may be down).. -*/ -enum ConsistencyLevel { - ONE = 1, - QUORUM = 2, - LOCAL_QUORUM = 3, - EACH_QUORUM = 4, - ALL = 5, - ANY = 6, - TWO = 7, - THREE = 8, - SERIAL = 9, - LOCAL_SERIAL = 10, - LOCAL_ONE = 11, -} - -/** - ColumnParent is used when selecting groups of columns from the same ColumnFamily. In directory structure terms, imagine - ColumnParent as ColumnPath + '/../'. - - See also ColumnPath - */ -struct ColumnParent { - 3: required string column_family, - 4: optional binary super_column, -} - -/** The ColumnPath is the path to a single column in Cassandra. It might make sense to think of ColumnPath and - * ColumnParent in terms of a directory structure. - * - * ColumnPath is used to looking up a single column. - * - * @param column_family. The name of the CF of the column being looked up. - * @param super_column. The super column name. - * @param column. The column name. - */ -struct ColumnPath { - 3: required string column_family, - 4: optional binary super_column, - 5: optional binary column, -} - -/** - A slice range is a structure that stores basic range, ordering and limit information for a query that will return - multiple columns. It could be thought of as Cassandra's version of LIMIT and ORDER BY - - @param start. The column name to start the slice with. This attribute is not required, though there is no default value, - and can be safely set to '', i.e., an empty byte array, to start with the first column name. Otherwise, it - must a valid value under the rules of the Comparator defined for the given ColumnFamily. - @param finish. The column name to stop the slice at. This attribute is not required, though there is no default value, - and can be safely set to an empty byte array to not stop until 'count' results are seen. Otherwise, it - must also be a valid value to the ColumnFamily Comparator. - @param reversed. Whether the results should be ordered in reversed order. Similar to ORDER BY blah DESC in SQL. - @param count. How many columns to return. Similar to LIMIT in SQL. May be arbitrarily large, but Thrift will - materialize the whole result into memory before returning it to the client, so be aware that you may - be better served by iterating through slices by passing the last value of one call in as the 'start' - of the next instead of increasing 'count' arbitrarily large. - */ -struct SliceRange { - 1: required binary start, - 2: required binary finish, - 3: required bool reversed=0, - 4: required i32 count=100, -} - -/** - A SlicePredicate is similar to a mathematic predicate (see http://en.wikipedia.org/wiki/Predicate_(mathematical_logic)), - which is described as "a property that the elements of a set have in common." - - SlicePredicate's in Cassandra are described with either a list of column_names or a SliceRange. If column_names is - specified, slice_range is ignored. - - @param column_name. A list of column names to retrieve. This can be used similar to Memcached's "multi-get" feature - to fetch N known column names. For instance, if you know you wish to fetch columns 'Joe', 'Jack', - and 'Jim' you can pass those column names as a list to fetch all three at once. - @param slice_range. A SliceRange describing how to range, order, and/or limit the slice. - */ -struct SlicePredicate { - 1: optional list column_names, - 2: optional SliceRange slice_range, -} - -enum IndexOperator { - EQ, - GTE, - GT, - LTE, - LT -} - -struct IndexExpression { - 1: required binary column_name, - 2: required IndexOperator op, - 3: required binary value, -} - -/** - * @deprecated use a KeyRange with row_filter in get_range_slices instead - */ -struct IndexClause { - 1: required list expressions, - 2: required binary start_key, - 3: required i32 count=100, -} - - -/** -The semantics of start keys and tokens are slightly different. -Keys are start-inclusive; tokens are start-exclusive. Token -ranges may also wrap -- that is, the end token may be less -than the start one. Thus, a range from keyX to keyX is a -one-element range, but a range from tokenY to tokenY is the -full ring. -*/ -struct KeyRange { - 1: optional binary start_key, - 2: optional binary end_key, - 3: optional string start_token, - 4: optional string end_token, - 6: optional list row_filter, - 5: required i32 count=100 -} - -/** - A KeySlice is key followed by the data it maps to. A collection of KeySlice is returned by the get_range_slice operation. - - @param key. a row key - @param columns. List of data represented by the key. Typically, the list is pared down to only the columns specified by - a SlicePredicate. - */ -struct KeySlice { - 1: required binary key, - 2: required list columns, -} - -struct KeyCount { - 1: required binary key, - 2: required i32 count -} - -/** - * Note that the timestamp is only optional in case of counter deletion. - */ -struct Deletion { - 1: optional i64 timestamp, - 2: optional binary super_column, - 3: optional SlicePredicate predicate, -} - -/** - A Mutation is either an insert (represented by filling column_or_supercolumn) or a deletion (represented by filling the deletion attribute). - @param column_or_supercolumn. An insert to a column or supercolumn (possibly counter column or supercolumn) - @param deletion. A deletion of a column or supercolumn -*/ -struct Mutation { - 1: optional ColumnOrSuperColumn column_or_supercolumn, - 2: optional Deletion deletion, -} - -struct EndpointDetails { - 1: string host, - 2: string datacenter, - 3: optional string rack -} - -struct CASResult { - 1: required bool success, - 2: optional list current_values, -} - -/** - A TokenRange describes part of the Cassandra ring, it is a mapping from a range to - endpoints responsible for that range. - @param start_token The first token in the range - @param end_token The last token in the range - @param endpoints The endpoints responsible for the range (listed by their configured listen_address) - @param rpc_endpoints The endpoints responsible for the range (listed by their configured rpc_address) -*/ -struct TokenRange { - 1: required string start_token, - 2: required string end_token, - 3: required list endpoints, - 4: optional list rpc_endpoints - 5: optional list endpoint_details, -} - -/** - Authentication requests can contain any data, dependent on the IAuthenticator used -*/ -struct AuthenticationRequest { - 1: required map credentials -} - -enum IndexType { - KEYS, - CUSTOM, - COMPOSITES -} - -/* describes a column in a column family. */ -struct ColumnDef { - 1: required binary name, - 2: required string validation_class, - 3: optional IndexType index_type, - 4: optional string index_name, - 5: optional map index_options -} - -/** - Describes a trigger. - `options` should include at least 'class' param. - Other options are not supported yet. -*/ -struct TriggerDef { - 1: required string name, - 2: required map options -} - -/* describes a column family. */ -struct CfDef { - 1: required string keyspace, - 2: required string name, - 3: optional string column_type="Standard", - 5: optional string comparator_type="BytesType", - 6: optional string subcomparator_type, - 8: optional string comment, - 12: optional double read_repair_chance, - 13: optional list column_metadata, - 14: optional i32 gc_grace_seconds, - 15: optional string default_validation_class, - 16: optional i32 id, - 17: optional i32 min_compaction_threshold, - 18: optional i32 max_compaction_threshold, - 26: optional string key_validation_class, - 28: optional binary key_alias, - 29: optional string compaction_strategy, - 30: optional map compaction_strategy_options, - 32: optional map compression_options, - 33: optional double bloom_filter_fp_chance, - 34: optional string caching="keys_only", - 37: optional double dclocal_read_repair_chance = 0.0, - 39: optional i32 memtable_flush_period_in_ms, - 40: optional i32 default_time_to_live, - 42: optional string speculative_retry="NONE", - 43: optional list triggers, - 44: optional string cells_per_row_to_cache = "100", - 45: optional i32 min_index_interval, - 46: optional i32 max_index_interval, - - /* All of the following are now ignored and unsupplied. */ - - /** @deprecated */ - 9: optional double row_cache_size, - /** @deprecated */ - 11: optional double key_cache_size, - /** @deprecated */ - 19: optional i32 row_cache_save_period_in_seconds, - /** @deprecated */ - 20: optional i32 key_cache_save_period_in_seconds, - /** @deprecated */ - 21: optional i32 memtable_flush_after_mins, - /** @deprecated */ - 22: optional i32 memtable_throughput_in_mb, - /** @deprecated */ - 23: optional double memtable_operations_in_millions, - /** @deprecated */ - 24: optional bool replicate_on_write, - /** @deprecated */ - 25: optional double merge_shards_chance, - /** @deprecated */ - 27: optional string row_cache_provider, - /** @deprecated */ - 31: optional i32 row_cache_keys_to_save, - /** @deprecated */ - 38: optional bool populate_io_cache_on_flush, - /** @deprecated */ - 41: optional i32 index_interval, -} - -/* describes a keyspace. */ -struct KsDef { - 1: required string name, - 2: required string strategy_class, - 3: optional map strategy_options, - - /** @deprecated ignored */ - 4: optional i32 replication_factor, - - 5: required list cf_defs, - 6: optional bool durable_writes=1, -} - -/** CQL query compression */ -enum Compression { - GZIP = 1, - NONE = 2 -} - -enum CqlResultType { - ROWS = 1, - VOID = 2, - INT = 3 -} - -/** - Row returned from a CQL query. - - This struct is used for both CQL2 and CQL3 queries. For CQL2, the partition key - is special-cased and is always returned. For CQL3, it is not special cased; - it will be included in the columns list if it was included in the SELECT and - the key field is always null. -*/ -struct CqlRow { - 1: required binary key, - 2: required list columns -} - -struct CqlMetadata { - 1: required map name_types, - 2: required map value_types, - 3: required string default_name_type, - 4: required string default_value_type -} - -struct CqlResult { - 1: required CqlResultType type, - 2: optional list rows, - 3: optional i32 num, - 4: optional CqlMetadata schema -} - -struct CqlPreparedResult { - 1: required i32 itemId, - 2: required i32 count, - 3: optional list variable_types, - 4: optional list variable_names -} - -/** Represents input splits used by hadoop ColumnFamilyRecordReaders */ -struct CfSplit { - 1: required string start_token, - 2: required string end_token, - 3: required i64 row_count -} - -/** The ColumnSlice is used to select a set of columns from inside a row. - * If start or finish are unspecified they will default to the start-of - * end-of value. - * @param start. The start of the ColumnSlice inclusive - * @param finish. The end of the ColumnSlice inclusive - */ -struct ColumnSlice { - 1: optional binary start, - 2: optional binary finish -} - -/** - * Used to perform multiple slices on a single row key in one rpc operation - * @param key. The row key to be multi sliced - * @param column_parent. The column family (super columns are unsupported) - * @param column_slices. 0 to many ColumnSlice objects each will be used to select columns - * @param reversed. Direction of slice - * @param count. Maximum number of columns - * @param consistency_level. Level to perform the operation at - */ -struct MultiSliceRequest { - 1: optional binary key, - 2: optional ColumnParent column_parent, - 3: optional list column_slices, - 4: optional bool reversed=false, - 5: optional i32 count=1000, - 6: optional ConsistencyLevel consistency_level=ConsistencyLevel.ONE -} - -service Cassandra { - # auth methods - void login(1: required AuthenticationRequest auth_request) throws (1:AuthenticationException authnx, 2:AuthorizationException authzx), - - # set keyspace - void set_keyspace(1: required string keyspace) throws (1:InvalidRequestException ire), - - # retrieval methods - - /** - Get the Column or SuperColumn at the given column_path. If no value is present, NotFoundException is thrown. (This is - the only method that can throw an exception under non-failure conditions.) - */ - ColumnOrSuperColumn get(1:required binary key, - 2:required ColumnPath column_path, - 3:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:NotFoundException nfe, 3:UnavailableException ue, 4:TimedOutException te), - - /** - Get the group of columns contained by column_parent (either a ColumnFamily name or a ColumnFamily/SuperColumn name - pair) specified by the given SlicePredicate. If no matching values are found, an empty list is returned. - */ - list get_slice(1:required binary key, - 2:required ColumnParent column_parent, - 3:required SlicePredicate predicate, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - returns the number of columns matching predicate for a particular key, - ColumnFamily and optionally SuperColumn. - */ - i32 get_count(1:required binary key, - 2:required ColumnParent column_parent, - 3:required SlicePredicate predicate, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Performs a get_slice for column_parent and predicate for the given keys in parallel. - */ - map> multiget_slice(1:required list keys, - 2:required ColumnParent column_parent, - 3:required SlicePredicate predicate, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Perform a get_count in parallel on the given list keys. The return value maps keys to the count found. - */ - map multiget_count(1:required list keys, - 2:required ColumnParent column_parent, - 3:required SlicePredicate predicate, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - returns a subset of columns for a contiguous range of keys. - */ - list get_range_slices(1:required ColumnParent column_parent, - 2:required SlicePredicate predicate, - 3:required KeyRange range, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - returns a range of columns, wrapping to the next rows if necessary to collect max_results. - */ - list get_paged_slice(1:required string column_family, - 2:required KeyRange range, - 3:required binary start_column, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Returns the subset of columns specified in SlicePredicate for the rows matching the IndexClause - @deprecated use get_range_slices instead with range.row_filter specified - */ - list get_indexed_slices(1:required ColumnParent column_parent, - 2:required IndexClause index_clause, - 3:required SlicePredicate column_predicate, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - # modification methods - - /** - * Insert a Column at the given column_parent.column_family and optional column_parent.super_column. - */ - void insert(1:required binary key, - 2:required ColumnParent column_parent, - 3:required Column column, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - * Increment or decrement a counter. - */ - void add(1:required binary key, - 2:required ColumnParent column_parent, - 3:required CounterColumn column, - 4:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - * Atomic compare and set. - * - * If the cas is successful, the success boolean in CASResult will be true and there will be no current_values. - * Otherwise, success will be false and current_values will contain the current values for the columns in - * expected (that, by definition of compare-and-set, will differ from the values in expected). - * - * A cas operation takes 2 consistency level. The first one, serial_consistency_level, simply indicates the - * level of serialization required. This can be either ConsistencyLevel.SERIAL or ConsistencyLevel.LOCAL_SERIAL. - * The second one, commit_consistency_level, defines the consistency level for the commit phase of the cas. This - * is a more traditional consistency level (the same CL than for traditional writes are accepted) that impact - * the visibility for reads of the operation. For instance, if commit_consistency_level is QUORUM, then it is - * guaranteed that a followup QUORUM read will see the cas write (if that one was successful obviously). If - * commit_consistency_level is ANY, you will need to use a SERIAL/LOCAL_SERIAL read to be guaranteed to see - * the write. - */ - CASResult cas(1:required binary key, - 2:required string column_family, - 3:list expected, - 4:list updates, - 5:required ConsistencyLevel serial_consistency_level=ConsistencyLevel.SERIAL, - 6:required ConsistencyLevel commit_consistency_level=ConsistencyLevel.QUORUM) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Remove data from the row specified by key at the granularity specified by column_path, and the given timestamp. Note - that all the values in column_path besides column_path.column_family are truly optional: you can remove the entire - row by just specifying the ColumnFamily, or you can remove a SuperColumn or a single Column by specifying those levels too. - */ - void remove(1:required binary key, - 2:required ColumnPath column_path, - 3:required i64 timestamp, - 4:ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - * Remove a counter at the specified location. - * Note that counters have limited support for deletes: if you remove a counter, you must wait to issue any following update - * until the delete has reached all the nodes and all of them have been fully compacted. - */ - void remove_counter(1:required binary key, - 2:required ColumnPath path, - 3:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Mutate many columns or super columns for many row keys. See also: Mutation. - - mutation_map maps key to column family to a list of Mutation objects to take place at that scope. - **/ - void batch_mutate(1:required map>> mutation_map, - 2:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Atomically mutate many columns or super columns for many row keys. See also: Mutation. - - mutation_map maps key to column family to a list of Mutation objects to take place at that scope. - **/ - void atomic_batch_mutate(1:required map>> mutation_map, - 2:required ConsistencyLevel consistency_level=ConsistencyLevel.ONE) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - /** - Truncate will mark and entire column family as deleted. - From the user's perspective a successful call to truncate will result complete data deletion from cfname. - Internally, however, disk space will not be immediatily released, as with all deletes in cassandra, this one - only marks the data as deleted. - The operation succeeds only if all hosts in the cluster at available and will throw an UnavailableException if - some hosts are down. - */ - void truncate(1:required string cfname) - throws (1: InvalidRequestException ire, 2: UnavailableException ue, 3: TimedOutException te), - - /** - * Select multiple slices of a key in a single RPC operation - */ - list get_multi_slice(1:required MultiSliceRequest request) - throws (1:InvalidRequestException ire, 2:UnavailableException ue, 3:TimedOutException te), - - // Meta-APIs -- APIs to get information about the node or cluster, - // rather than user data. The nodeprobe program provides usage examples. - - /** - * for each schema version present in the cluster, returns a list of nodes at that version. - * hosts that do not respond will be under the key DatabaseDescriptor.INITIAL_VERSION. - * the cluster is all on the same version if the size of the map is 1. - */ - map> describe_schema_versions() - throws (1: InvalidRequestException ire), - - /** list the defined keyspaces in this cluster */ - list describe_keyspaces() - throws (1:InvalidRequestException ire), - - /** get the cluster name */ - string describe_cluster_name(), - - /** get the thrift api version */ - string describe_version(), - - /** get the token ring: a map of ranges to host addresses, - represented as a set of TokenRange instead of a map from range - to list of endpoints, because you can't use Thrift structs as - map keys: - https://issues.apache.org/jira/browse/THRIFT-162 - - for the same reason, we can't return a set here, even though - order is neither important nor predictable. */ - list describe_ring(1:required string keyspace) - throws (1:InvalidRequestException ire), - - - /** same as describe_ring, but considers only nodes in the local DC */ - list describe_local_ring(1:required string keyspace) - throws (1:InvalidRequestException ire), - - /** get the mapping between token->node ip - without taking replication into consideration - https://issues.apache.org/jira/browse/CASSANDRA-4092 */ - map describe_token_map() - throws (1:InvalidRequestException ire), - - /** returns the partitioner used by this cluster */ - string describe_partitioner(), - - /** returns the snitch used by this cluster */ - string describe_snitch(), - - /** describe specified keyspace */ - KsDef describe_keyspace(1:required string keyspace) - throws (1:NotFoundException nfe, 2:InvalidRequestException ire), - - /** experimental API for hadoop/parallel query support. - may change violently and without warning. - - returns list of token strings such that first subrange is (list[0], list[1]], - next is (list[1], list[2]], etc. */ - list describe_splits(1:required string cfName, - 2:required string start_token, - 3:required string end_token, - 4:required i32 keys_per_split) - throws (1:InvalidRequestException ire), - - /** Enables tracing for the next query in this connection and returns the UUID for that trace session - The next query will be traced independently of trace probability and the returned UUID can be used to query the trace keyspace */ - binary trace_next_query(), - - list describe_splits_ex(1:required string cfName, - 2:required string start_token, - 3:required string end_token, - 4:required i32 keys_per_split) - throws (1:InvalidRequestException ire), - - /** adds a column family. returns the new schema id. */ - string system_add_column_family(1:required CfDef cf_def) - throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), - - /** drops a column family. returns the new schema id. */ - string system_drop_column_family(1:required string column_family) - throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), - - /** adds a keyspace and any column families that are part of it. returns the new schema id. */ - string system_add_keyspace(1:required KsDef ks_def) - throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), - - /** drops a keyspace and any column families that are part of it. returns the new schema id. */ - string system_drop_keyspace(1:required string keyspace) - throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), - - /** updates properties of a keyspace. returns the new schema id. */ - string system_update_keyspace(1:required KsDef ks_def) - throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), - - /** updates properties of a column family. returns the new schema id. */ - string system_update_column_family(1:required CfDef cf_def) - throws (1:InvalidRequestException ire, 2:SchemaDisagreementException sde), - - - /** - * @deprecated Throws InvalidRequestException since 3.0. Please use the CQL3 version instead. - */ - CqlResult execute_cql_query(1:required binary query, 2:required Compression compression) - throws (1:InvalidRequestException ire, - 2:UnavailableException ue, - 3:TimedOutException te, - 4:SchemaDisagreementException sde) - - /** - * Executes a CQL3 (Cassandra Query Language) statement and returns a - * CqlResult containing the results. - */ - CqlResult execute_cql3_query(1:required binary query, 2:required Compression compression, 3:required ConsistencyLevel consistency) - throws (1:InvalidRequestException ire, - 2:UnavailableException ue, - 3:TimedOutException te, - 4:SchemaDisagreementException sde) - - - /** - * @deprecated Throws InvalidRequestException since 3.0. Please use the CQL3 version instead. - */ - CqlPreparedResult prepare_cql_query(1:required binary query, 2:required Compression compression) - throws (1:InvalidRequestException ire) - - /** - * Prepare a CQL3 (Cassandra Query Language) statement by compiling and returning - * - the type of CQL statement - * - an id token of the compiled CQL stored on the server side. - * - a count of the discovered bound markers in the statement - */ - CqlPreparedResult prepare_cql3_query(1:required binary query, 2:required Compression compression) - throws (1:InvalidRequestException ire) - - - /** - * @deprecated Throws InvalidRequestException since 3.0. Please use the CQL3 version instead. - */ - CqlResult execute_prepared_cql_query(1:required i32 itemId, 2:required list values) - throws (1:InvalidRequestException ire, - 2:UnavailableException ue, - 3:TimedOutException te, - 4:SchemaDisagreementException sde) - - /** - * Executes a prepared CQL3 (Cassandra Query Language) statement by passing an id token, a list of variables - * to bind, and the consistency level, and returns a CqlResult containing the results. - */ - CqlResult execute_prepared_cql3_query(1:required i32 itemId, 2:required list values, 3:required ConsistencyLevel consistency) - throws (1:InvalidRequestException ire, - 2:UnavailableException ue, - 3:TimedOutException te, - 4:SchemaDisagreementException sde) - - /** - * @deprecated This is now a no-op. Please use the CQL3 specific methods instead. - */ - void set_cql_version(1: required string version) throws (1:InvalidRequestException ire) -} diff --git a/main.cc b/main.cc index a1a28495b7..783b577f94 100644 --- a/main.cc +++ b/main.cc @@ -83,7 +83,6 @@ #include "sstables_loader.hh" #include "cql3/cql_config.hh" #include "transport/controller.hh" -#include "thrift/controller.hh" #include "service/memory_limiter.hh" #include "service/endpoint_lifecycle_subscriber.hh" #include "db/schema_tables.hh" @@ -2047,7 +2046,6 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl // Register controllers after drain_on_shutdown() below, so that even on start // failure drain is called and stops controllers cql_transport::controller cql_server_ctl(auth_service, mm_notifier, gossiper, qp, service_memory_limiter, sl_controller, lifecycle_notifier, *cfg, cql_sg_stats_key, maintenance_socket_enabled::no, dbcfg.statement_scheduling_group); - ::thrift_controller thrift_ctl(db, auth_service, qp, service_memory_limiter, ss, proxy, dbcfg.statement_scheduling_group); alternator::controller alternator_ctl(gossiper, proxy, mm, sys_dist_ks, cdc_generation_service, service_memory_limiter, auth_service, sl_controller, *cfg, dbcfg.statement_scheduling_group); redis::controller redis_ctl(proxy, auth_service, mm, *cfg, gossiper, dbcfg.statement_scheduling_group); @@ -2062,10 +2060,9 @@ To start the scylla server proper, simply invoke as: scylla server (or just scyl api::unset_transport_controller(ctx).get(); }); - ss.local().register_protocol_server(thrift_ctl, cfg->start_rpc()).get(); - api::set_rpc_controller(ctx, thrift_ctl).get(); - auto stop_rpc_controller = defer_verbose_shutdown("rpc controller API", [&ctx] { - api::unset_rpc_controller(ctx).get(); + api::set_thrift_controller(ctx).get(); + auto stop_thrift_controller = defer_verbose_shutdown("thrift controller API", [&ctx] { + api::unset_thrift_controller(ctx).get(); }); ss.local().register_protocol_server(alternator_ctl, cfg->alternator_port() || cfg->alternator_https_port()).get(); diff --git a/schema/schema.cc b/schema/schema.cc index e639020267..26c6637e89 100644 --- a/schema/schema.cc +++ b/schema/schema.cc @@ -328,9 +328,6 @@ void schema::rebuild() { _column_mapping = column_mapping(std::move(cm_columns), static_columns_count()); } - thrift()._compound = is_compound(); - thrift()._is_dynamic = clustering_key_size() > 0; - if (is_counter()) { for (auto&& cdef : boost::range::join(static_columns(), regular_columns())) { if (!cdef.type->is_counter()) { @@ -415,8 +412,6 @@ schema::schema(private_tag, const raw_schema& raw, std::optional def._dropped_at = std::max(def._dropped_at, dropped_at_it->second.timestamp); } - def._thrift_bits = column_definition::thrift_bits(); - { // is_on_all_components // TODO : In origin, this predicate is "componentIndex == null", which is true in @@ -436,7 +431,6 @@ schema::schema(private_tag, const raw_schema& raw, std::optional [[fallthrough]]; default: // Or any other column where "comparator" is not compound - def._thrift_bits.is_on_all_components = !thrift().has_compound_comparator(); break; } } @@ -519,18 +513,6 @@ schema::registry_entry() const noexcept { return _registry_entry; } -sstring schema::thrift_key_validator() const { - if (partition_key_size() == 1) { - return partition_key_columns().begin()->type->name(); - } else { - auto type_params = fmt::join(partition_key_columns() - | boost::adaptors::transformed(std::mem_fn(&column_definition::type)) - | boost::adaptors::transformed(std::mem_fn(&abstract_type::name)), - ", "); - return format("org.apache.cassandra.db.marshal.CompositeType({})", type_params); - } -} - bool schema::has_multi_cell_collections() const { return boost::algorithm::any_of(all_columns(), [] (const column_definition& cdef) { @@ -699,7 +681,6 @@ auto fmt::formatter::format(const schema& s, fmt::format_context& ctx) c out = fmt::format_to(out, ",comment={}", s._raw._comment); out = fmt::format_to(out, ",tombstoneGcOptions={}", s.tombstone_gc_options().to_sstring()); out = fmt::format_to(out, ",gcGraceSeconds={}", s._raw._gc_grace_seconds); - out = fmt::format_to(out, ",keyValidator={}", s.thrift_key_validator()); out = fmt::format_to(out, ",minCompactionThreshold={}", s._raw._min_compaction_threshold); out = fmt::format_to(out, ",maxCompactionThreshold={}", s._raw._max_compaction_threshold); out = fmt::format_to(out, ",columnMetadata=["); @@ -1053,14 +1034,6 @@ generate_legacy_id(const sstring& ks_name, const sstring& cf_name) { return table_id(utils::UUID_gen::get_name_UUID(ks_name + cf_name)); } -bool thrift_schema::has_compound_comparator() const { - return _compound; -} - -bool thrift_schema::is_dynamic() const { - return _is_dynamic; -} - schema_builder& schema_builder::set_compaction_strategy_options(std::map&& options) { _raw._compaction_strategy_options = std::move(options); return *this; diff --git a/schema/schema.hh b/schema/schema.hh index b93b4ec430..29a01be5d7 100644 --- a/schema/schema.hh +++ b/schema/schema.hh @@ -305,7 +305,6 @@ public: , _is_counter(other._is_counter) , _is_view_virtual(other._is_view_virtual) , _computation(other.get_computation_ptr()) - , _thrift_bits(other._thrift_bits) , type(other.type) , id(other.id) , ordinal_id(other.ordinal_id) @@ -376,18 +375,6 @@ public: class schema_builder; -/* - * Sub-schema for thrift aspects. Should be kept isolated (and starved) - */ -class thrift_schema { - bool _compound = true; - bool _is_dynamic = false; -public: - bool has_compound_comparator() const; - bool is_dynamic() const; - friend class schema; -}; - bool operator==(const column_definition&, const column_definition&); static constexpr int DEFAULT_MIN_COMPACTION_THRESHOLD = 4; @@ -610,7 +597,6 @@ private: }; raw_schema _raw; schema_static_props _static_props; - thrift_schema _thrift; v3_columns _v3_columns; mutable schema_registry_entry* _registry_entry = nullptr; std::unique_ptr<::view_info> _view_info; @@ -674,7 +660,6 @@ public: double bloom_filter_fp_chance() const { return _raw._bloom_filter_fp_chance; } - sstring thrift_key_validator() const; const compression_parameters& get_compressor_params() const { return _raw._compressor_params; } @@ -699,12 +684,6 @@ public: return !is_super() && !is_dense() && !is_compound(); } - thrift_schema& thrift() { - return _thrift; - } - const thrift_schema& thrift() const { - return _thrift; - } const table_id& id() const { return _raw._id; } diff --git a/scripts/metrics-config.yml b/scripts/metrics-config.yml index df650bc7f8..d3b96522d2 100644 --- a/scripts/metrics-config.yml +++ b/scripts/metrics-config.yml @@ -33,7 +33,6 @@ _long_description_prefix: ["total number of write requests", "number of write requests that failed", "background_replica_writes_failed", "number of write operations in a read repair context"] _category: "storage_proxy_coordinator" allowmismatch: true -"thrift/server.cc": skip "tracing/tracing.cc": params: "max_pending_trace_records + write_event_records_threshold": "max_pending_trace_records + write_event_records_threshold" diff --git a/service/client_state.hh b/service/client_state.hh index 805e2cf4aa..071bce8498 100644 --- a/service/client_state.hh +++ b/service/client_state.hh @@ -69,7 +69,6 @@ private: , _user(cs->_user) , _auth_state(cs->_auth_state) , _is_internal(cs->_is_internal) - , _is_thrift(cs->_is_thrift) , _remote_address(cs->_remote_address) , _auth_service(auth_service ? &auth_service->local() : nullptr) , _sl_controller(sl_controller ? &sl_controller->local() : nullptr) @@ -110,7 +109,6 @@ private: // isInternal is used to mark ClientState as used by some internal component // that should have an ability to modify system keyspace. bool _is_internal; - bool _is_thrift; // The biggest timestamp that was returned by getTimestamp/assigned to a query static thread_local api::timestamp_type _last_timestamp_micros; @@ -162,10 +160,8 @@ public: auth::service& auth_service, qos::service_level_controller* sl_controller, timeout_config timeout_config, - const socket_address& remote_address = socket_address(), - bool thrift = false) + const socket_address& remote_address = socket_address()) : _is_internal(false) - , _is_thrift(thrift) , _remote_address(remote_address) , _auth_service(&auth_service) , _sl_controller(sl_controller) @@ -202,7 +198,6 @@ public: client_state(internal_tag, const timeout_config& config) : _keyspace("system") , _is_internal(true) - , _is_thrift(false) , _default_timeout_config(config) , _timeout_config(config) {} @@ -211,7 +206,6 @@ public: : _user(auth::authenticated_user(username)) , _auth_state(auth_state::READY) , _is_internal(true) - , _is_thrift(false) , _auth_service(&auth_service) , _sl_controller(&sl_controller) {} @@ -226,10 +220,6 @@ public: return _auth_service; } - bool is_thrift() const { - return _is_thrift; - } - bool is_internal() const { return _is_internal; } diff --git a/service/migration_manager.cc b/service/migration_manager.cc index a553536153..301f4a127c 100644 --- a/service/migration_manager.cc +++ b/service/migration_manager.cc @@ -715,7 +715,7 @@ future<> prepare_new_column_family_announcement(std::vector& mutations } future> prepare_column_family_update_announcement(storage_proxy& sp, - schema_ptr cfm, bool from_thrift, std::vector view_updates, api::timestamp_type ts) { + schema_ptr cfm, std::vector view_updates, api::timestamp_type ts) { warn(unimplemented::cause::VALIDATION); #if 0 cfm.validate(); @@ -731,7 +731,7 @@ future> prepare_column_family_update_announcement(storage_ auto mutations = co_await seastar::async([&] { // Can call notifier when it creates new indexes, so needs to run in Seastar thread - return db::schema_tables::make_update_table_mutations(db, keyspace, old_schema, cfm, ts, from_thrift); + return db::schema_tables::make_update_table_mutations(db, keyspace, old_schema, cfm, ts); }); for (auto&& view : view_updates) { auto& old_view = keyspace->cf_meta_data().at(view->cf_name()); @@ -836,7 +836,7 @@ future> prepare_column_family_drop_announcement(storage_pr std::vector drop_si_mutations; if (!schema->all_indices().empty()) { auto builder = schema_builder(schema).without_indexes(); - drop_si_mutations = db::schema_tables::make_update_table_mutations(db, keyspace, schema, builder.build(), ts, false); + drop_si_mutations = db::schema_tables::make_update_table_mutations(db, keyspace, schema, builder.build(), ts); } auto mutations = db::schema_tables::make_drop_table_mutations(keyspace, schema, ts); mutations.insert(mutations.end(), std::make_move_iterator(drop_si_mutations.begin()), std::make_move_iterator(drop_si_mutations.end())); diff --git a/service/migration_manager.hh b/service/migration_manager.hh index 31bfeed69a..7357ec0ee0 100644 --- a/service/migration_manager.hh +++ b/service/migration_manager.hh @@ -215,7 +215,7 @@ std::vector prepare_new_keyspace_announcement(replica::database& db, l // The timestamp parameter can be used to ensure that all nodes update their internal tables' schemas // with identical timestamps, which can prevent an undeeded schema exchange future> prepare_column_family_update_announcement(storage_proxy& sp, - schema_ptr cfm, bool from_thrift, std::vector view_updates, api::timestamp_type ts); + schema_ptr cfm, std::vector view_updates, api::timestamp_type ts); future> prepare_new_column_family_announcement(storage_proxy& sp, schema_ptr cfm, api::timestamp_type timestamp); // The ksm parameter can describe a keyspace that hasn't been created yet. diff --git a/table_helper.cc b/table_helper.cc index 739f8a9f6c..db78fd6ebc 100644 --- a/table_helper.cc +++ b/table_helper.cc @@ -76,7 +76,7 @@ future<> table_helper::cache_table_info(cql3::query_processor& qp, service::migr return now(); } - return qp.prepare(_insert_cql, qs.get_client_state(), false) + return qp.prepare(_insert_cql, qs.get_client_state()) .then([this] (shared_ptr msg_ptr) noexcept { _prepared_stmt = std::move(msg_ptr->get_prepared()); shared_ptr cql_stmt = _prepared_stmt->statement; @@ -91,7 +91,7 @@ future<> table_helper::cache_table_info(cql3::query_processor& qp, service::migr // we have already prepared the fallback statement return now(); } - return qp.prepare(_insert_cql_fallback.value(), qs.get_client_state(), false) + return qp.prepare(_insert_cql_fallback.value(), qs.get_client_state()) .then([this] (shared_ptr msg_ptr) noexcept { _prepared_stmt = std::move(msg_ptr->get_prepared()); shared_ptr cql_stmt = _prepared_stmt->statement; diff --git a/test/boost/config_test.cc b/test/boost/config_test.cc index 49350ac0b2..4457d0909e 100644 --- a/test/boost/config_test.cc +++ b/test/boost/config_test.cc @@ -158,10 +158,10 @@ partitioner: org.apache.cassandra.dht.Murmur3Partitioner # commitlog_directory: /var/lib/cassandra/commitlog # policy for data disk failures: -# die: shut down gossip and Thrift and kill the JVM for any fs errors or +# die: shut down gossip and kill the JVM for any fs errors or # single-sstable errors, so the node can be replaced. -# stop_paranoid: shut down gossip and Thrift even for single-sstable errors. -# stop: shut down gossip and Thrift, leaving the node effectively dead, but +# stop_paranoid: shut down gossip even for single-sstable errors. +# stop: shut down gossip, leaving the node effectively dead, but # can still be inspected via JMX. # best_effort: stop using the failed disk and respond to requests based on # remaining available sstables. This means you WILL see obsolete @@ -170,8 +170,8 @@ partitioner: org.apache.cassandra.dht.Murmur3Partitioner disk_failure_policy: stop # policy for commit disk failures: -# die: shut down gossip and Thrift and kill the JVM, so the node can be replaced. -# stop: shut down gossip and Thrift, leaving the node effectively dead, but +# die: shut down gossip and kill the JVM, so the node can be replaced. +# stop: shut down gossip, leaving the node effectively dead, but # can still be inspected via JMX. # stop_commit: shutdown the commit log, letting writes collect but # continuing to service reads, as in pre-2.0.5 Cassandra @@ -441,10 +441,7 @@ native_transport_port: 9042 # be rejected as invalid. The default is 256MB. # native_transport_max_frame_size_in_mb: 256 -# Whether to start the thrift rpc server. -start_rpc: true - -# The address or interface to bind the Thrift RPC service and native transport +# The address or interface to bind the native transport # server to. # # Set rpc_address OR rpc_interface, not both. Interfaces must correspond @@ -458,9 +455,6 @@ start_rpc: true rpc_address: localhost # rpc_interface: eth1 -# port for Thrift to listen for clients on -rpc_port: 9160 - # RPC address to broadcast to drivers and other Cassandra nodes. This cannot # be set to 0.0.0.0. If left blank, this will be set to the value of # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must @@ -470,43 +464,6 @@ rpc_port: 9160 # enable or disable keepalive on rpc/native connections rpc_keepalive: true -# Cassandra provides two out-of-the-box options for the RPC Server: -# -# sync -> One thread per thrift connection. For a very large number of clients, memory -# will be your limiting factor. On a 64 bit JVM, 180KB is the minimum stack size -# per thread, and that will correspond to your use of virtual memory (but physical memory -# may be limited depending on use of stack space). -# -# hsha -> Stands for "half synchronous, half asynchronous." All thrift clients are handled -# asynchronously using a small number of threads that does not vary with the amount -# of thrift clients (and thus scales well to many clients). The rpc requests are still -# synchronous (one thread per active request). If hsha is selected then it is essential -# that rpc_max_threads is changed from the default value of unlimited. -# -# The default is sync because on Windows hsha is about 30% slower. On Linux, -# sync/hsha performance is about the same, with hsha of course using less memory. -# -# Alternatively, can provide your own RPC server by providing the fully-qualified class name -# of an o.a.c.t.TServerFactory that can create an instance of it. -rpc_server_type: sync - -# Uncomment rpc_min|max_thread to set request pool size limits. -# -# Regardless of your choice of RPC server (see above), the number of maximum requests in the -# RPC thread pool dictates how many concurrent requests are possible (but if you are using the sync -# RPC server, it also dictates the number of clients that can be connected at all). -# -# The default is unlimited and thus provides no protection against clients overwhelming the server. You are -# encouraged to set a maximum that makes sense for you in production, but do keep in mind that -# rpc_max_threads represents the maximum number of client requests this server may execute concurrently. -# -# rpc_min_threads: 16 -# rpc_max_threads: 2048 - -# uncomment to set socket buffer sizes on rpc connections -# rpc_send_buff_size_in_bytes: -# rpc_recv_buff_size_in_bytes: - # Uncomment to set socket buffer size for internode communication # Note that when setting this, the buffer size is limited by net.core.wmem_max # and when not setting it it is defined by net.ipv4.tcp_wmem @@ -519,9 +476,6 @@ rpc_server_type: sync # internode_send_buff_size_in_bytes: # internode_recv_buff_size_in_bytes: -# Frame size for thrift (maximum message length). -thrift_framed_transport_size_in_mb: 15 - # Set to true to have Cassandra create a hard link to each sstable # flushed or streamed locally in a backups/ subdirectory of the # keyspace data. Removing these links is the operator's diff --git a/test/boost/cql_functions_test.cc b/test/boost/cql_functions_test.cc index 49590ca824..9458ebb5a6 100644 --- a/test/boost/cql_functions_test.cc +++ b/test/boost/cql_functions_test.cc @@ -54,7 +54,6 @@ SEASTAR_TEST_CASE(test_functions) { virtual void visit(const result_message::void_message&) override { throw "bad"; } virtual void visit(const result_message::set_keyspace&) override { throw "bad"; } virtual void visit(const result_message::prepared::cql&) override { throw "bad"; } - virtual void visit(const result_message::prepared::thrift&) override { throw "bad"; } virtual void visit(const result_message::schema_change&) override { throw "bad"; } virtual void visit(const result_message::rows& rows) override { const auto& rs = rows.rs().result_set(); diff --git a/test/boost/schema_change_test.cc b/test/boost/schema_change_test.cc index 651025f5f0..405412ca16 100644 --- a/test/boost/schema_change_test.cc +++ b/test/boost/schema_change_test.cc @@ -63,7 +63,7 @@ SEASTAR_TEST_CASE(test_new_schema_with_no_structural_change_is_propagated) { auto group0_guard = mm.start_group0_operation().get(); auto ts = group0_guard.write_timestamp(); mm.announce(service::prepare_column_family_update_announcement(mm.get_storage_proxy(), - new_schema, false, std::vector(), ts).get(), std::move(group0_guard), "").get(); + new_schema, std::vector(), ts).get(), std::move(group0_guard), "").get(); BOOST_REQUIRE_NE(e.db().local().find_schema(old_schema->id())->version(), old_table_version); BOOST_REQUIRE_NE(e.db().local().get_version(), old_node_version); @@ -100,7 +100,7 @@ SEASTAR_TEST_CASE(test_schema_is_updated_in_keyspace) { auto group0_guard = mm.start_group0_operation().get(); auto ts = group0_guard.write_timestamp(); mm.announce(service::prepare_column_family_update_announcement(mm.get_storage_proxy(), - new_schema, false, std::vector(), ts).get(), std::move(group0_guard), "").get(); + new_schema, std::vector(), ts).get(), std::move(group0_guard), "").get(); s = e.local_db().find_schema(old_schema->id()); BOOST_REQUIRE_NE(*old_schema, *s); @@ -198,7 +198,7 @@ SEASTAR_TEST_CASE(test_concurrent_column_addition) { auto group0_guard = mm.start_group0_operation().get(); auto&& keyspace = e.db().local().find_keyspace(s0->ks_name()).metadata(); auto muts = db::schema_tables::make_update_table_mutations(e.db().local(), keyspace, s0, s2, - group0_guard.write_timestamp(), false); + group0_guard.write_timestamp()); mm.announce(std::move(muts), std::move(group0_guard), "").get(); } @@ -364,7 +364,7 @@ SEASTAR_TEST_CASE(test_combined_column_add_and_drop) { { auto group0_guard = mm.start_group0_operation().get(); auto muts = db::schema_tables::make_update_table_mutations(e.db().local(), keyspace, s1, s2, - group0_guard.write_timestamp(), false); + group0_guard.write_timestamp()); mm.announce(std::move(muts), std::move(group0_guard), "").get(); } @@ -382,7 +382,7 @@ SEASTAR_TEST_CASE(test_combined_column_add_and_drop) { auto group0_guard = mm.start_group0_operation().get(); auto muts = db::schema_tables::make_update_table_mutations(e.db().local(), keyspace, s3, s4, - group0_guard.write_timestamp(), false); + group0_guard.write_timestamp()); mm.announce(std::move(muts), std::move(group0_guard), "").get(); } diff --git a/test/nodetool/test_info.py b/test/nodetool/test_info.py index 5e4e26eb41..60b33b3848 100644 --- a/test/nodetool/test_info.py +++ b/test/nodetool/test_info.py @@ -100,7 +100,7 @@ def test_info(request, nodetool, display_all_tokens): expected_requests = [ expected_request('GET', '/storage_service/gossiping', response=True), expected_request('GET', '/storage_service/hostid/local', response=host_id), - expected_request('GET', '/storage_service/rpc_server', response=True), + expected_request('GET', '/storage_service/rpc_server', response=False), expected_request('GET', '/storage_service/native_transport', response=True), expected_request('GET', '/storage_service/load', response=load), expected_request('GET', '/storage_service/generation_number', response=generation_number), @@ -175,7 +175,7 @@ def test_info(request, nodetool, display_all_tokens): expected_output = f'''\ {'ID':<23}: {host_id} {'Gossip active':<23}: true -{'Thrift active':<23}: true +{'Thrift active':<23}: false {'Native Transport active':<23}: true {'Load':<23}: {format_size(load)} {'Generation No':<23}: {generation_number} diff --git a/thrift/CMakeLists.txt b/thrift/CMakeLists.txt deleted file mode 100644 index a3d33a668d..0000000000 --- a/thrift/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -add_library(thrift STATIC) -target_sources(thrift - PRIVATE - controller.cc - handler.cc - server.cc - thrift_validation.cc) -target_include_directories(thrift - PUBLIC - ${CMAKE_SOURCE_DIR}) -target_link_libraries(thrift - PUBLIC - Seastar::seastar - xxHash::xxhash - PRIVATE - interface - absl::headers - Thrift::thrift) - -check_headers(check-headers thrift - GLOB_RECURSE ${CMAKE_CURRENT_SOURCE_DIR}/*.hh) diff --git a/thrift/controller.cc b/thrift/controller.cc deleted file mode 100644 index 4aa511822e..0000000000 --- a/thrift/controller.cc +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (C) 2020-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -#include "thrift/controller.hh" -#include -#include "thrift/server.hh" -#include "replica/database.hh" -#include "db/config.hh" -#include "log.hh" - -static logging::logger clogger("thrift_controller"); - -thrift_controller::thrift_controller(distributed& db, sharded& auth, - sharded& qp, sharded& ml, - sharded& ss, sharded& proxy, - seastar::scheduling_group sg) - : protocol_server(sg) - , _ops_sem(1) - , _db(db) - , _auth_service(auth) - , _qp(qp) - , _mem_limiter(ml) - , _ss(ss) - , _proxy(proxy) -{ } - -sstring thrift_controller::name() const { - return "rpc"; -} - -sstring thrift_controller::protocol() const { - return "thrift"; -} - -sstring thrift_controller::protocol_version() const { - return ::cassandra::thrift_version; -} - -std::vector thrift_controller::listen_addresses() const { - if (_server && _addr) { - return {*_addr}; - } - return {}; -} - -future<> thrift_controller::start_server() { - if (!_ops_sem.try_wait()) { - throw std::runtime_error(format("Thrift server is stopping, try again later")); - } - - return do_start_server().finally([this] { _ops_sem.signal(); }); -} - -future<> thrift_controller::do_start_server() { - if (_server) { - return make_ready_future<>(); - } - - seastar::thread_attributes attr; - attr.sched_group = _sched_group; - return seastar::async(std::move(attr), [this] { - auto tserver = std::make_unique>(); - _addr.reset(); - - auto& cfg = _db.local().get_config(); - auto preferred = cfg.rpc_interface_prefer_ipv6() ? std::make_optional(net::inet_address::family::INET6) : std::nullopt; - auto family = cfg.enable_ipv6_dns_lookup() || preferred ? std::nullopt : std::make_optional(net::inet_address::family::INET); - auto keepalive = cfg.rpc_keepalive(); - - auto ip = utils::resolve(cfg.rpc_address, family, preferred).get(); - auto port = cfg.rpc_port(); - _addr.emplace(ip, port); - - auto tsc = sharded_parameter([&cfg] { - return thrift_server_config { - .timeout_config = updateable_timeout_config(cfg), - .max_request_size = cfg.thrift_max_message_length_in_mb() * (uint64_t(1) << 20), - }; - }); - tserver->start(sharded_parameter([this] { return _db.local().as_data_dictionary(); }), std::ref(_qp), std::ref(_ss), std::ref(_proxy), std::ref(_auth_service), std::ref(_mem_limiter), std::move(tsc)).get(); - // #293 - do not stop anything - //engine().at_exit([tserver] { - // return tserver->stop(); - //}); - tserver->invoke_on_all(&thrift_server::listen, socket_address{ip, port}, keepalive).get(); - clogger.info("Thrift server listening on {}:{} ...", ip, port); - _server = std::move(tserver); - }); -} - -future<> thrift_controller::stop_server() { - assert(this_shard_id() == 0); - - if (_stopped) { - return make_ready_future<>(); - } - - return _ops_sem.wait().then([this] { - _stopped = true; - _ops_sem.broken(); - _addr.reset(); - return do_stop_server(); - }); -} - -future<> thrift_controller::request_stop_server() { - if (!_ops_sem.try_wait()) { - throw std::runtime_error(format("Thrift server is starting, try again later")); - } - - return with_scheduling_group(_sched_group, [this] { - return do_stop_server(); - }).finally([this] { _ops_sem.signal(); }); -} - -future<> thrift_controller::do_stop_server() { - return do_with(std::move(_server), [] (std::unique_ptr>& tserver) { - if (tserver) { - return tserver->stop().then([] { - clogger.info("Thrift server stopped"); - }); - } - return make_ready_future<>(); - }); -} diff --git a/thrift/controller.hh b/thrift/controller.hh deleted file mode 100644 index 89423518fe..0000000000 --- a/thrift/controller.hh +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2020-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -#pragma once - -#include -#include -#include -#include "service/memory_limiter.hh" -#include "protocol_server.hh" - -using namespace seastar; - -class thrift_server; - -namespace replica { -class database; -} - -namespace auth { class service; } -namespace cql3 { class query_processor; } -namespace service { -class storage_service; -class storage_proxy; -} - -class thrift_controller : public protocol_server { - std::unique_ptr> _server; - std::optional _addr; - semaphore _ops_sem; /* protects start/stop operations on _server */ - bool _stopped = false; - - distributed& _db; - sharded& _auth_service; - sharded& _qp; - sharded& _mem_limiter; - sharded& _ss; - sharded& _proxy; - - future<> do_start_server(); - future<> do_stop_server(); - -public: - thrift_controller(distributed&, sharded&, sharded&, sharded&, sharded& ss, sharded& proxy, seastar::scheduling_group sg); - virtual sstring name() const override; - virtual sstring protocol() const override; - virtual sstring protocol_version() const override; - virtual std::vector listen_addresses() const override; - virtual future<> start_server() override; - virtual future<> stop_server() override; - virtual future<> request_stop_server() override; -}; diff --git a/thrift/handler.cc b/thrift/handler.cc deleted file mode 100644 index c0147dcea5..0000000000 --- a/thrift/handler.cc +++ /dev/null @@ -1,2082 +0,0 @@ -/* - * Copyright (C) 2014-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -// Some thrift headers include other files from within namespaces, -// which is totally broken. Include those files here to avoid -// breakage: -#include -// end thrift workaround -#include "Cassandra.h" -#include -#include -#include "replica/database.hh" // for database::get_version() -#include "data_dictionary/data_dictionary.hh" -#include -#include -#include "mutation/frozen_mutation.hh" -#include "utils/UUID_gen.hh" -#include -#include -#include "db/marshal/type_parser.hh" -#include "service/migration_manager.hh" -#include "service/storage_proxy.hh" -#include "utils/class_registrator.hh" -#include "noexcept_traits.hh" -#include "schema/schema_registry.hh" -#include "thrift/utils.hh" -#include "schema/schema_builder.hh" -#include "thrift/thrift_validation.hh" -#include "service/storage_service.hh" -#include "service/query_state.hh" -#include "cql3/query_processor.hh" -#include "cql3/column_identifier.hh" -#include "timeout_config.hh" -#include "mutation/mutation.hh" -#include -#include -#include -#include -#include -#include -#include "query-result-reader.hh" -#include "thrift/server.hh" -#include "db/config.hh" -#include "locator/abstract_replication_strategy.hh" - -#include - -#ifdef THRIFT_USES_BOOST -namespace thrift_fn = tcxx; -#else -namespace thrift_fn = std; -#endif - -using namespace ::apache::thrift; -using namespace ::apache::thrift::protocol; -using namespace ::apache::thrift::async; - -using namespace ::cassandra; - -using namespace thrift; - -template <> struct fmt::formatter : fmt::ostream_formatter {}; - -class unimplemented_exception : public std::exception { -public: - virtual const char* what() const throw () override { return "sorry, not implemented"; } -}; - -void pass_unimplemented(const thrift_fn::function& exn_cob) { - exn_cob(::apache::thrift::TDelayedException::delayException(unimplemented_exception())); -} - -class delayed_exception_wrapper : public ::apache::thrift::TDelayedException { - std::exception_ptr _ex; -public: - delayed_exception_wrapper(std::exception_ptr ex) : _ex(std::move(ex)) {} - virtual void throw_it() override { - // Thrift auto-wraps unexpected exceptions (those not derived from TException) - // with a TException, but with a fairly bad what(). So detect this, and - // provide our own TException with a better what(). - try { - std::rethrow_exception(std::move(_ex)); - } catch (const ::apache::thrift::TException&) { - // It's an expected exception, so assume the message - // is fine. Also, we don't want to change its type. - throw; - } catch (no_such_class& nc) { - throw make_exception(nc.what()); - } catch (marshal_exception& me) { - throw make_exception(me.what()); - } catch (exceptions::already_exists_exception& ae) { - throw make_exception(ae.what()); - } catch (exceptions::configuration_exception& ce) { - throw make_exception(ce.what()); - } catch (exceptions::invalid_request_exception& ire) { - throw make_exception(ire.what()); - } catch (data_dictionary::no_such_column_family& nocf) { - throw make_exception(nocf.what()); - } catch (data_dictionary::no_such_keyspace&) { - throw NotFoundException(); - } catch (exceptions::syntax_exception& se) { - throw make_exception("syntax error: {}", se.what()); - } catch (exceptions::authentication_exception& ae) { - throw make_exception(ae.what()); - } catch (exceptions::unauthorized_exception& ue) { - throw make_exception(ue.what()); - } catch (std::exception& e) { - // Unexpected exception, wrap it - throw ::apache::thrift::TException(std::string("Internal server error: ") + e.what()); - } catch (...) { - // Unexpected exception, wrap it, unfortunately without any info - throw ::apache::thrift::TException("Internal server error"); - } - } -}; - -template -void -with_cob(thrift_fn::function&& cob, - thrift_fn::function&& exn_cob, - Func&& func) { - // then_wrapped() terminates the fiber by calling one of the cob objects - (void)futurize_invoke([func = std::forward(func)] { - return noexcept_movable::wrap(func()); - }).then_wrapped([cob = std::move(cob), exn_cob = std::move(exn_cob)] (auto&& f) { - try { - cob(noexcept_movable::unwrap(f.get())); - } catch (...) { - delayed_exception_wrapper dew(std::current_exception()); - exn_cob(&dew); - } - }); -} - -template -void -with_cob(thrift_fn::function&& cob, - thrift_fn::function&& exn_cob, - Func&& func) { - // then_wrapped() terminates the fiber by calling one of the cob objects - (void)futurize_invoke(func).then_wrapped([cob = std::move(cob), exn_cob = std::move(exn_cob)] (future<> f) { - try { - f.get(); - cob(); - } catch (...) { - delayed_exception_wrapper dew(std::current_exception()); - exn_cob(&dew); - } - }); -} - -template -void -with_exn_cob(thrift_fn::function&& exn_cob, Func&& func) { - // then_wrapped() terminates the fiber by calling one of the cob objects - (void)futurize_invoke(func).then_wrapped([exn_cob = std::move(exn_cob)] (future<> f) { - try { - f.get(); - } catch (...) { - delayed_exception_wrapper dew(std::current_exception()); - exn_cob(&dew); - } - }); -} - -std::string bytes_to_string(bytes_view v) { - return { reinterpret_cast(v.begin()), v.size() }; -} - -std::string bytes_to_string(query::result_bytes_view v) { - std::string str; - str.reserve(v.size_bytes()); - using boost::range::for_each; - for_each(v, [&] (bytes_view fragment) { - auto view = std::string_view(reinterpret_cast(fragment.data()), fragment.size()); - str.insert(str.end(), view.begin(), view.end()); - }); - return str; -} - -std::string bytes_to_string(managed_bytes_view v) { - std::string str; - str.reserve(v.size_bytes()); - for (auto fragment : fragment_range(v)) { - auto view = std::string_view(reinterpret_cast(fragment.data()), fragment.size()); - str.insert(str.end(), view.begin(), view.end()); - } - return str; -} - -namespace thrift { -template -concept Aggregator = - requires() { typename T::type; } - && requires(T aggregator, typename T::type* aggregation, const bytes& name, const query::result_atomic_cell_view& cell) { - { aggregator.on_column(aggregation, name, cell) } -> std::same_as; - }; -} - -enum class query_order { no, yes }; - -class thrift_handler : public CassandraCobSvIf { - data_dictionary::database _db; - distributed& _query_processor; - sharded& _ss; - sharded& _proxy; - ::timeout_config _timeout_config; - service::client_state _client_state; - service::query_state _query_state; - service_permit& _current_permit; -private: - template - void - with_schema(Cob&& cob, - thrift_fn::function&& exn_cob, - const std::string& cf, - Func&& func) { - with_cob(std::move(cob), std::move(exn_cob), [this, &cf, func = std::move(func)] { - auto schema = lookup_schema(_db, current_keyspace(), cf); - return func(std::move(schema)); - }); - } -public: - explicit thrift_handler(data_dictionary::database db, distributed& qp, sharded& ss, sharded& proxy, - auth::service& auth_service, ::timeout_config timeout_config, service_permit& current_permit) - : _db(db) - , _query_processor(qp) - , _ss(ss) - , _proxy(proxy) - , _timeout_config(timeout_config) - , _client_state(service::client_state::external_tag{}, auth_service, nullptr, _timeout_config, socket_address(), true) - // FIXME: Handlers are not created per query, but rather per connection, so it makes little sense to store - // service permits in here. The query state should be reinstantiated per query - AFAIK it's only used - // for CQL queries which piggy-back on Thrift protocol. - , _query_state(_client_state, /*FIXME: pass real permit*/empty_service_permit()) - , _current_permit(current_permit) - { } - - const sstring& current_keyspace() const { - return _query_state.get_client_state().get_raw_keyspace(); - } - - void validate_login() const { - return _query_state.get_client_state().validate_login(); - }; - - void login(thrift_fn::function cob, thrift_fn::function exn_cob, const AuthenticationRequest& auth_request) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - auth::authenticator::credentials_map creds(auth_request.credentials.begin(), auth_request.credentials.end()); - auto& auth_service = *_query_state.get_client_state().get_auth_service(); - return auth_service.underlying_authenticator().authenticate(creds).then([this] (auto user) { - _query_state.get_client_state().set_login(std::move(user)); - }); - }); - } - - void set_keyspace(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& keyspace) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - _query_state.get_client_state().set_keyspace(_db.real_database(), keyspace); - }); - } - - void get(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const ColumnPath& column_path, const ConsistencyLevel::type consistency_level) { - return get_slice([cob = std::move(cob)](auto&& results) { - if (results.empty()) { - throw NotFoundException(); - } - return cob(std::move(results.front())); - }, exn_cob, key, column_path_to_column_parent(column_path), column_path_to_slice_predicate(column_path), std::move(consistency_level)); - } - - void get_slice(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& key, const ColumnParent& column_parent, const SlicePredicate& predicate, const ConsistencyLevel::type consistency_level) { - return multiget_slice([cob = std::move(cob)](auto&& results) { - if (!results.empty()) { - return cob(std::move(results.begin()->second)); - } - return cob({ }); - }, exn_cob, {key}, column_parent, predicate, consistency_level); - } - - void get_count(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const ColumnParent& column_parent, const SlicePredicate& predicate, const ConsistencyLevel::type consistency_level) { - return multiget_count([cob = std::move(cob)](auto&& results) { - if (!results.empty()) { - return cob(results.begin()->second); - } - return cob(0); - }, exn_cob, {key}, column_parent, predicate, consistency_level); - } - - void multiget_slice(thrift_fn::function > const& _return)> cob, thrift_fn::function exn_cob, const std::vector & keys, const ColumnParent& column_parent, const SlicePredicate& predicate, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_parent.column_family, [&](schema_ptr schema) { - if (!column_parent.super_column.empty()) { - fail(unimplemented::cause::SUPER); - } - auto& proxy = _proxy.local(); - auto cmd = slice_pred_to_read_cmd(proxy, *schema, predicate); - auto cell_limit = predicate.__isset.slice_range ? static_cast(predicate.slice_range.count) : std::numeric_limits::max(); - auto pranges = make_partition_ranges(*schema, keys); - auto f = _query_state.get_client_state().has_schema_access(*schema, auth::permission::SELECT); - return f.then([this, &proxy, schema, cmd, pranges = std::move(pranges), cell_limit, consistency_level, keys, permit = std::move(permit)]() mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.read_timeout; - return proxy.query(schema, cmd, std::move(pranges), cl_from_thrift(consistency_level), {timeout, std::move(permit), _query_state.get_client_state()}).then( - [schema, cmd, cell_limit, keys = std::move(keys)](service::storage_proxy::coordinator_query_result qr) { - return query::result_view::do_with(*qr.query_result, [schema, cmd, cell_limit, keys = std::move(keys)](query::result_view v) mutable { - if (schema->is_counter()) { - counter_column_aggregator aggregator(*schema, cmd->slice, cell_limit, std::move(keys)); - v.consume(cmd->slice, aggregator); - return aggregator.release(); - } - column_aggregator aggregator(*schema, cmd->slice, cell_limit, std::move(keys)); - v.consume(cmd->slice, aggregator); - return aggregator.release(); - }); - }); - }); - }); - } - - void multiget_count(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::vector & keys, const ColumnParent& column_parent, const SlicePredicate& predicate, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_parent.column_family, [&](schema_ptr schema) { - if (!column_parent.super_column.empty()) { - fail(unimplemented::cause::SUPER); - } - auto& proxy = _proxy.local(); - auto cmd = slice_pred_to_read_cmd(proxy, *schema, predicate); - auto cell_limit = predicate.__isset.slice_range ? static_cast(predicate.slice_range.count) : std::numeric_limits::max(); - auto pranges = make_partition_ranges(*schema, keys); - auto f = _query_state.get_client_state().has_schema_access(*schema, auth::permission::SELECT); - return f.then([this, &proxy, schema, cmd, pranges = std::move(pranges), cell_limit, consistency_level, keys, permit = std::move(permit)]() mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.read_timeout; - return proxy.query(schema, cmd, std::move(pranges), cl_from_thrift(consistency_level), {timeout, std::move(permit), _query_state.get_client_state()}).then( - [schema, cmd, cell_limit, keys = std::move(keys)](service::storage_proxy::coordinator_query_result qr) { - return query::result_view::do_with(*qr.query_result, [schema, cmd, cell_limit, keys = std::move(keys)](query::result_view v) mutable { - column_counter counter(*schema, cmd->slice, cell_limit, std::move(keys)); - v.consume(cmd->slice, counter); - return counter.release(); - }); - }); - }); - }); - } - - /** - * In origin, empty partitions are returned as part of the KeySlice, for which the key will be filled - * in but the columns vector will be empty. Since in our case we don't return empty partitions, we - * don't know which partition keys in the specified range we should return back to the client. So for - * now our behavior differs from Origin. - */ - void get_range_slices(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const ColumnParent& column_parent, const SlicePredicate& predicate, const KeyRange& range, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_parent.column_family, [&](schema_ptr schema) { - if (!column_parent.super_column.empty()) { - fail(unimplemented::cause::SUPER); - } - auto& proxy = _proxy.local(); - auto&& prange = make_partition_range(*schema, range); - auto cmd = slice_pred_to_read_cmd(proxy, *schema, predicate); - // KeyRange::count is the number of thrift rows to return, while - // SlicePredicte::slice_range::count limits the number of thrift columns. - if (schema->thrift().is_dynamic()) { - // For dynamic CFs we must limit the number of partitions returned. - cmd->partition_limit = range.count; - } else { - // For static CFs each thrift row maps to a CQL row. - cmd->set_row_limit(static_cast(range.count)); - } - auto f = _query_state.get_client_state().has_schema_access(*schema, auth::permission::SELECT); - return f.then([this, &proxy, schema, cmd, prange = std::move(prange), consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.range_read_timeout; - return proxy.query(schema, cmd, std::move(prange), cl_from_thrift(consistency_level), {timeout, std::move(permit), _query_state.get_client_state()}).then( - [schema, cmd](service::storage_proxy::coordinator_query_result qr) { - return query::result_view::do_with(*qr.query_result, [schema, cmd](query::result_view v) { - return to_key_slices(*schema, cmd->slice, v, std::numeric_limits::max()); - }); - }); - }); - }); - } - - static lw_shared_ptr make_paged_read_cmd(service::storage_proxy& proxy, const schema& s, uint32_t column_limit, const std::string* start_column, const dht::partition_range_vector& range) { - auto opts = query_opts(s); - std::vector clustering_ranges; - query::column_id_vector regular_columns; - uint64_t row_limit; - uint32_t partition_limit; - std::unique_ptr specific_ranges = nullptr; - // KeyRange::count is the number of thrift columns to return (unlike get_range_slices). - if (s.thrift().is_dynamic()) { - // For dynamic CFs we must limit the number of rows returned. We use the query::specific_ranges to constrain - // the first partition, of which we are only interested in the columns after start_column. - row_limit = static_cast(column_limit); - partition_limit = query::max_partitions; - if (start_column) { - auto sr = query::specific_ranges(*range[0].start()->value().key(), {make_clustering_range_and_validate(s, *start_column, std::string())}); - specific_ranges = std::make_unique(std::move(sr)); - } - regular_columns.emplace_back(s.regular_begin()->id); - } else { - // For static CFs we must limit the number of columns returned. Since we don't implement a cell limit, - // we ask for as many partitions as those that are capable of exhausting the limit and later filter out - // any excess cells. - row_limit = static_cast(std::numeric_limits::max()); - partition_limit = (column_limit + s.regular_columns_count() - 1) / s.regular_columns_count(); - schema::const_iterator start_col = start_column - ? s.regular_lower_bound(to_bytes(*start_column)) - : s.regular_begin(); - regular_columns = add_columns(start_col, s.regular_end(), false); - } - clustering_ranges.emplace_back(query::clustering_range::make_open_ended_both_sides()); - auto slice = query::partition_slice(std::move(clustering_ranges), { }, std::move(regular_columns), opts, - std::move(specific_ranges)); - auto cmd = make_lw_shared(s.id(), s.version(), std::move(slice), proxy.get_max_result_size(slice), - query::tombstone_limit(proxy.get_tombstone_limit()), query::row_limit(row_limit), query::partition_limit(partition_limit)); - cmd->allow_limit = db::allow_per_partition_rate_limit::yes; - return cmd; - } - - static future<> do_get_paged_slice( - sharded& proxy, - schema_ptr schema, - uint32_t column_limit, - dht::partition_range_vector range, - const std::string* start_column, - db::consistency_level consistency_level, - const ::timeout_config& timeout_config, - std::vector& output, - service::query_state& qs, - service_permit permit) { - auto cmd = make_paged_read_cmd(proxy.local(), *schema, column_limit, start_column, range); - std::optional start_key; - auto end = range[0].end(); - if (start_column && !schema->thrift().is_dynamic()) { - // For static CFs, we must first query for a specific key so as to consume the remainder - // of columns in that partition. - start_key = range[0].start()->value().key(); - range = {dht::partition_range::make_singular(std::move(range[0].start()->value()))}; - } - auto range1 = range; // query() below accepts an rvalue, so need a copy to reuse later - auto timeout = db::timeout_clock::now() + timeout_config.range_read_timeout; - return proxy.local().query(schema, cmd, std::move(range), consistency_level, {timeout, std::move(permit), qs.get_client_state()}).then( - [schema, cmd, column_limit](service::storage_proxy::coordinator_query_result qr) { - return query::result_view::do_with(*qr.query_result, [schema, cmd, column_limit](query::result_view v) { - return to_key_slices(*schema, cmd->slice, v, column_limit); - }); - }).then([&proxy, schema, cmd, column_limit, range = std::move(range1), consistency_level, start_key = std::move(start_key), end = std::move(end), &timeout_config, &output, &qs, permit = std::move(permit)](auto&& slices) mutable { - auto columns = std::accumulate(slices.begin(), slices.end(), 0u, [](auto&& acc, auto&& ks) { - return acc + ks.columns.size(); - }); - std::move(slices.begin(), slices.end(), std::back_inserter(output)); - if (columns == 0 || columns == column_limit || (slices.size() < cmd->partition_limit && columns < cmd->get_row_limit())) { - if (!output.empty() || !start_key) { - if (range.size() > 1 && columns < column_limit) { - range.erase(range.begin()); - return do_get_paged_slice(proxy, std::move(schema), column_limit - columns, std::move(range), nullptr, consistency_level, timeout_config, output, qs, std::move(permit)); - } - return make_ready_future(); - } - // The single, first partition we queried was empty, so retry with no start column. - } else { - start_key = key_from_thrift(*schema, to_bytes_view(output.back().key)); - } - auto start = dht::decorate_key(*schema, std::move(*start_key)); - range[0] = dht::partition_range(dht::partition_range::bound(std::move(start), false), std::move(end)); - return do_get_paged_slice(proxy, schema, column_limit - columns, std::move(range), nullptr, consistency_level, timeout_config, output, qs, std::move(permit)); - }); - } - - void get_paged_slice(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& column_family, const KeyRange& range, const std::string& start_column, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_family, [&](schema_ptr schema) { - return do_with(std::vector(), [&](auto& output) { - if (range.__isset.row_filter) { - throw make_exception("Cross-row paging is not supported along with index clauses"); - } - if (range.count <= 0) { - throw make_exception("Count must be positive"); - } - auto&& prange = make_partition_range(*schema, range); - if (!start_column.empty()) { - auto&& start_bound = prange[0].start(); - if (!(start_bound && start_bound->is_inclusive() && start_bound->value().has_key())) { - // According to Orign's DataRange#Paging#slicesForKey. - throw make_exception("If start column is provided, so must the start key"); - } - } - auto f = _query_state.get_client_state().has_schema_access(*schema, auth::permission::SELECT); - return f.then([this, schema, count = range.count, start_column, prange = std::move(prange), consistency_level, &output, permit = std::move(permit)] () mutable { - return do_get_paged_slice(_proxy, std::move(schema), count, std::move(prange), &start_column, - cl_from_thrift(consistency_level), _timeout_config, output, _query_state, std::move(permit)).then([&output] { - return std::move(output); - }); - }); - }); - }); - } - - void get_indexed_slices(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const ColumnParent& column_parent, const IndexClause& index_clause, const SlicePredicate& column_predicate, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - std::vector _return; - warn(unimplemented::cause::INDEXES); - // FIXME: implement - return pass_unimplemented(exn_cob); - } - - void insert(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const ColumnParent& column_parent, const Column& column, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_parent.column_family, [&](schema_ptr schema) { - if (column_parent.__isset.super_column) { - fail(unimplemented::cause::SUPER); - } - - if (schema->is_view()) { - throw make_exception("Cannot modify Materialized Views directly"); - } - - mutation m_to_apply(schema, key_from_thrift(*schema, to_bytes_view(key))); - add_to_mutation(*schema, column, m_to_apply); - return _query_state.get_client_state().has_schema_access(*schema, auth::permission::MODIFY).then([this, m_to_apply = std::move(m_to_apply), consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.write_timeout; - return _proxy.local().mutate({std::move(m_to_apply)}, cl_from_thrift(consistency_level), timeout, nullptr, std::move(permit), db::allow_per_partition_rate_limit::yes); - }); - }); - } - - void add(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const ColumnParent& column_parent, const CounterColumn& column, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_parent.column_family, [&](schema_ptr schema) { - if (column_parent.__isset.super_column) { - fail(unimplemented::cause::SUPER); - } - - mutation m_to_apply(schema, key_from_thrift(*schema, to_bytes_view(key))); - add_to_mutation(*schema, column, m_to_apply); - return _query_state.get_client_state().has_schema_access(*schema, auth::permission::MODIFY).then([this, m_to_apply = std::move(m_to_apply), consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.write_timeout; - return _proxy.local().mutate({std::move(m_to_apply)}, cl_from_thrift(consistency_level), timeout, nullptr, std::move(permit), db::allow_per_partition_rate_limit::yes); - }); - }); - } - - void cas(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const std::string& column_family, const std::vector & expected, const std::vector & updates, const ConsistencyLevel::type serial_consistency_level, const ConsistencyLevel::type commit_consistency_level) { - service_permit permit = obtain_permit(); - CASResult _return; - warn(unimplemented::cause::LWT); - // FIXME: implement - return pass_unimplemented(exn_cob); - } - - void remove(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const ColumnPath& column_path, const int64_t timestamp, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_path.column_family, [&](schema_ptr schema) { - if (schema->is_view()) { - throw make_exception("Cannot modify Materialized Views directly"); - } - - mutation m_to_apply(schema, key_from_thrift(*schema, to_bytes_view(key))); - - if (column_path.__isset.super_column) { - fail(unimplemented::cause::SUPER); - } else if (column_path.__isset.column) { - Deletion d; - d.__set_timestamp(timestamp); - d.__set_predicate(column_path_to_slice_predicate(column_path)); - Mutation m; - m.__set_deletion(d); - add_to_mutation(*schema, m, m_to_apply); - } else { - m_to_apply.partition().apply(tombstone(timestamp, gc_clock::now())); - } - - return _query_state.get_client_state().has_schema_access(*schema, auth::permission::MODIFY).then([this, m_to_apply = std::move(m_to_apply), consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.write_timeout; - return _proxy.local().mutate({std::move(m_to_apply)}, cl_from_thrift(consistency_level), timeout, nullptr, std::move(permit), db::allow_per_partition_rate_limit::yes); - }); - }); - } - - void remove_counter(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& key, const ColumnPath& column_path, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), column_path.column_family, [&](schema_ptr schema) { - mutation m_to_apply(schema, key_from_thrift(*schema, to_bytes_view(key))); - - auto timestamp = api::new_timestamp(); - if (column_path.__isset.super_column) { - fail(unimplemented::cause::SUPER); - } else if (column_path.__isset.column) { - Deletion d; - d.__set_timestamp(timestamp); - d.__set_predicate(column_path_to_slice_predicate(column_path)); - Mutation m; - m.__set_deletion(d); - add_to_mutation(*schema, m, m_to_apply); - } else { - m_to_apply.partition().apply(tombstone(timestamp, gc_clock::now())); - } - - return _query_state.get_client_state().has_schema_access(*schema, auth::permission::MODIFY).then([this, m_to_apply = std::move(m_to_apply), consistency_level, permit = std::move(permit)] () mutable { - // This mutation contains only counter tombstones so it can be applied like non-counter mutations. - auto timeout = db::timeout_clock::now() + _timeout_config.counter_write_timeout; - return _proxy.local().mutate({std::move(m_to_apply)}, cl_from_thrift(consistency_level), timeout, nullptr, std::move(permit), db::allow_per_partition_rate_limit::yes); - }); - }); - } - - void batch_mutate(thrift_fn::function cob, thrift_fn::function exn_cob, const std::map > > & mutation_map, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - auto p = prepare_mutations(_db, current_keyspace(), mutation_map); - return parallel_for_each(std::move(p.second), [this](auto&& schema) { - return _query_state.get_client_state().has_schema_access(*schema, auth::permission::MODIFY); - }).then([this, muts = std::move(p.first), consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.write_timeout; - return _proxy.local().mutate(std::move(muts), cl_from_thrift(consistency_level), timeout, nullptr, std::move(permit), db::allow_per_partition_rate_limit::yes); - }); - }); - } - - void atomic_batch_mutate(thrift_fn::function cob, thrift_fn::function exn_cob, const std::map > > & mutation_map, const ConsistencyLevel::type consistency_level) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - auto p = prepare_mutations(_db, current_keyspace(), mutation_map); - return parallel_for_each(std::move(p.second), [this](auto&& schema) { - return _query_state.get_client_state().has_schema_access(*schema, auth::permission::MODIFY); - }).then([this, muts = std::move(p.first), consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.write_timeout; - return _proxy.local().mutate_atomically(std::move(muts), cl_from_thrift(consistency_level), timeout, nullptr, std::move(permit)); - }); - }); - } - - void truncate(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& cfname) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - if (current_keyspace().empty()) { - throw make_exception("keyspace not set"); - } - - return _query_state.get_client_state().has_column_family_access(current_keyspace(), cfname, auth::permission::MODIFY).then([this, cfname] { - if (_db.find_schema(current_keyspace(), cfname)->is_view()) { - throw make_exception("Cannot truncate Materialized Views"); - } - return _proxy.local().truncate_blocking(current_keyspace(), cfname); - }); - }); - } - - void get_multi_slice(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const MultiSliceRequest& request) { - service_permit permit = obtain_permit(); - with_schema(std::move(cob), std::move(exn_cob), request.column_parent.column_family, [&](schema_ptr schema) { - if (!request.__isset.key) { - throw make_exception("Key may not be empty"); - } - if (!request.__isset.column_parent || request.column_parent.column_family.empty()) { - throw make_exception("non-empty table is required"); - } - if (!request.column_parent.super_column.empty()) { - throw make_exception("get_multi_slice does not support super columns"); - } - auto& s = *schema; - auto pk = key_from_thrift(s, to_bytes(request.key)); - auto dk = dht::decorate_key(s, pk); - query::column_id_vector regular_columns; - std::vector clustering_ranges; - auto opts = query_opts(s); - uint64_t row_limit; - if (s.thrift().is_dynamic()) { - row_limit = request.count; - auto cmp = bound_view::compare(s); - clustering_ranges = make_non_overlapping_ranges(std::move(request.column_slices), [&s](auto&& cslice) { - return make_clustering_range(s, cslice.start, cslice.finish); - }, clustering_key_prefix::prefix_equal_tri_compare(s), [cmp = std::move(cmp)](auto& range) { - auto bounds = bound_view::from_range(range); - return cmp(bounds.second, bounds.first); - }, request.reversed); - regular_columns.emplace_back(s.regular_begin()->id); - if (request.reversed) { - opts.set(query::partition_slice::option::reversed); - } - } else { - row_limit = static_cast(std::numeric_limits::max()); - clustering_ranges.emplace_back(query::clustering_range::make_open_ended_both_sides()); - auto cmp = [&s](auto&& s1, auto&& s2) { return s.regular_column_name_type()->compare(s1, s2); }; - auto ranges = make_non_overlapping_ranges(std::move(request.column_slices), [](auto&& cslice) { - return make_range(cslice.start, cslice.finish); - }, cmp, [&](auto& range) { return range.is_wrap_around(cmp); }, request.reversed); - auto on_range = [&](auto&& range) { - auto start = range.start() ? s.regular_lower_bound(range.start()->value()) : s.regular_begin(); - auto end = range.end() ? s.regular_upper_bound(range.end()->value()) : s.regular_end(); - regular_columns = add_columns(start, end, request.reversed); - }; - if (request.reversed) { - std::for_each(ranges.rbegin(), ranges.rend(), on_range); - } else { - std::for_each(ranges.begin(), ranges.end(), on_range); - } - } - auto slice = query::partition_slice(std::move(clustering_ranges), {}, std::move(regular_columns), opts, nullptr); - auto& proxy = _proxy.local(); - auto cmd = make_lw_shared(schema->id(), schema->version(), std::move(slice), proxy.get_max_result_size(slice), - query::tombstone_limit(proxy.get_tombstone_limit()), query::row_limit(row_limit)); - cmd->allow_limit = db::allow_per_partition_rate_limit::yes; - auto f = _query_state.get_client_state().has_schema_access(*schema, auth::permission::SELECT); - return f.then([this, &proxy, dk = std::move(dk), cmd, schema, column_limit = request.count, cl = request.consistency_level, permit = std::move(permit)] () mutable { - auto timeout = db::timeout_clock::now() + _timeout_config.read_timeout; - return proxy.query(schema, cmd, {dht::partition_range::make_singular(dk)}, cl_from_thrift(cl), {timeout, std::move(permit), _query_state.get_client_state()}).then( - [schema, cmd, column_limit](service::storage_proxy::coordinator_query_result qr) { - return query::result_view::do_with(*qr.query_result, [schema, cmd, column_limit](query::result_view v) { - column_aggregator aggregator(*schema, cmd->slice, column_limit, { }); - v.consume(cmd->slice, aggregator); - auto cols = aggregator.release(); - return !cols.empty() ? std::move(cols.begin()->second) : std::vector(); - }); - }); - }); - }); - } - - void describe_schema_versions(thrift_fn::function > const& _return)> cob, thrift_fn::function exn_cob) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this] { - return _ss.local().describe_schema_versions().then([](auto&& m) { - std::map> ret; - for (auto&& p : m) { - ret[p.first] = std::vector(p.second.begin(), p.second.end()); - } - return ret; - }); - }); - } - - void describe_keyspaces(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - validate_login(); - std::vector ret; - for (auto&& ks : _db.get_keyspaces()) { - ret.emplace_back(get_keyspace_definition(ks)); - } - return ret; - }); - } - - void describe_cluster_name(thrift_fn::function cob) { - service_permit permit = obtain_permit(); - cob(_db.get_config().cluster_name()); - } - - void describe_version(thrift_fn::function cob) { - service_permit permit = obtain_permit(); - cob(::cassandra::thrift_version); - } - - void do_describe_ring(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& keyspace, bool local) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] () -> future> { - auto ks = _db.find_keyspace(keyspace); - if (ks.get_replication_strategy().get_type() == locator::replication_strategy_type::local) { - throw make_exception("There is no ring for the keyspace: {}", keyspace); - } - - auto ring = co_await _ss.local().describe_ring(keyspace, local); - std::vector ret; - ret.reserve(ring.size()); - std::transform(ring.begin(), ring.end(), std::back_inserter(ret), [](auto&& tr) { - TokenRange token_range; - token_range.__set_start_token(std::move(tr._start_token)); - token_range.__set_end_token(std::move(tr._end_token)); - token_range.__set_endpoints(std::vector(tr._endpoints.begin(), tr._endpoints.end())); - std::vector eds; - std::transform(tr._endpoint_details.begin(), tr._endpoint_details.end(), std::back_inserter(eds), [](auto&& ed) { - EndpointDetails detail; - detail.__set_host(fmt::to_string(ed._host)); - detail.__set_datacenter(ed._datacenter); - detail.__set_rack(ed._rack); - return detail; - }); - token_range.__set_endpoint_details(std::move(eds)); - token_range.__set_rpc_endpoints(std::vector(tr._rpc_endpoints.begin(), tr._rpc_endpoints.end())); - return token_range; - }); - co_return ret; - }); - } - - void describe_ring(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& keyspace) { - do_describe_ring(std::move(cob), std::move(exn_cob), keyspace, false); - } - - void describe_local_ring(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& keyspace) { - do_describe_ring(std::move(cob), std::move(exn_cob), keyspace, true); - } - - void describe_token_map(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this] { - auto m = _ss.local().get_token_to_endpoint_map(); - std::map ret; - for (auto&& p : m) { - ret[format("{}", p.first)] = fmt::to_string(p.second); - } - return ret; - }); - } - - void describe_partitioner(thrift_fn::function cob) { - service_permit permit = obtain_permit(); - cob(_db.get_config().partitioner()); - } - - void describe_snitch(thrift_fn::function cob) { - service_permit permit = obtain_permit(); - cob(format("org.apache.cassandra.locator.{}", _db.real_database().get_snitch_name())); - } - - void describe_keyspace(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& keyspace) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&] { - validate_login(); - auto ks = _db.find_keyspace(keyspace); - return get_keyspace_definition(ks); - }); - } - - void describe_splits(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& cfName, const std::string& start_token, const std::string& end_token, const int32_t keys_per_split) { - service_permit permit = obtain_permit(); - return describe_splits_ex([cob = std::move(cob)](auto&& results) { - std::vector res; - res.reserve(results.size() + 1); - res.emplace_back(results[0].start_token); - for (auto&& s : results) { - res.emplace_back(std::move(s.end_token)); - } - return cob(std::move(res)); - }, exn_cob, cfName, start_token, end_token, keys_per_split); - } - - void trace_next_query(thrift_fn::function cob) { - service_permit permit = obtain_permit(); - std::string _return; - // FIXME: implement - return cob("dummy trace"); - } - - void describe_splits_ex(thrift_fn::function const& _return)> cob, thrift_fn::function exn_cob, const std::string& cfName, const std::string& start_token, const std::string& end_token, const int32_t keys_per_split) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [&]{ - dht::token_range_vector ranges; - auto tstart = start_token.empty() ? dht::minimum_token() : dht::token::from_sstring(sstring(start_token)); - auto tend = end_token.empty() ? dht::maximum_token() : dht::token::from_sstring(sstring(end_token)); - wrapping_interval r({{ std::move(tstart), false }}, {{ std::move(tend), true }}); - auto cf = sstring(cfName); - auto splits = _ss.local().get_splits(current_keyspace(), cf, std::move(r), keys_per_split); - - std::vector res; - for (auto&& s : splits) { - res.emplace_back(); - assert(s.first.start() && s.first.end()); - auto start_token = s.first.start()->value().to_sstring(); - auto end_token = s.first.end()->value().to_sstring(); - res.back().__set_start_token(bytes_to_string(to_bytes_view(start_token))); - res.back().__set_end_token(bytes_to_string(to_bytes_view(end_token))); - res.back().__set_row_count(s.second); - } - return res; - }); - } - - future execute_schema_command(std::function>(data_dictionary::database, api::timestamp_type)> ddl, std::string_view description) { - return _query_processor.invoke_on(0, [ddl = std::move(ddl), description = std::move(description)] (cql3::query_processor& qp) mutable { - return qp.execute_thrift_schema_command(std::move(ddl), std::move(description)); - }); - } - - void system_add_column_family(thrift_fn::function cob, thrift_fn::function exn_cob, const CfDef& cf_def) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this, def = cf_def] () -> future { - auto& t = *this; - auto cf_def = def; - - co_await t._query_state.get_client_state().has_keyspace_access(cf_def.keyspace, auth::permission::CREATE); - - co_return co_await t.execute_schema_command([&p = t._proxy.local(), &cf_def] (data_dictionary::database db, api::timestamp_type ts) -> future> { - if (!db.has_keyspace(cf_def.keyspace)) { - throw NotFoundException(); - } - if (db.has_schema(cf_def.keyspace, cf_def.name)) { - throw make_exception("Column family {} already exists", cf_def.name); - } - - auto s = schema_from_thrift(cf_def, cf_def.keyspace); - co_return co_await service::prepare_new_column_family_announcement(p, std::move(s), ts); - }, format("thrift: create column family {}", cf_def.name)); - }); - } - void system_drop_column_family(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& column_family) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this, cfm = column_family] () -> future { - auto& t = *this; - auto column_family = cfm; - co_await t._query_state.get_client_state().has_column_family_access(t.current_keyspace(), column_family, auth::permission::DROP); - - co_return co_await t.execute_schema_command( - [&p = t._proxy.local(), &column_family, ¤t_keyspace = t.current_keyspace()] (data_dictionary::database db, api::timestamp_type ts) -> future> { - auto cf = db.find_table(current_keyspace, column_family); - if (cf.schema()->is_view()) { - throw make_exception("Cannot drop Materialized Views from Thrift"); - } - if (!cf.views().empty()) { - throw make_exception("Cannot drop table with Materialized Views {}", column_family); - } - - co_return co_await service::prepare_column_family_drop_announcement(p, current_keyspace, column_family, ts); - }, format("thrift: drop column family {}", column_family)); - }); - } - - void system_add_keyspace(thrift_fn::function cob, thrift_fn::function exn_cob, const KsDef& ks_def) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this, def = ks_def] () -> future { - auto& t = *this; - auto ks_def = def; - - co_await t._query_state.get_client_state().has_all_keyspaces_access(auth::permission::CREATE); - - co_return co_await t.execute_schema_command([&ks_def] (data_dictionary::database db, api::timestamp_type ts) -> future> { - co_return service::prepare_new_keyspace_announcement(db.real_database(), keyspace_from_thrift(ks_def), ts); - }, format("thrift: add {} keyspace", ks_def.name)); - }); - } - - void system_drop_keyspace(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& keyspace) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this, ks = keyspace] () -> future { - auto& t = *this; - auto keyspace = ks; - - co_await t._query_state.get_client_state().has_keyspace_access(keyspace, auth::permission::DROP); - - co_return co_await t.execute_schema_command([&keyspace] (data_dictionary::database db, api::timestamp_type ts) -> future> { - thrift_validation::validate_keyspace_not_system(keyspace); - if (!db.has_keyspace(keyspace)) { - throw NotFoundException(); - } - - co_return co_await service::prepare_keyspace_drop_announcement(db.real_database(), keyspace, ts); - }, format("thrift: drop {} keyspace", keyspace)); - }); - } - - void system_update_keyspace(thrift_fn::function cob, thrift_fn::function exn_cob, const KsDef& ks_def) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this, def = ks_def] () -> future { - auto& t = *this; - auto ks_def = def; - thrift_validation::validate_keyspace_not_system(ks_def.name); - - co_await t._query_state.get_client_state().has_keyspace_access(ks_def.name, auth::permission::ALTER); - - co_return co_await t.execute_schema_command([&ks_def] (data_dictionary::database db, api::timestamp_type ts) -> future> { - if (!db.has_keyspace(ks_def.name)) { - throw NotFoundException(); - } - if (!ks_def.cf_defs.empty()) { - throw make_exception("Keyspace update must not contain any column family definitions."); - } - - auto ksm = keyspace_from_thrift(ks_def); - co_return service::prepare_keyspace_update_announcement(db.real_database(), std::move(ksm), ts); - }, format("thrift: update {} keyspace", ks_def.name)); - }); - } - - void system_update_column_family(thrift_fn::function cob, thrift_fn::function exn_cob, const CfDef& cf_def) { - service_permit permit = obtain_permit(); - with_cob(std::move(cob), std::move(exn_cob), [this, def = cf_def] () -> future { - auto& t = *this; - auto cf_def = def; - - co_await t._query_state.get_client_state().has_schema_access(cf_def.keyspace, cf_def.name, auth::permission::ALTER); - - co_return co_await t.execute_schema_command([&p = t._proxy.local(), &cf_def] (data_dictionary::database db, api::timestamp_type ts) -> future> { - auto cf = db.find_table(cf_def.keyspace, cf_def.name); - auto schema = cf.schema(); - - if (schema->is_cql3_table()) { - throw make_exception("Cannot modify CQL3 table {} as it may break the schema. You should use cqlsh to modify CQL3 tables instead.", cf_def.name); - } - - if (schema->is_view()) { - throw make_exception("Cannot modify Materialized View table {} as it may break the schema. " - "You should use cqlsh to modify Materialized View tables instead.", cf_def.name); - } - - if (!cf.views().empty()) { - throw make_exception("Cannot modify table with Materialized Views {} as it may break the schema. " - "You should use cqlsh to modify Materialized View tables instead.", cf_def.name); - } - - auto s = schema_from_thrift(cf_def, cf_def.keyspace, schema->id()); - if (schema->thrift().is_dynamic() != s->thrift().is_dynamic()) { - fail(unimplemented::cause::MIXED_CF); - } - co_return co_await service::prepare_column_family_update_announcement(p, std::move(s), true, std::vector(), ts); - }, format("thrift: update column family {}", cf_def.name)); - }); - } - - void execute_cql_query(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& query, const Compression::type compression) { - throw make_exception("CQL2 is not supported"); - } - - class cql3_result_visitor final : public cql_transport::messages::result_message::visitor { - CqlResult _result; - public: - const CqlResult& result() const { - return _result; - } - virtual void visit(const cql_transport::messages::result_message::void_message&) override { - _result.__set_type(CqlResultType::VOID); - } - virtual void visit(const cql_transport::messages::result_message::set_keyspace& m) override { - _result.__set_type(CqlResultType::VOID); - } - virtual void visit(const cql_transport::messages::result_message::prepared::cql& m) override { - throw make_exception("Cannot convert prepared query result to CqlResult"); - } - virtual void visit(const cql_transport::messages::result_message::prepared::thrift& m) override { - throw make_exception("Cannot convert prepared query result to CqlResult"); - } - virtual void visit(const cql_transport::messages::result_message::schema_change& m) override { - _result.__set_type(CqlResultType::VOID); - } - virtual void visit(const cql_transport::messages::result_message::rows& m) override { - _result = to_thrift_result(m.rs()); - } - virtual void visit(const cql_transport::messages::result_message::bounce_to_shard& m) override { - throw TProtocolException(TProtocolException::TProtocolExceptionType::NOT_IMPLEMENTED, "Thrift does not support executing LWT statements"); - } - virtual void visit(const cql_transport::messages::result_message::exception& m) override { - m.throw_me(); - } - }; - - void execute_cql3_query(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& query, const Compression::type compression, const ConsistencyLevel::type consistency) { - with_exn_cob(std::move(exn_cob), [&] { - if (compression != Compression::type::NONE) { - throw make_exception("Compressed query strings are not supported"); - } - auto& qp = _query_processor.local(); - auto opts = std::make_unique(qp.get_cql_config(), cl_from_thrift(consistency), std::nullopt, std::vector(), - false, cql3::query_options::specific_options::DEFAULT); - auto f = qp.execute_direct(query, _query_state, *opts); - return f.then([cob = std::move(cob), opts = std::move(opts)](auto&& ret) { - cql3_result_visitor visitor; - ret->accept(visitor); - return cob(visitor.result()); - }); - }); - } - - void prepare_cql_query(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& query, const Compression::type compression) { - throw make_exception("CQL2 is not supported"); - } - - class prepared_result_visitor final : public cql_transport::messages::result_message::visitor_base { - CqlPreparedResult _result; - public: - const CqlPreparedResult& result() const { - return _result; - } - virtual void visit(const cql_transport::messages::result_message::prepared::cql& m) override { - throw std::runtime_error("Unexpected result message type."); - } - virtual void visit(const cql_transport::messages::result_message::prepared::thrift& m) override { - _result.__set_itemId(m.get_id()); - auto& names = m.metadata().names(); - _result.__set_count(names.size()); - std::vector variable_types; - std::vector variable_names; - for (auto csp : names) { - variable_types.emplace_back(csp->type->name()); - variable_names.emplace_back(csp->name->to_string()); - } - _result.__set_variable_types(std::move(variable_types)); - _result.__set_variable_names(std::move(variable_names)); - } - }; - - void prepare_cql3_query(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& query, const Compression::type compression) { - with_exn_cob(std::move(exn_cob), [&] { - validate_login(); - if (compression != Compression::type::NONE) { - throw make_exception("Compressed query strings are not supported"); - } - return _query_processor.local().prepare(query, _query_state).then([cob = std::move(cob)](auto&& stmt) { - prepared_result_visitor visitor; - stmt->accept(visitor); - cob(visitor.result()); - }); - }); - } - - void execute_prepared_cql_query(thrift_fn::function cob, thrift_fn::function exn_cob, const int32_t itemId, const std::vector & values) { - throw make_exception("CQL2 is not supported"); - } - - void execute_prepared_cql3_query(thrift_fn::function cob, thrift_fn::function exn_cob, const int32_t itemId, const std::vector & values, const ConsistencyLevel::type consistency) { - with_exn_cob(std::move(exn_cob), [&] { - cql3::prepared_cache_key_type cache_key(itemId); - bool needs_authorization = false; - - auto prepared = _query_processor.local().get_prepared(_query_state.get_client_state().user(), cache_key); - if (!prepared) { - needs_authorization = true; - - prepared = _query_processor.local().get_prepared(cache_key); - if (!prepared) { - throw make_exception("Prepared query with id {} not found", itemId); - } - } - auto stmt = prepared->statement; - if (stmt->get_bound_terms() != values.size()) { - throw make_exception("Wrong number of values specified. Expected {}, got {}.", stmt->get_bound_terms(), values.size()); - } - std::vector bytes_values; - std::transform(values.begin(), values.end(), std::back_inserter(bytes_values), [](auto&& s) { - return cql3::raw_value::make_value(to_bytes(s)); - }); - auto& qp = _query_processor.local(); - auto opts = std::make_unique(qp.get_cql_config(), cl_from_thrift(consistency), std::nullopt, std::move(bytes_values), - false, cql3::query_options::specific_options::DEFAULT); - auto f = qp.execute_prepared(std::move(prepared), std::move(cache_key), _query_state, *opts, needs_authorization); - return f.then([cob = std::move(cob), opts = std::move(opts)](auto&& ret) { - cql3_result_visitor visitor; - ret->accept(visitor); - return cob(visitor.result()); - }); - }); - } - - void set_cql_version(thrift_fn::function cob, thrift_fn::function exn_cob, const std::string& version) { - // No-op. - cob(); - } - -private: - template - static sstring class_from_compound_type(const compound_type& ct) { - if (ct.is_singular()) { - return ct.types().front()->name(); - } - sstring type = "org.apache.cassandra.db.marshal.CompositeType("; - for (auto& dt : ct.types()) { - type += dt->name(); - if (&dt != &*ct.types().rbegin()) { - type += ","; - } - } - type += ")"; - return type; - } - static std::pair, bool> get_types(const std::string& thrift_type) { - static const char composite_type[] = "CompositeType"; - std::vector ret; - auto t = sstring_view(thrift_type); - auto composite_idx = t.find(composite_type); - bool is_compound = false; - if (composite_idx == sstring_view::npos) { - ret.emplace_back(db::marshal::type_parser::parse(t)); - } else { - t.remove_prefix(composite_idx + sizeof(composite_type) - 1); - auto types = db::marshal::type_parser(t).get_type_parameters(false); - std::move(types.begin(), types.end(), std::back_inserter(ret)); - is_compound = true; - } - return std::make_pair(std::move(ret), is_compound); - } - static CqlResult to_thrift_result(const cql3::result& rs) { - CqlResult result; - result.__set_type(CqlResultType::ROWS); - - constexpr static const char* utf8 = "UTF8Type"; - - CqlMetadata mtd; - std::map name_types; - std::map value_types; - for (auto&& c : rs.get_metadata().get_names()) { - auto&& name = c->name->to_string(); - name_types.emplace(name, utf8); - value_types.emplace(name, c->type->name()); - } - mtd.__set_name_types(name_types); - mtd.__set_value_types(value_types); - mtd.__set_default_name_type(utf8); - mtd.__set_default_value_type(utf8); - result.__set_schema(mtd); - - struct visitor { - std::vector _rows; - const cql3::metadata& _metadata; - std::vector _columns; - column_id _column_id; - - void start_row() { - _column_id = 0; - _columns.reserve(_metadata.column_count()); - } - void accept_value(managed_bytes_view_opt cell) { - auto& col = _metadata.get_names()[_column_id++]; - - Column& c = _columns.emplace_back(); - c.__set_name(col->name->to_string()); - if (cell) { - c.__set_value(bytes_to_string(*cell)); - } - - } - void end_row() { - CqlRow& r = _rows.emplace_back(); - r.__set_key(std::string()); - r.__set_columns(std::move(_columns)); - _columns = { }; - } - }; - - visitor v { {}, rs.get_metadata(), {}, {} }; - rs.visit(v); - result.__set_rows(std::move(v._rows)); - return result; - } - static KsDef get_keyspace_definition(const data_dictionary::keyspace& ks) { - auto make_options = [](auto&& m) { - return std::map(m.begin(), m.end()); - }; - auto&& meta = ks.metadata(); - KsDef def; - def.__set_name(meta->name()); - def.__set_strategy_class(meta->strategy_name()); - def.__set_strategy_options(make_options(meta->strategy_options())); - std::vector cfs; - for (auto&& s : meta->tables()) { - if (s->is_cql3_table()) { - continue; - } - CfDef cf_def; - cf_def.__set_keyspace(s->ks_name()); - cf_def.__set_name(s->cf_name()); - cf_def.__set_column_type(cf_type_to_sstring(s->type())); - cf_def.__set_comparator_type(cell_comparator::to_sstring(*s)); - cf_def.__set_comment(s->comment()); - std::vector columns; - if (!s->thrift().is_dynamic()) { - for (auto&& c : s->regular_columns()) { - ColumnDef c_def; - c_def.__set_name(c.name_as_text()); - c_def.__set_validation_class(c.type->name()); - columns.emplace_back(std::move(c_def)); - } - } - cf_def.__set_column_metadata(columns); - cf_def.__set_gc_grace_seconds(s->gc_grace_seconds().count()); - cf_def.__set_default_validation_class(s->make_legacy_default_validator()->name()); - cf_def.__set_min_compaction_threshold(s->min_compaction_threshold()); - cf_def.__set_max_compaction_threshold(s->max_compaction_threshold()); - cf_def.__set_key_validation_class(class_from_compound_type(*s->partition_key_type())); - cf_def.__set_key_alias(s->partition_key_columns().begin()->name_as_text()); - cf_def.__set_compaction_strategy(sstables::compaction_strategy::name(s->compaction_strategy())); - cf_def.__set_compaction_strategy_options(make_options(s->compaction_strategy_options())); - cf_def.__set_compression_options(make_options(s->get_compressor_params().get_options())); - cf_def.__set_bloom_filter_fp_chance(s->bloom_filter_fp_chance()); - cf_def.__set_caching("all"); - cf_def.__set_memtable_flush_period_in_ms(s->memtable_flush_period()); - cf_def.__set_default_time_to_live(s->default_time_to_live().count()); - cf_def.__set_speculative_retry(s->speculative_retry().to_sstring()); - cfs.emplace_back(std::move(cf_def)); - } - def.__set_cf_defs(cfs); - def.__set_durable_writes(meta->durable_writes()); - return def; - } - static std::optional index_metadata_from_thrift(const ColumnDef& def) { - std::optional idx_name; - std::optional> idx_opts; - std::optional idx_type; - if (def.__isset.index_type) { - idx_type = [&def]() -> std::optional { - switch (def.index_type) { - case IndexType::type::KEYS: return index_metadata_kind::keys; - case IndexType::type::COMPOSITES: return index_metadata_kind::composites; - case IndexType::type::CUSTOM: return index_metadata_kind::custom; - default: return {}; - }; - }(); - } - if (def.__isset.index_name) { - idx_name = to_sstring(def.index_name); - } - if (def.__isset.index_options) { - idx_opts = std::unordered_map(def.index_options.begin(), def.index_options.end()); - } - if (idx_name && idx_opts && idx_type) { - return index_metadata(idx_name.value(), idx_opts.value(), idx_type.value(), index_metadata::is_local_index::no); - } - return {}; - } - static schema_ptr schema_from_thrift(const CfDef& cf_def, const sstring ks_name, std::optional id = { }) { - thrift_validation::validate_cf_def(cf_def); - schema_builder builder(ks_name, cf_def.name, id); - schema_builder::default_names names(builder); - - if (cf_def.__isset.key_validation_class) { - auto pk_types = std::move(get_types(cf_def.key_validation_class).first); - if (pk_types.size() == 1 && cf_def.__isset.key_alias) { - builder.with_column(to_bytes(cf_def.key_alias), std::move(pk_types.back()), column_kind::partition_key); - } else { - for (uint32_t i = 0; i < pk_types.size(); ++i) { - builder.with_column(to_bytes(names.partition_key_name()), std::move(pk_types[i]), column_kind::partition_key); - } - } - } else { - builder.with_column(to_bytes(names.partition_key_name()), bytes_type, column_kind::partition_key); - } - - auto default_validator = cf_def.__isset.default_validation_class - ? db::marshal::type_parser::parse(to_sstring(cf_def.default_validation_class)) - : bytes_type; - - if (cf_def.column_metadata.empty()) { - // Dynamic CF - builder.set_is_dense(true); - auto p = get_types(cf_def.comparator_type); - auto ck_types = std::move(p.first); - builder.set_is_compound(p.second); - for (uint32_t i = 0; i < ck_types.size(); ++i) { - builder.with_column(to_bytes(names.clustering_name()), std::move(ck_types[i]), column_kind::clustering_key); - } - builder.with_column(to_bytes(names.compact_value_name()), default_validator); - } else { - // Static CF - builder.set_is_compound(false); - auto column_name_type = db::marshal::type_parser::parse(to_sstring(cf_def.comparator_type)); - for (const ColumnDef& col_def : cf_def.column_metadata) { - auto col_name = to_bytes(col_def.name); - column_name_type->validate(col_name); - builder.with_column(std::move(col_name), db::marshal::type_parser::parse(to_sstring(col_def.validation_class)), - column_kind::regular_column); - auto index = index_metadata_from_thrift(col_def); - if (index) { - builder.with_index(index.value()); - } - } - builder.set_regular_column_name_type(column_name_type); - } - builder.set_default_validation_class(default_validator); - if (cf_def.__isset.comment) { - builder.set_comment(cf_def.comment); - } - if (cf_def.__isset.gc_grace_seconds) { - builder.set_gc_grace_seconds(cf_def.gc_grace_seconds); - } - if (cf_def.__isset.min_compaction_threshold) { - builder.set_min_compaction_threshold(cf_def.min_compaction_threshold); - } - if (cf_def.__isset.max_compaction_threshold) { - builder.set_max_compaction_threshold(cf_def.max_compaction_threshold); - } - if (cf_def.__isset.compaction_strategy) { - builder.set_compaction_strategy(sstables::compaction_strategy::type(cf_def.compaction_strategy)); - } - auto make_options = [](const std::map& m) { - return std::map{m.begin(), m.end()}; - }; - if (cf_def.__isset.compaction_strategy_options) { - builder.set_compaction_strategy_options(make_options(cf_def.compaction_strategy_options)); - } - if (cf_def.__isset.compression_options) { - builder.set_compressor_params(compression_parameters(make_options(cf_def.compression_options))); - } - if (cf_def.__isset.bloom_filter_fp_chance) { - builder.set_bloom_filter_fp_chance(cf_def.bloom_filter_fp_chance); - } - if (cf_def.__isset.memtable_flush_period_in_ms) { - builder.set_memtable_flush_period(cf_def.memtable_flush_period_in_ms); - } - if (cf_def.__isset.default_time_to_live) { - builder.set_default_time_to_live(gc_clock::duration(cf_def.default_time_to_live)); - } - if (cf_def.__isset.speculative_retry) { - builder.set_speculative_retry(cf_def.speculative_retry); - } - if (cf_def.__isset.min_index_interval) { - builder.set_min_index_interval(cf_def.min_index_interval); - } - if (cf_def.__isset.max_index_interval) { - builder.set_max_index_interval(cf_def.max_index_interval); - } - return builder.build(); - } - static lw_shared_ptr keyspace_from_thrift(const KsDef& ks_def) { - thrift_validation::validate_ks_def(ks_def); - std::vector cf_defs; - cf_defs.reserve(ks_def.cf_defs.size()); - for (const CfDef& cf_def : ks_def.cf_defs) { - if (cf_def.keyspace != ks_def.name) { - throw make_exception("CfDef ({}) had a keyspace definition that did not match KsDef", cf_def.keyspace); - } - cf_defs.emplace_back(schema_from_thrift(cf_def, ks_def.name)); - } - return make_lw_shared( - ks_def.name, - ks_def.strategy_class, - std::map{ks_def.strategy_options.begin(), ks_def.strategy_options.end()}, - std::nullopt, - ks_def.durable_writes, - std::move(cf_defs)); - } - static schema_ptr lookup_schema(data_dictionary::database db, const sstring& ks_name, const sstring& cf_name) { - if (ks_name.empty()) { - throw make_exception("keyspace not set"); - } - return db.find_schema(ks_name, cf_name); - } - static partition_key key_from_thrift(const schema& s, bytes_view k) { - thrift_validation::validate_key(s, k); - if (s.partition_key_size() == 1) { - return partition_key::from_single_value(s, to_bytes(k)); - } - auto composite = composite_view(k); - return partition_key::from_exploded(composite.values()); - } - static db::consistency_level cl_from_thrift(const ConsistencyLevel::type consistency_level) { - switch (consistency_level) { - case ConsistencyLevel::type::ONE: return db::consistency_level::ONE; - case ConsistencyLevel::type::QUORUM: return db::consistency_level::QUORUM; - case ConsistencyLevel::type::LOCAL_QUORUM: return db::consistency_level::LOCAL_QUORUM; - case ConsistencyLevel::type::EACH_QUORUM: return db::consistency_level::EACH_QUORUM; - case ConsistencyLevel::type::ALL: return db::consistency_level::ALL; - case ConsistencyLevel::type::ANY: return db::consistency_level::ANY; - case ConsistencyLevel::type::TWO: return db::consistency_level::TWO; - case ConsistencyLevel::type::THREE: return db::consistency_level::THREE; - case ConsistencyLevel::type::SERIAL: return db::consistency_level::SERIAL; - case ConsistencyLevel::type::LOCAL_SERIAL: return db::consistency_level::LOCAL_SERIAL; - case ConsistencyLevel::type::LOCAL_ONE: return db::consistency_level::LOCAL_ONE; - default: throw make_exception("undefined consistency_level {}", consistency_level); - } - } - static ttl_opt maybe_ttl(const schema& s, const Column& col) { - if (col.__isset.ttl) { - auto ttl = std::chrono::duration_cast(std::chrono::seconds(col.ttl)); - if (ttl.count() <= 0) { - throw make_exception("ttl must be positive"); - } - if (ttl > max_ttl) { - throw make_exception("ttl is too large"); - } - return {ttl}; - } else if (s.default_time_to_live().count() > 0) { - return {s.default_time_to_live()}; - } else { - return { }; - } - } - static void validate_key(const schema& s, const clustering_key& ck, bytes_view v) { - auto ck_size = ck.size(s); - if (ck_size > s.clustering_key_size()) { - throw std::runtime_error(format("Cell name of {}.{} has too many components, expected {} but got {} in 0x{}", - s.ks_name(), s.cf_name(), s.clustering_key_size(), ck_size, to_hex(v))); - } - } - static clustering_key_prefix make_clustering_prefix(const schema& s, bytes_view v) { - auto composite = composite_view(v, s.thrift().has_compound_comparator()); - auto ck = clustering_key_prefix::from_exploded(composite.values()); - validate_key(s, ck, v); - return ck; - } - static query::clustering_range::bound make_clustering_bound(const schema& s, bytes_view v, composite::eoc exclusiveness_marker) { - auto composite = composite_view(v, s.thrift().has_compound_comparator()); - auto last = composite::eoc::none; - auto&& ck = clustering_key_prefix::from_exploded(composite.components() | boost::adaptors::transformed([&](auto&& c) { - last = c.second; - return c.first; - })); - validate_key(s, ck, v); - return query::clustering_range::bound(std::move(ck), last != exclusiveness_marker); - } - static wrapping_interval make_clustering_range(const schema& s, const std::string& start, const std::string& end) { - using bound = wrapping_interval::bound; - std::optional start_bound; - if (!start.empty()) { - start_bound = make_clustering_bound(s, to_bytes_view(start), composite::eoc::end); - } - std::optional end_bound; - if (!end.empty()) { - end_bound = make_clustering_bound(s, to_bytes_view(end), composite::eoc::start); - } - return { std::move(start_bound), std::move(end_bound) }; - } - static query::clustering_range make_clustering_range_and_validate(const schema& s, const std::string& start, const std::string& end) { - auto range = make_clustering_range(s, start, end); - auto bounds = bound_view::from_range(range); - if (bound_view::compare(s)(bounds.second, bounds.first)) { - throw make_exception("Range finish must come after start in the order of traversal"); - } - return query::clustering_range(std::move(range)); - } - static wrapping_interval make_range(const std::string& start, const std::string& end) { - using bound = wrapping_interval::bound; - std::optional start_bound; - if (!start.empty()) { - start_bound = bound(to_bytes(start)); - } - std::optional end_bound; - if (!end.empty()) { - end_bound = bound(to_bytes(end)); - } - return { std::move(start_bound), std::move(end_bound) }; - } - static std::pair make_column_range(const schema& s, const std::string& start, const std::string& end) { - auto start_it = start.empty() ? s.regular_begin() : s.regular_lower_bound(to_bytes(start)); - auto end_it = end.empty() ? s.regular_end() : s.regular_upper_bound(to_bytes(end)); - if (start_it > end_it) { - throw make_exception("Range finish must come after start in the order of traversal"); - } - return std::make_pair(std::move(start_it), std::move(end_it)); - } - // Adds the column_ids from the specified range of column_definitions to the out vector, - // according to the order defined by reversed. - template - static query::column_id_vector add_columns(Iterator beg, Iterator end, bool reversed) { - auto range = boost::make_iterator_range(std::move(beg), std::move(end)) - | boost::adaptors::filtered(std::mem_fn(&column_definition::is_atomic)) - | boost::adaptors::transformed(std::mem_fn(&column_definition::id)); - return reversed ? boost::copy_range(range | boost::adaptors::reversed) - : boost::copy_range(range); - } - static query::partition_slice::option_set query_opts(const schema& s) { - query::partition_slice::option_set opts; - if (!s.is_counter()) { - opts.set(query::partition_slice::option::send_timestamp); - opts.set(query::partition_slice::option::send_ttl); - } - if (s.thrift().is_dynamic()) { - opts.set(query::partition_slice::option::send_clustering_key); - } - opts.set(query::partition_slice::option::send_partition_key); - return opts; - } - static void sort_ranges(const schema& s, std::vector& ranges) { - position_in_partition::less_compare less(s); - std::sort(ranges.begin(), ranges.end(), - [&less] (const query::clustering_range& r1, const query::clustering_range& r2) { - return less( - position_in_partition_view::for_range_start(r1), - position_in_partition_view::for_range_start(r2)); - }); - } - static lw_shared_ptr slice_pred_to_read_cmd(service::storage_proxy& proxy, const schema& s, const SlicePredicate& predicate) { - auto opts = query_opts(s); - std::vector clustering_ranges; - query::column_id_vector regular_columns; - uint64_t per_partition_row_limit = static_cast(std::numeric_limits::max()); - if (predicate.__isset.column_names) { - thrift_validation::validate_column_names(predicate.column_names); - auto unique_column_names = boost::copy_range>(predicate.column_names | boost::adaptors::uniqued); - if (s.thrift().is_dynamic()) { - for (auto&& name : unique_column_names) { - auto ckey = make_clustering_prefix(s, to_bytes(name)); - clustering_ranges.emplace_back(query::clustering_range::make_singular(std::move(ckey))); - } - sort_ranges(s, clustering_ranges); - regular_columns.emplace_back(s.regular_begin()->id); - } else { - clustering_ranges.emplace_back(query::clustering_range::make_open_ended_both_sides()); - auto&& defs = unique_column_names - | boost::adaptors::transformed([&s](auto&& name) { return s.get_column_definition(to_bytes(name)); }) - | boost::adaptors::filtered([](auto* def) { return def; }) - | boost::adaptors::indirected; - regular_columns = add_columns(defs.begin(), defs.end(), false); - } - } else if (predicate.__isset.slice_range) { - auto range = predicate.slice_range; - if (range.count < 0) { - throw make_exception("SliceRange requires non-negative count"); - } - if (range.reversed) { - std::swap(range.start, range.finish); - opts.set(query::partition_slice::option::reversed); - } - per_partition_row_limit = static_cast(range.count); - if (s.thrift().is_dynamic()) { - clustering_ranges.emplace_back(make_clustering_range_and_validate(s, range.start, range.finish)); - regular_columns.emplace_back(s.regular_begin()->id); - } else { - clustering_ranges.emplace_back(query::clustering_range::make_open_ended_both_sides()); - auto r = make_column_range(s, range.start, range.finish); - // For static CFs, the limit is enforced on the result as we do not implement - // a cell limit in the database engine. - regular_columns = add_columns(r.first, r.second, range.reversed); - } - } else { - throw make_exception("SlicePredicate column_names and slice_range may not both be null"); - } - auto slice = query::partition_slice(std::move(clustering_ranges), {}, std::move(regular_columns), opts, - nullptr, per_partition_row_limit); - auto cmd = make_lw_shared(s.id(), s.version(), std::move(slice), proxy.get_max_result_size(slice), - query::tombstone_limit(proxy.get_tombstone_limit())); - cmd->allow_limit = db::allow_per_partition_rate_limit::yes; - return cmd; - } - static ColumnParent column_path_to_column_parent(const ColumnPath& column_path) { - ColumnParent ret; - ret.__set_column_family(column_path.column_family); - if (column_path.__isset.super_column) { - ret.__set_super_column(column_path.super_column); - } - return ret; - } - static SlicePredicate column_path_to_slice_predicate(const ColumnPath& column_path) { - SlicePredicate ret; - if (column_path.__isset.column) { - ret.__set_column_names({column_path.column}); - } - return ret; - } - static dht::partition_range_vector make_partition_ranges(const schema& s, const std::vector& keys) { - dht::partition_range_vector ranges; - for (auto&& key : keys) { - auto pk = key_from_thrift(s, to_bytes_view(key)); - auto dk = dht::decorate_key(s, pk); - ranges.emplace_back(dht::partition_range::make_singular(std::move(dk))); - } - return ranges; - } - static Column make_column(const bytes& col, const query::result_atomic_cell_view& cell) { - Column ret; - ret.__set_name(bytes_to_string(col)); - ret.__set_value(bytes_to_string(cell.value())); - ret.__set_timestamp(cell.timestamp()); - if (cell.ttl()) { - ret.__set_ttl(cell.ttl()->count()); - } - return ret; - } - static ColumnOrSuperColumn column_to_column_or_supercolumn(Column&& col) { - ColumnOrSuperColumn ret; - ret.__set_column(std::move(col)); - return ret; - } - static ColumnOrSuperColumn make_column_or_supercolumn(const bytes& col, const query::result_atomic_cell_view& cell) { - return column_to_column_or_supercolumn(make_column(col, cell)); - } - static CounterColumn make_counter_column(const bytes& col, const query::result_atomic_cell_view& cell) { - CounterColumn ret; - ret.__set_name(bytes_to_string(col)); - cell.value().with_linearized([&] (bytes_view value_view) { - ret.__set_value(value_cast(long_type->deserialize_value(value_view))); - }); - return ret; - } - static ColumnOrSuperColumn counter_column_to_column_or_supercolumn(CounterColumn&& col) { - ColumnOrSuperColumn ret; - ret.__set_counter_column(std::move(col)); - return ret; - } - static ColumnOrSuperColumn make_counter_column_or_supercolumn(const bytes& col, const query::result_atomic_cell_view& cell) { - return counter_column_to_column_or_supercolumn(make_counter_column(col, cell)); - } - static std::string partition_key_to_string(const schema& s, const partition_key& key) { - return bytes_to_string(to_legacy(*s.partition_key_type(), key.representation())); - } - - template - struct partition_index; - - template - struct partition_index { - using partition_type = std::map; - partition_type _aggregation; - partition_index(std::vector&& expected) { - // For compatibility reasons, return expected keys even if they don't exist - for (auto&& k : expected) { - _aggregation[std::move(k)] = { }; - } - } - typename Aggregator::type* begin_aggregation(std::string partition_key) { - return &_aggregation[std::move(partition_key)]; - } - }; - template - struct partition_index { - using partition_type = std::vector>; - partition_type _aggregation; - partition_index(std::vector&& expected) { - } - typename Aggregator::type* begin_aggregation(std::string partition_key) { - _aggregation.emplace_back(std::move(partition_key), typename Aggregator::type()); - return &_aggregation.back().second; - } - }; - - template - requires thrift::Aggregator - class column_visitor : public Aggregator { - const schema& _s; - const query::partition_slice& _slice; - const uint32_t _cell_limit; - uint32_t _current_cell_limit; - typename Aggregator::type* _current_aggregation; - partition_index _index; - public: - column_visitor(const schema& s, const query::partition_slice& slice, uint32_t cell_limit, std::vector&& expected) - : _s(s), _slice(slice), _cell_limit(cell_limit), _current_cell_limit(0), _index(std::move(expected)) { - } - typename partition_index::partition_type&& release() { - return std::move(_index._aggregation); - } - void accept_new_partition(const partition_key& key, uint32_t row_count) { - _current_aggregation = _index.begin_aggregation(partition_key_to_string(_s, key)); - _current_cell_limit = _cell_limit; - } - void accept_new_partition(uint32_t row_count) { - // We always ask for the partition_key to be sent in query_opts(). - abort(); - } - void accept_new_row(const clustering_key_prefix& key, const query::result_row_view& static_row, const query::result_row_view& row) { - auto it = row.iterator(); - auto cell = it.next_atomic_cell(); - if (cell && _current_cell_limit > 0) { - bytes column_name = composite::serialize_value(key.components(), _s.thrift().has_compound_comparator()).release_bytes(); - Aggregator::on_column(_current_aggregation, column_name, *cell); - _current_cell_limit -= 1; - } - } - void accept_new_row(const query::result_row_view& static_row, const query::result_row_view& row) { - auto it = row.iterator(); - for (auto&& id : _slice.regular_columns) { - auto cell = it.next_atomic_cell(); - if (cell && _current_cell_limit > 0) { - Aggregator::on_column(_current_aggregation, _s.regular_column_at(id).name(), *cell); - _current_cell_limit -= 1; - } - } - } - void accept_partition_end(const query::result_row_view& static_row) { - } - }; - struct column_or_supercolumn_builder { - using type = std::vector; - void on_column(std::vector* current_cols, const bytes& name, const query::result_atomic_cell_view& cell) { - current_cols->emplace_back(make_column_or_supercolumn(name, cell)); - } - }; - template - using column_aggregator = column_visitor; - struct counter_column_or_supercolumn_builder { - using type = std::vector; - void on_column(std::vector* current_cols, const bytes& name, const query::result_atomic_cell_view& cell) { - current_cols->emplace_back(make_counter_column_or_supercolumn(name, cell)); - } - }; - using counter_column_aggregator = column_visitor; - struct counter { - using type = int32_t; - void on_column(int32_t* current_cols, const bytes_view& name, const query::result_atomic_cell_view& cell) { - *current_cols += 1; - } - }; - using column_counter = column_visitor; - static dht::partition_range_vector make_partition_range(const schema& s, const KeyRange& range) { - if (range.__isset.row_filter) { - fail(unimplemented::cause::INDEXES); - } - if ((range.__isset.start_key == range.__isset.start_token) - || (range.__isset.end_key == range.__isset.end_token)) { - throw make_exception( - "Exactly one each of {start key, start token} and {end key, end token} must be specified"); - } - if (range.__isset.start_token && range.__isset.end_key) { - throw make_exception("Start token + end key is not a supported key range"); - } - - auto&& partitioner = s.get_partitioner(); - - if (range.__isset.start_key && range.__isset.end_key) { - auto start = range.start_key.empty() - ? dht::ring_position::starting_at(dht::minimum_token()) - : partitioner.decorate_key(s, key_from_thrift(s, to_bytes(range.start_key))); - auto end = range.end_key.empty() - ? dht::ring_position::ending_at(dht::maximum_token()) - : partitioner.decorate_key(s, key_from_thrift(s, to_bytes(range.end_key))); - if (end.less_compare(s, start)) { - throw make_exception( - "Start key's token sorts after end key's token. This is not allowed; you probably should not specify end key at all except with an ordered partitioner"); - } - return {{dht::partition_range::bound(std::move(start), true), - dht::partition_range::bound(std::move(end), true)}}; - } - - if (range.__isset.start_key && range.__isset.end_token) { - // start_token/end_token can wrap, but key/token should not - auto start = range.start_key.empty() - ? dht::ring_position::starting_at(dht::minimum_token()) - : partitioner.decorate_key(s, key_from_thrift(s, to_bytes(range.start_key))); - auto end = dht::ring_position::ending_at(dht::token::from_sstring(sstring(range.end_token))); - if (end.token().is_minimum()) { - end = dht::ring_position::ending_at(dht::maximum_token()); - } else if (end.less_compare(s, start)) { - throw make_exception("Start key's token sorts after end token"); - } - return {{dht::partition_range::bound(std::move(start), true), - dht::partition_range::bound(std::move(end), true)}}; - } - - // Token range can wrap; the start token is exclusive. - auto start = dht::ring_position::ending_at(dht::token::from_sstring(sstring(range.start_token))); - auto end = dht::ring_position::ending_at(dht::token::from_sstring(sstring(range.end_token))); - if (end.token().is_minimum()) { - end = dht::ring_position::ending_at(dht::maximum_token()); - } - // Special case of start == end also generates wrap-around range - if (start.token() >= end.token()) { - return {dht::partition_range(dht::partition_range::bound(std::move(start), false), {}), - dht::partition_range({}, dht::partition_range::bound(std::move(end), true))}; - } - return {{dht::partition_range::bound(std::move(start), false), - dht::partition_range::bound(std::move(end), true)}}; - } - static std::vector to_key_slices(const schema& s, const query::partition_slice& slice, query::result_view v, uint32_t cell_limit) { - column_aggregator aggregator(s, slice, cell_limit, { }); - v.consume(slice, aggregator); - auto&& cols = aggregator.release(); - std::vector ret; - std::transform( - std::make_move_iterator(cols.begin()), - std::make_move_iterator(cols.end()), - boost::back_move_inserter(ret), - [](auto&& p) { - KeySlice ks; - ks.__set_key(std::move(p.first)); - ks.__set_columns(std::move(p.second)); - return ks; - }); - return ret; - } - template - static std::vector> make_non_overlapping_ranges( - std::vector column_slices, - const std::function(ColumnSlice&&)> mapper, - Comparator&& cmp, - RangeComparator&& is_wrap_around, - bool reversed) { - std::vector> ranges; - std::transform(column_slices.begin(), column_slices.end(), std::back_inserter(ranges), [&](auto&& cslice) { - const std::string cslice_start = cslice.start; - const std::string cslice_finish = cslice.finish; - auto range = mapper(std::move(cslice)); - if (!reversed && is_wrap_around(range)) { - throw make_exception("Column slice had start {} greater than finish {}", cslice_start, cslice_finish); - } else if (reversed && !is_wrap_around(range)) { - throw make_exception("Reversed column slice had start {} less than finish {}", cslice_start, cslice_finish); - } else if (reversed) { - range.reverse(); - if (is_wrap_around(range)) { - // If a wrap around range is still wrapping after reverse, then it's (a, a). This is equivalent - // to an open ended range. - range = wrapping_interval::make_open_ended_both_sides(); - } - } - return interval(std::move(range)); - }); - return interval::deoverlap(std::move(ranges), std::forward(cmp)); - } - static range_tombstone make_range_tombstone(const schema& s, const SliceRange& range, tombstone tomb) { - using bound = query::clustering_range::bound; - std::optional start_bound; - if (!range.start.empty()) { - start_bound = make_clustering_bound(s, to_bytes_view(range.start), composite::eoc::end); - } - std::optional end_bound; - if (!range.finish.empty()) { - end_bound = make_clustering_bound(s, to_bytes_view(range.finish), composite::eoc::start); - } - return {start_bound ? std::move(*start_bound).value() : clustering_key_prefix::make_empty(), - !start_bound || start_bound->is_inclusive() ? bound_kind::incl_start : bound_kind::excl_start, - end_bound ? std::move(*end_bound).value() : clustering_key_prefix::make_empty(), - !end_bound || end_bound->is_inclusive() ? bound_kind::incl_end : bound_kind::excl_end, - std::move(tomb)}; - } - static void delete_cell(const column_definition& def, api::timestamp_type timestamp, gc_clock::time_point deletion_time, mutation& m_to_apply) { - if (def.is_atomic()) { - auto dead_cell = atomic_cell::make_dead(timestamp, deletion_time); - m_to_apply.set_clustered_cell(clustering_key_prefix::make_empty(), def, std::move(dead_cell)); - } - } - static void delete_column(const schema& s, const sstring& column_name, api::timestamp_type timestamp, gc_clock::time_point deletion_time, mutation& m_to_apply) { - auto&& def = s.get_column_definition(to_bytes(column_name)); - if (def) { - delete_cell(*def, timestamp, deletion_time, m_to_apply); - } - } - static void apply_delete(const schema& s, const SlicePredicate& predicate, api::timestamp_type timestamp, mutation& m_to_apply) { - auto deletion_time = gc_clock::now(); - if (predicate.__isset.column_names) { - thrift_validation::validate_column_names(predicate.column_names); - if (s.thrift().is_dynamic()) { - for (auto&& name : predicate.column_names) { - auto ckey = make_clustering_prefix(s, to_bytes(name)); - m_to_apply.partition().apply_delete(s, std::move(ckey), tombstone(timestamp, deletion_time)); - } - } else { - for (auto&& name : predicate.column_names) { - delete_column(s, name, timestamp, deletion_time, m_to_apply); - } - } - } else if (predicate.__isset.slice_range) { - auto&& range = predicate.slice_range; - if (s.thrift().is_dynamic()) { - m_to_apply.partition().apply_delete(s, make_range_tombstone(s, range, tombstone(timestamp, deletion_time))); - } else { - auto r = make_column_range(s, range.start, range.finish); - std::for_each(r.first, r.second, [&](auto&& def) { - delete_cell(def, timestamp, deletion_time, m_to_apply); - }); - } - } else { - throw make_exception("SlicePredicate column_names and slice_range may not both be null"); - } - } - static void add_live_cell(const schema& s, const Column& col, const column_definition& def, clustering_key_prefix ckey, mutation& m_to_apply) { - thrift_validation::validate_column(col, def); - auto cell = atomic_cell::make_live(*def.type, col.timestamp, to_bytes_view(col.value), maybe_ttl(s, col)); - m_to_apply.set_clustered_cell(std::move(ckey), def, std::move(cell)); - } - static void add_live_cell(const schema& s, const CounterColumn& col, const column_definition& def, clustering_key_prefix ckey, mutation& m_to_apply) { - //thrift_validation::validate_column(col, def); - auto cell = atomic_cell::make_live_counter_update(api::new_timestamp(), col.value); - m_to_apply.set_clustered_cell(std::move(ckey), def, std::move(cell)); - } - static void add_to_mutation(const schema& s, const CounterColumn& col, mutation& m_to_apply) { - thrift_validation::validate_column_name(col.name); - if (s.thrift().is_dynamic()) { - auto&& value_col = s.regular_begin(); - add_live_cell(s, col, *value_col, make_clustering_prefix(s, to_bytes_view(col.name)), m_to_apply); - } else { - auto def = s.get_column_definition(to_bytes(col.name)); - if (def) { - if (def->kind != column_kind::regular_column) { - throw make_exception("Column {} is not settable", col.name); - } - add_live_cell(s, col, *def, clustering_key_prefix::make_empty(s), m_to_apply); - } else { - fail(unimplemented::cause::MIXED_CF); - } - } - } - static void add_to_mutation(const schema& s, const Column& col, mutation& m_to_apply) { - thrift_validation::validate_column_name(col.name); - if (s.thrift().is_dynamic()) { - auto&& value_col = s.regular_begin(); - add_live_cell(s, col, *value_col, make_clustering_prefix(s, to_bytes_view(col.name)), m_to_apply); - } else { - auto def = s.get_column_definition(to_bytes(col.name)); - if (def) { - if (def->kind != column_kind::regular_column) { - throw make_exception("Column {} is not settable", col.name); - } - add_live_cell(s, col, *def, clustering_key_prefix::make_empty(s), m_to_apply); - } else { - fail(unimplemented::cause::MIXED_CF); - } - } - } - static void add_to_mutation(const schema& s, const Mutation& m, mutation& m_to_apply) { - if (m.__isset.column_or_supercolumn) { - if (m.__isset.deletion) { - throw make_exception("Mutation must have one and only one of column_or_supercolumn or deletion"); - } - auto&& cosc = m.column_or_supercolumn; - if (cosc.__isset.column + cosc.__isset.super_column + cosc.__isset.counter_column + cosc.__isset.counter_super_column != 1) { - throw make_exception("ColumnOrSuperColumn must have one (and only one) of column, super_column, counter and counter_super_column"); - } - if (cosc.__isset.column) { - add_to_mutation(s, cosc.column, m_to_apply); - } else if (cosc.__isset.super_column) { - fail(unimplemented::cause::SUPER); - } else if (cosc.__isset.counter_column) { - add_to_mutation(s, cosc.counter_column, m_to_apply); - } else if (cosc.__isset.counter_super_column) { - fail(unimplemented::cause::SUPER); - } - } else if (m.__isset.deletion) { - auto&& del = m.deletion; - if (del.__isset.super_column) { - fail(unimplemented::cause::SUPER); - } else if (del.__isset.predicate) { - apply_delete(s, del.predicate, del.timestamp, m_to_apply); - } else { - m_to_apply.partition().apply(tombstone(del.timestamp, gc_clock::now())); - } - } else { - throw make_exception("Mutation must have either column or deletion"); - } - } - using mutation_map = std::map>>; - using mutation_map_by_cf = std::unordered_map>>; - static mutation_map_by_cf group_by_cf(mutation_map& m) { - mutation_map_by_cf ret; - for (auto&& key_cf : m) { - for (auto&& cf_mutations : key_cf.second) { - auto& mutations = ret[std::move(cf_mutations.first)][std::move(key_cf.first)]; - std::move(cf_mutations.second.begin(), cf_mutations.second.end(), std::back_inserter(mutations)); - } - } - return ret; - } - static std::pair, std::vector> prepare_mutations(data_dictionary::database db, const sstring& ks_name, const mutation_map& m) { - std::vector muts; - std::vector schemas; - auto m_by_cf = group_by_cf(const_cast(m)); - for (auto&& cf_key : m_by_cf) { - auto schema = lookup_schema(db, ks_name, cf_key.first); - if (schema->is_view()) { - throw make_exception("Cannot modify Materialized Views directly"); - } - schemas.emplace_back(schema); - for (auto&& key_mutations : cf_key.second) { - mutation m_to_apply(schema, key_from_thrift(*schema, to_bytes_view(key_mutations.first))); - for (auto&& m : key_mutations.second) { - add_to_mutation(*schema, m, m_to_apply); - } - muts.emplace_back(std::move(m_to_apply)); - } - } - return {std::move(muts), std::move(schemas)}; - } -protected: - service_permit obtain_permit() { - return std::move(_current_permit); - } -}; - -class handler_factory : public CassandraCobSvIfFactory { - data_dictionary::database _db; - distributed& _query_processor; - sharded& _ss; - sharded& _proxy; - auth::service& _auth_service; - const updateable_timeout_config& _timeout_config; - service_permit& _current_permit; -public: - explicit handler_factory(data_dictionary::database db, - distributed& qp, - sharded& ss, - sharded& proxy, - auth::service& auth_service, - const ::updateable_timeout_config& timeout_config, - service_permit& current_permit) - : _db(db), _query_processor(qp), _ss(ss), _proxy(proxy), _auth_service(auth_service), _timeout_config(timeout_config), _current_permit(current_permit) {} - typedef CassandraCobSvIf Handler; - virtual CassandraCobSvIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) { - return new thrift_handler(_db, _query_processor, _ss, _proxy, _auth_service, _timeout_config.current_values(), _current_permit); - } - virtual void releaseHandler(CassandraCobSvIf* handler) { - delete handler; - } -}; - -std::unique_ptr -create_handler_factory(data_dictionary::database db, distributed& qp, - sharded& ss, sharded& proxy, - auth::service& auth_service, const ::updateable_timeout_config& timeout_config, service_permit& current_permit) { - return std::make_unique(db, qp, ss, proxy, auth_service, timeout_config, current_permit); -} diff --git a/thrift/handler.hh b/thrift/handler.hh deleted file mode 100644 index 72bf94734f..0000000000 --- a/thrift/handler.hh +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (C) 2014-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -#ifndef APPS_SEASTAR_THRIFT_HANDLER_HH_ -#define APPS_SEASTAR_THRIFT_HANDLER_HH_ - -#include "Cassandra.h" -#include "auth/service.hh" -#include "cql3/query_processor.hh" -#include - -struct timeout_config; -class service_permit; -namespace service { class storage_service; } - -namespace data_dictionary { -class database; -} - -std::unique_ptr<::cassandra::CassandraCobSvIfFactory> create_handler_factory(data_dictionary::database db, distributed& qp, sharded& ss, sharded& proxy, auth::service&, const updateable_timeout_config&, service_permit& current_permit); - -#endif /* APPS_SEASTAR_THRIFT_HANDLER_HH_ */ diff --git a/thrift/server.cc b/thrift/server.cc deleted file mode 100644 index d2daf17430..0000000000 --- a/thrift/server.cc +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright (C) 2014-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -#include "server.hh" -#include "handler.hh" -#include "db/config.hh" -#include -#include -#include -#include -#include -#include -#include -#include -#include "log.hh" -#include -#include -#include -#include -#include -#include -#include - -#ifdef THRIFT_USES_BOOST -#include -#endif - -static logging::logger tlogger("thrift"); - -using namespace apache::thrift; -using namespace apache::thrift::transport; -using namespace apache::thrift::protocol; -using namespace apache::thrift::async; -using namespace ::cassandra; - -using namespace std::chrono_literals; - -class thrift_stats { - seastar::metrics::metric_groups _metrics; -public: - thrift_stats(thrift_server& server); -}; - -thrift_server::thrift_server(data_dictionary::database db, - distributed& qp, - sharded& ss, - sharded& proxy, - auth::service& auth_service, - service::memory_limiter& ml, - thrift_server_config config) - : _stats(new thrift_stats(*this)) - , _config(std::move(config)) - , _handler_factory(create_handler_factory(db, qp, ss, proxy, auth_service, _config.timeout_config, _current_permit).release()) - , _protocol_factory(new TBinaryProtocolFactoryT()) - , _processor_factory(new CassandraAsyncProcessorFactory(_handler_factory)) - , _memory_available(ml.get_semaphore()) - , _max_concurrent_requests(db.get_config().max_concurrent_requests_per_shard) { -} - -thrift_server::~thrift_server() { -} - -future<> thrift_server::stop() { - auto f = _stop_gate.close(); - std::for_each(_listeners.begin(), _listeners.end(), std::mem_fn(&server_socket::abort_accept)); - std::for_each(_connections_list.begin(), _connections_list.end(), std::mem_fn(&connection::shutdown)); - return f; -} - -struct handler_deleter { - CassandraCobSvIfFactory* hf; - void operator()(CassandraCobSvIf* h) const { - hf->releaseHandler(h); - } -}; - -// thrift uses a shared_ptr to refer to the transport (= connection), -// while we do not, so we can't have connection inherit from TTransport. -struct thrift_server::connection::fake_transport : TTransport { - fake_transport(thrift_server::connection* c) : conn(c) {} - thrift_server::connection* conn; -}; - -thrift_server::connection::connection(thrift_server& server, connected_socket&& fd, socket_address addr) - : _server(server), _fd(std::move(fd)), _read_buf(_fd.input()) - , _write_buf(_fd.output()) - , _transport(thrift_std::make_shared(this)) - , _input(thrift_std::make_shared()) - , _output(thrift_std::make_shared()) - , _in_proto(_server._protocol_factory->getProtocol(_input)) - , _out_proto(_server._protocol_factory->getProtocol(_output)) - , _processor(_server._processor_factory->getProcessor({ _in_proto, _out_proto, _transport })) { - ++_server._total_connections; - ++_server._current_connections; - _server._connections_list.push_back(*this); -} - -thrift_server::connection::~connection() { - if (is_linked()) { - --_server._current_connections; - _server._connections_list.erase(_server._connections_list.iterator_to(*this)); - } -} - -thrift_server::connection::connection(connection&& other) - : _server(other._server) - , _fd(std::move(other._fd)) - , _read_buf(std::move(other._read_buf)) - , _write_buf(std::move(other._write_buf)) - , _transport(std::move(other._transport)) - , _input(std::move(other._input)) - , _output(std::move(other._output)) - , _in_proto(std::move(other._in_proto)) - , _out_proto(std::move(other._out_proto)) - , _processor(std::move(other._processor)) { - if (other.is_linked()) { - boost::intrusive::list::node_algorithms::init(this_ptr()); - boost::intrusive::list::node_algorithms::swap_nodes(other.this_ptr(), this_ptr()); - } -} - -future<> -thrift_server::connection::process() { - return do_until([this] { return _read_buf.eof(); }, - [this] { return process_one_request(); }) - .finally([this] { - return _write_buf.close(); - }); -} - -future<> -thrift_server::connection::process_one_request() { - _input->resetBuffer(); - _output->resetBuffer(); - co_await read(); - if (_server._requests_serving >= _server._max_concurrent_requests) { - _server._requests_shed++; - tlogger.debug("message dropped due to overload"); - co_return; - } - ++_server._requests_serving; - ++_server._requests_served; - auto ret = _processor_promise.get_future().handle_exception([&server = _server] (const std::exception_ptr&) { - server._requests_serving--; - }); - // adapt from "continuation object style" to future/promise - auto complete = [this] (bool success) mutable { - // FIXME: look at success? - _server._requests_serving--; - write().forward_to(std::move(_processor_promise)); - _processor_promise = promise<>(); - }; - // Heuristics copied from transport/server.cc - size_t mem_estimate = 8000 + 2 * _input->available_read(); - auto fut = get_units(_server._memory_available, mem_estimate); - if (_server._memory_available.waiters()) { - ++_server._requests_blocked_memory; - } - auto units = co_await std::move(fut); - // NOTICE: this permit is put in the server under the assumption that no other - // connection will overwrite this permit *until* it's extracted by the code - // which handles the Thrift request (via calling obtain_permit()). - // This assumption is true because there are no preemption points between this - // insertion and the call to obtain_permit(), which was verified both by - // code inspection and confirmed empirically by running manual tests. - if (_server._current_permit.count() > 0) { - tlogger.debug("Current service permit is overwritten while its units are still held ({}). " - "This situation likely means that there's a bug in passing service permits to message handlers.", - _server._current_permit.count()); - } - _server._current_permit = make_service_permit(std::move(units)); - _processor->process(complete, _in_proto, _out_proto); - co_return co_await std::move(ret); -} - -future<> -thrift_server::connection::read() { - return _read_buf.read_exactly(4).then([this] (temporary_buffer size_buf) { - if (size_buf.size() != 4) { - return make_ready_future<>(); - } - union { - uint32_t n; - char b[4]; - } data; - std::copy_n(size_buf.get(), 4, data.b); - auto n = ntohl(data.n); - if (n > _server._config.max_request_size) { - // Close connection silently, we can't return a response because we did not - // read a complete frame. - tlogger.info("message size {} exceeds configured maximum {}, closing connection", n, _server._config.max_request_size); - return make_ready_future<>(); - } - return _read_buf.read_exactly(n).then([this, n] (temporary_buffer buf) { - if (buf.size() != n) { - // FIXME: exception perhaps? - return; - } - _in_tmp = std::move(buf); // keep ownership of the data - auto b = reinterpret_cast(_in_tmp.get_write()); - _input->resetBuffer(b, _in_tmp.size()); - }); - }); -} - -future<> -thrift_server::connection::write() { - uint8_t* data; - uint32_t len; - _output->getBuffer(&data, &len); - net::packed plen = { net::hton(len) }; - return _write_buf.write(reinterpret_cast(&plen), 4).then([this, data, len] { - // FIXME: zero-copy - return _write_buf.write(reinterpret_cast(data), len); - }).then([this] { - return _write_buf.flush(); - }); -} - -void -thrift_server::connection::shutdown() { - try { - _fd.shutdown_input(); - _fd.shutdown_output(); - } catch (...) { - } -} - -future<> -thrift_server::listen(socket_address addr, bool keepalive) { - listen_options lo; - lo.reuse_address = true; - _listeners.push_back(seastar::listen(addr, lo)); - do_accepts(_listeners.size() - 1, keepalive, 0); - return make_ready_future<>(); -} - -void -thrift_server::do_accepts(int which, bool keepalive, int num_attempts) { - if (_stop_gate.is_closed()) { - return; - } - // Future is waited on indirectly in `stop()` (via `_stop_gate`). - (void)with_gate(_stop_gate, [&, this] { - return _listeners[which].accept().then([this, which, keepalive] (accept_result ar) { - auto&& fd = ar.connection; - auto&& addr = ar.remote_address; - fd.set_nodelay(true); - fd.set_keepalive(keepalive); - // Future is waited on indirectly in `stop()` (via `_stop_gate`). - (void)with_gate(_stop_gate, [&, this] { - return do_with(connection(*this, std::move(fd), addr), [] (auto& conn) { - return conn.process().then_wrapped([&conn] (future<> f) { - conn.shutdown(); - try { - f.get(); - } catch (std::exception& ex) { - tlogger.debug("request error {}", ex.what()); - } - }); - }); - }); - do_accepts(which, keepalive, 0); - }).handle_exception([this, which, keepalive, num_attempts] (auto ex) { - tlogger.debug("accept failed {}", ex); - try { - std::rethrow_exception(std::move(ex)); - } catch (const seastar::gate_closed_exception&) { - return; - } catch (...) { - if (_stop_gate.is_closed()) { - return; - } - // Done in the background. - (void)with_gate(_stop_gate, [this, which, keepalive, num_attempts] { - int backoff = 2 << std::max(num_attempts, 10); - tlogger.debug("sleeping for {}ms", backoff); - return sleep(std::chrono::milliseconds(backoff)).then([this, which, keepalive, num_attempts] { - tlogger.debug("retrying accept after failure"); - do_accepts(which, keepalive, num_attempts + 1); - }); - }); - } - }); - }); -} - -uint64_t -thrift_server::total_connections() const { - return _total_connections; -} - -uint64_t -thrift_server::current_connections() const { - return _current_connections; -} - -uint64_t -thrift_server::requests_served() const { - return _requests_served; -} - -uint64_t -thrift_server::requests_serving() const { - return _requests_serving; -} - -size_t -thrift_server::max_request_size() const { - return _config.max_request_size; -} - -const semaphore& -thrift_server::memory_available() const { - return _memory_available; -} - -uint64_t -thrift_server::requests_blocked_memory() const { - return _requests_blocked_memory; -} - -uint64_t -thrift_server::requests_shed() const { - return _requests_shed; -} - -thrift_stats::thrift_stats(thrift_server& server) { - namespace sm = seastar::metrics; - - _metrics.add_group("thrift", { - sm::make_counter("thrift-connections", [&server] { return server.total_connections(); }, - sm::description("Rate of creation of new Thrift connections.")), - - sm::make_gauge("current_connections", [&server] { return server.current_connections(); }, - sm::description("Holds a current number of opened Thrift connections.")), - - sm::make_counter("served", [&server] { return server.requests_served(); }, - sm::description("Rate of serving Thrift requests.")), - sm::make_gauge("serving", [&server] { return server.requests_serving(); }, - sm::description("Number of Thrift requests being currently served.")), - sm::make_gauge("requests_blocked_memory_current", [&server] { return server.memory_available().waiters(); }, - sm::description( - seastar::format("Holds the number of Thrift requests that are currently blocked due to reaching the memory quota limit ({}B). " - "Non-zero value indicates that our bottleneck is memory and more specifically - the memory quota allocated for the \"Thrift transport\" component.", server.max_request_size()))), - sm::make_counter("requests_blocked_memory", [&server] { return server.requests_blocked_memory(); }, - sm::description( - seastar::format("Holds an incrementing counter with the Thrift requests that ever blocked due to reaching the memory quota limit ({}B). " - "The first derivative of this value shows how often we block due to memory exhaustion in the \"Thrift transport\" component.", server.max_request_size()))), - sm::make_counter("requests_shed", [&server] { return server.requests_shed(); }, - sm::description("Holds an incrementing counter with the requests that were shed due to exceeding the threshold configured via max_concurrent_requests_per_shard. " - "The first derivative of this value shows how often we shed requests due to exceeding the limit in the \"Thrift transport\" component.")), - sm::make_gauge("requests_memory_available", [&server] { return server.memory_available().current(); }, - sm::description( - seastar::format("Holds the amount of available memory for admitting new Thrift requests (max is {}B)." - "Zero value indicates that our bottleneck is memory and more specifically - the memory quota allocated for the \"Thrift transport\" component.", server.max_request_size()))) - }); -} - diff --git a/thrift/server.hh b/thrift/server.hh deleted file mode 100644 index 5222d2308c..0000000000 --- a/thrift/server.hh +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (C) 2014-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -#ifndef APPS_SEASTAR_THRIFT_SERVER_HH_ -#define APPS_SEASTAR_THRIFT_SERVER_HH_ - -#include -#include -#include "cql3/query_processor.hh" -#include "timeout_config.hh" -#include "service/memory_limiter.hh" -#include -#include -#include -#include -#include "utils/updateable_value.hh" -#include "service_permit.hh" - -class thrift_server; -class thrift_stats; - -#ifdef THRIFT_USES_BOOST -namespace thrift_std = boost; -#else -namespace thrift_std = std; -#endif - -namespace cassandra { - -static const sstring thrift_version = "20.1.0"; - -class CassandraCobSvIfFactory; - -} - -namespace apache { namespace thrift { namespace protocol { - -class TProtocolFactory; -class TProtocol; - -}}} - -namespace apache { namespace thrift { namespace async { - -class TAsyncProcessor; -class TAsyncProcessorFactory; - -}}} - -namespace apache { namespace thrift { namespace transport { - -class TMemoryBuffer; - -}}} - -namespace auth { -class service; -} - -namespace service { class storage_service; } - -namespace data_dictionary { -class database; -} - -struct thrift_server_config { - ::updateable_timeout_config timeout_config; - uint64_t max_request_size; - std::function get_service_memory_limiter_semaphore; -}; - -class thrift_server { - class connection : public boost::intrusive::list_base_hook<> { - struct fake_transport; - thrift_server& _server; - connected_socket _fd; - input_stream _read_buf; - output_stream _write_buf; - temporary_buffer _in_tmp; - thrift_std::shared_ptr _transport; - thrift_std::shared_ptr _input; - thrift_std::shared_ptr _output; - thrift_std::shared_ptr _in_proto; - thrift_std::shared_ptr _out_proto; - thrift_std::shared_ptr _processor; - promise<> _processor_promise; - public: - connection(thrift_server& server, connected_socket&& fd, socket_address addr); - ~connection(); - connection(connection&&); - future<> process(); - future<> read(); - future<> write(); - void shutdown(); - private: - future<> process_one_request(); - }; -private: - std::vector _listeners; - std::unique_ptr _stats; - service_permit _current_permit = empty_service_permit(); - thrift_server_config _config; - thrift_std::shared_ptr<::cassandra::CassandraCobSvIfFactory> _handler_factory; - std::unique_ptr _protocol_factory; - thrift_std::shared_ptr _processor_factory; - uint64_t _total_connections = 0; - uint64_t _current_connections = 0; - uint64_t _requests_served = 0; - uint64_t _requests_serving = 0; - uint64_t _requests_blocked_memory = 0; - semaphore& _memory_available; - utils::updateable_value _max_concurrent_requests; - size_t _requests_shed; - boost::intrusive::list _connections_list; - seastar::gate _stop_gate; -public: - thrift_server(data_dictionary::database db, distributed& qp, sharded& ss, sharded& proxy, auth::service&, service::memory_limiter& ml, thrift_server_config config); - ~thrift_server(); - future<> listen(socket_address addr, bool keepalive); - future<> stop(); - void do_accepts(int which, bool keepalive, int num_attempts); - uint64_t total_connections() const; - uint64_t current_connections() const; - uint64_t requests_served() const; - uint64_t requests_serving() const; - size_t max_request_size() const; - const semaphore& memory_available() const; - uint64_t requests_blocked_memory() const; - uint64_t requests_shed() const; - -private: - void maybe_retry_accept(int which, bool keepalive, std::exception_ptr ex); -}; - -#endif /* APPS_SEASTAR_THRIFT_SERVER_HH_ */ diff --git a/thrift/thrift_validation.cc b/thrift/thrift_validation.cc deleted file mode 100644 index c890cde3ad..0000000000 --- a/thrift/thrift_validation.cc +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2015-present ScyllaDB - * - * Modified by ScyllaDB - */ - -/* - * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0) - */ - -#include - -#include "thrift_validation.hh" -#include "thrift/utils.hh" -#include "db/system_keyspace.hh" -#include - -using namespace thrift; -using namespace ::apache::thrift; - -namespace thrift_validation { - -static constexpr uint32_t MAX_UNSIGNED_SHORT = std::numeric_limits::max(); - -void validate_key(const schema& s, const bytes_view& k) { - if (k.empty()) { - throw make_exception("Key may not be empty"); - } - - auto max = MAX_UNSIGNED_SHORT; - if (k.size() > max) { - throw make_exception("Key length of {} is longer than maximum of {}", k.size(), max); - } - - // FIXME: implement - //s.partition_key_type()->validate(k); -} - -void validate_keyspace_not_system(const std::string& keyspace) { - std::string name; - name.resize(keyspace.length()); - std::transform(keyspace.begin(), keyspace.end(), name.begin(), ::tolower); - if (is_system_keyspace(name)) { - throw make_exception("system keyspace is not user-modifiable"); - } -} - -void validate_ks_def(const KsDef& ks_def) { - validate_keyspace_not_system(ks_def.name); - boost::regex name_regex("\\w+"); - if (!boost::regex_match(ks_def.name, name_regex)) { - throw make_exception("\"{}\" is not a valid keyspace name", ks_def.name); - } - if (ks_def.name.length() > schema::NAME_LENGTH) { - throw make_exception("Keyspace names shouldn't be more than {} characters long (got \"{}\")", schema::NAME_LENGTH, ks_def.name); - } -} - -void validate_cf_def(const CfDef& cf_def) { - boost::regex name_regex("\\w+"); - if (!boost::regex_match(cf_def.name, name_regex)) { - throw make_exception("\"{}\" is not a valid column family name", cf_def.name); - } - if (cf_def.name.length() > schema::NAME_LENGTH) { - throw make_exception("Keyspace names shouldn't be more than {} characters long (got \"{}\")", schema::NAME_LENGTH, cf_def.name); - } -} - -void validate_column_name(const std::string& name) { - auto max_name_length = MAX_UNSIGNED_SHORT; - if (name.size() > max_name_length) { - throw make_exception("column name length must not be greater than {}", max_name_length); - } - if (name.empty()) { - throw make_exception("column name must not be empty"); - } -} - -void validate_column_names(const std::vector& names) { - for (auto&& name : names) { - validate_column_name(name); - } -} - -void validate_column(const Column& col, const column_definition& def) { - if (!col.__isset.value) { - throw make_exception("Column value is required"); - } - if (!col.__isset.timestamp) { - throw make_exception("Column timestamp is required"); - } - def.type->validate(to_bytes_view(col.value)); -} - -} diff --git a/thrift/thrift_validation.hh b/thrift/thrift_validation.hh deleted file mode 100644 index e7986e0793..0000000000 --- a/thrift/thrift_validation.hh +++ /dev/null @@ -1,643 +0,0 @@ -/* - * Copyright (C) 2015-present ScyllaDB - * - * Modified by ScyllaDB - */ - -/* - * SPDX-License-Identifier: (AGPL-3.0-or-later and Apache-2.0) - */ - -#pragma once - -#include "schema/schema.hh" -#include "bytes.hh" -#include "cassandra_types.h" - -using namespace ::cassandra; - -#if 0 -import java.nio.ByteBuffer; -import java.util.*; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.cassandra.config.*; -import org.apache.cassandra.cql3.ColumnIdentifier; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.composites.*; -import org.apache.cassandra.db.filter.IDiskAtomFilter; -import org.apache.cassandra.db.filter.NamesQueryFilter; -import org.apache.cassandra.db.filter.SliceQueryFilter; -import org.apache.cassandra.db.index.SecondaryIndexManager; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.ColumnToCollectionType; -import org.apache.cassandra.db.marshal.UTF8Type; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.Token; -import org.apache.cassandra.serializers.MarshalException; -import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; -#endif - -/** - * This has a lot of building blocks for CassandraServer to call to make sure it has valid input - * -- ensuring column names conform to the declared comparator, for instance. - * - * The methods here mostly try to do just one part of the validation so they can be combined - * for different needs -- supercolumns vs regular, range slices vs named, batch vs single-column. - * (ValidateColumnPath is the main exception in that it includes keyspace and CF validation.) - */ -namespace thrift_validation { - - -#if 0 - private static final Logger logger = LoggerFactory.getLogger(ThriftValidation.class); -#endif - -void validate_key(const schema& s, const bytes_view& key); -void validate_keyspace_not_system(const std::string& keyspace); -void validate_ks_def(const KsDef& ks_def); -void validate_cf_def(const CfDef& cf_def); -void validate_column_name(const std::string& name); -void validate_column_names(const std::vector& names); -void validate_column(const Column& col, const column_definition& def); - -#if 0 - public static void validateKeyspace(String keyspaceName) throws KeyspaceNotDefinedException - { - if (!Schema.instance.getKeyspaces().contains(keyspaceName)) - { - throw new KeyspaceNotDefinedException("Keyspace " + keyspaceName + " does not exist"); - } - } - - public static CFMetaData validateColumnFamily(String keyspaceName, String cfName, boolean isCommutativeOp) throws org.apache.cassandra.exceptions.InvalidRequestException - { - CFMetaData metadata = validateColumnFamily(keyspaceName, cfName); - - if (isCommutativeOp) - { - if (!metadata.isCounter()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for non commutative table " + cfName); - } - else - { - if (metadata.isCounter()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for commutative table " + cfName); - } - return metadata; - } - - /** - * validates all parts of the path to the column, including the column name - */ - public static void validateColumnPath(CFMetaData metadata, ColumnPath column_path) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (metadata.cfType == ColumnFamilyType.Standard) - { - if (column_path.super_column != null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn parameter is invalid for standard CF " + metadata.cfName); - } - if (column_path.column == null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("column parameter is not optional for standard CF " + metadata.cfName); - } - } - else - { - if (column_path.super_column == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn parameter is not optional for super CF " + metadata.cfName); - } - if (column_path.column != null) - { - validateColumnNames(metadata, column_path.super_column, Arrays.asList(column_path.column)); - } - if (column_path.super_column != null) - { - validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(column_path.super_column)); - } - } - - public static void validateColumnParent(CFMetaData metadata, ColumnParent column_parent) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (metadata.cfType == ColumnFamilyType.Standard) - { - if (column_parent.super_column != null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("table alone is required for standard CF " + metadata.cfName); - } - } - - if (column_parent.super_column != null) - { - validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(column_parent.super_column)); - } - } - - // column_path_or_parent is a ColumnPath for remove, where the "column" is optional even for a standard CF - static void validateColumnPathOrParent(CFMetaData metadata, ColumnPath column_path_or_parent) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (metadata.cfType == ColumnFamilyType.Standard) - { - if (column_path_or_parent.super_column != null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn may not be specified for standard CF " + metadata.cfName); - } - } - if (metadata.cfType == ColumnFamilyType.Super) - { - if (column_path_or_parent.super_column == null && column_path_or_parent.column != null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("A column cannot be specified without specifying a super column for removal on super CF " - + metadata.cfName); - } - } - if (column_path_or_parent.column != null) - { - validateColumnNames(metadata, column_path_or_parent.super_column, Arrays.asList(column_path_or_parent.column)); - } - if (column_path_or_parent.super_column != null) - { - validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(column_path_or_parent.super_column)); - } - } - - /** - * Validates the column names but not the parent path or data - */ - private static void validateColumnNames(CFMetaData metadata, ByteBuffer superColumnName, Iterable column_names) - throws org.apache.cassandra.exceptions.InvalidRequestException - { - int maxNameLength = Cell.MAX_NAME_LENGTH; - - if (superColumnName != null) - { - if (superColumnName.remaining() > maxNameLength) - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name length must not be greater than " + maxNameLength); - if (superColumnName.remaining() == 0) - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn name must not be empty"); - if (metadata.cfType == ColumnFamilyType.Standard) - throw new org.apache.cassandra.exceptions.InvalidRequestException("supercolumn specified to table " + metadata.cfName + " containing normal columns"); - } - AbstractType comparator = SuperColumns.getComparatorFor(metadata, superColumnName); - boolean isCQL3Table = !metadata.isThriftCompatible(); - for (ByteBuffer name : column_names) - { - if (name.remaining() > maxNameLength) - throw new org.apache.cassandra.exceptions.InvalidRequestException("column name length must not be greater than " + maxNameLength); - if (name.remaining() == 0) - throw new org.apache.cassandra.exceptions.InvalidRequestException("column name must not be empty"); - try - { - comparator.validate(name); - } - catch (MarshalException e) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); - } - - if (isCQL3Table) - { - // CQL3 table don't support having only part of their composite column names set - Composite composite = metadata.comparator.fromByteBuffer(name); - - int minComponents = metadata.comparator.clusteringPrefixSize() + 1; - if (composite.size() < minComponents) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Not enough components (found %d but %d expected) for column name since %s is a CQL3 table", - composite.size(), minComponents, metadata.cfName)); - - // Furthermore, the column name must be a declared one. - int columnIndex = metadata.comparator.clusteringPrefixSize(); - ByteBuffer CQL3ColumnName = composite.get(columnIndex); - if (!CQL3ColumnName.hasRemaining()) - continue; // Row marker, ok - - ColumnIdentifier columnId = new ColumnIdentifier(CQL3ColumnName, metadata.comparator.subtype(columnIndex)); - if (metadata.getColumnDefinition(columnId) == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid cell for CQL3 table %s. The CQL3 column component (%s) does not correspond to a defined CQL3 column", - metadata.cfName, columnId)); - - // On top of that, if we have a collection component, he (CQL3) column must be a collection - if (metadata.comparator.hasCollections() && composite.size() == metadata.comparator.size()) - { - ColumnToCollectionType collectionType = metadata.comparator.collectionType(); - if (!collectionType.defined.containsKey(CQL3ColumnName)) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Invalid collection component, %s is not a collection", UTF8Type.instance.getString(CQL3ColumnName))); - } - } - } - } - - public static void validateColumnNames(CFMetaData metadata, ColumnParent column_parent, Iterable column_names) throws org.apache.cassandra.exceptions.InvalidRequestException - { - validateColumnNames(metadata, column_parent.super_column, column_names); - } - - public static void validateRange(CFMetaData metadata, ColumnParent column_parent, SliceRange range) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (range.count < 0) - throw new org.apache.cassandra.exceptions.InvalidRequestException("get_slice requires non-negative count"); - - int maxNameLength = Cell.MAX_NAME_LENGTH; - if (range.start.remaining() > maxNameLength) - throw new org.apache.cassandra.exceptions.InvalidRequestException("range start length cannot be larger than " + maxNameLength); - if (range.finish.remaining() > maxNameLength) - throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish length cannot be larger than " + maxNameLength); - - AbstractType comparator = SuperColumns.getComparatorFor(metadata, column_parent.super_column); - try - { - comparator.validate(range.start); - comparator.validate(range.finish); - } - catch (MarshalException e) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException(e.getMessage()); - } - - Comparator orderedComparator = range.isReversed() ? comparator.reverseComparator : comparator; - if (range.start.remaining() > 0 - && range.finish.remaining() > 0 - && orderedComparator.compare(range.start, range.finish) > 0) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("range finish must come after start in the order of traversal"); - } - } - - public static void validateColumnOrSuperColumn(CFMetaData metadata, ColumnOrSuperColumn cosc) - throws org.apache.cassandra.exceptions.InvalidRequestException - { - boolean isCommutative = metadata.isCounter(); - - int nulls = 0; - if (cosc.column == null) nulls++; - if (cosc.super_column == null) nulls++; - if (cosc.counter_column == null) nulls++; - if (cosc.counter_super_column == null) nulls++; - - if (nulls != 3) - throw new org.apache.cassandra.exceptions.InvalidRequestException("ColumnOrSuperColumn must have one (and only one) of column, super_column, counter and counter_super_column"); - - if (cosc.column != null) - { - if (isCommutative) - throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for commutative table " + metadata.cfName); - - validateTtl(cosc.column); - validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.column.name)); - validateColumnData(metadata, null, cosc.column); - } - - if (cosc.super_column != null) - { - if (isCommutative) - throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for commutative table " + metadata.cfName); - - for (Column c : cosc.super_column.columns) - { - validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.super_column.name).setColumn(c.name)); - validateColumnData(metadata, cosc.super_column.name, c); - } - } - - if (cosc.counter_column != null) - { - if (!isCommutative) - throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for non commutative table " + metadata.cfName); - - validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column((ByteBuffer)null).setColumn(cosc.counter_column.name)); - } - - if (cosc.counter_super_column != null) - { - if (!isCommutative) - throw new org.apache.cassandra.exceptions.InvalidRequestException("invalid operation for non commutative table " + metadata.cfName); - - for (CounterColumn c : cosc.counter_super_column.columns) - validateColumnPath(metadata, new ColumnPath(metadata.cfName).setSuper_column(cosc.counter_super_column.name).setColumn(c.name)); - } - } - - private static void validateTtl(Column column) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (column.isSetTtl()) - { - if (column.ttl <= 0) - throw new org.apache.cassandra.exceptions.InvalidRequestException("ttl must be positive"); - - if (column.ttl > ExpiringCell.MAX_TTL) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("ttl is too large. requested (%d) maximum (%d)", column.ttl, ExpiringCell.MAX_TTL)); - } - else - { - // if it's not set, then it should be zero -- here we are just checking to make sure Thrift doesn't change that contract with us. - assert column.ttl == 0; - } - } - - public static void validateMutation(CFMetaData metadata, Mutation mut) - throws org.apache.cassandra.exceptions.InvalidRequestException - { - ColumnOrSuperColumn cosc = mut.column_or_supercolumn; - Deletion del = mut.deletion; - - int nulls = 0; - if (cosc == null) nulls++; - if (del == null) nulls++; - - if (nulls != 1) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("mutation must have one and only one of column_or_supercolumn or deletion"); - } - - if (cosc != null) - { - validateColumnOrSuperColumn(metadata, cosc); - } - else - { - validateDeletion(metadata, del); - } - } - - public static void validateDeletion(CFMetaData metadata, Deletion del) throws org.apache.cassandra.exceptions.InvalidRequestException - { - - if (del.super_column != null) - validateColumnNames(metadata, (ByteBuffer)null, Arrays.asList(del.super_column)); - - if (del.predicate != null) - validateSlicePredicate(metadata, del.super_column, del.predicate); - - if (metadata.cfType == ColumnFamilyType.Standard && del.super_column != null) - { - String msg = String.format("Deletion of super columns is not possible on a standard table (KeySpace=%s Table=%s Deletion=%s)", metadata.ksName, metadata.cfName, del); - throw new org.apache.cassandra.exceptions.InvalidRequestException(msg); - } - - if (metadata.isCounter()) - { - // forcing server timestamp even if a timestamp was set for coherence with other counter operation - del.timestamp = System.currentTimeMillis(); - } - else if (!del.isSetTimestamp()) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("Deletion timestamp is not optional for non commutative table " + metadata.cfName); - } - } - - public static void validateSlicePredicate(CFMetaData metadata, ByteBuffer scName, SlicePredicate predicate) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (predicate.column_names == null && predicate.slice_range == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("A SlicePredicate must be given a list of Columns, a SliceRange, or both"); - - if (predicate.slice_range != null) - validateRange(metadata, new ColumnParent(metadata.cfName).setSuper_column(scName), predicate.slice_range); - - if (predicate.column_names != null) - validateColumnNames(metadata, scName, predicate.column_names); - } - - /** - * Validates the data part of the column (everything in the column object but the name, which is assumed to be valid) - */ - public static void validateColumnData(CFMetaData metadata, ByteBuffer scName, Column column) throws org.apache.cassandra.exceptions.InvalidRequestException - { - validateTtl(column); - if (!column.isSetValue()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("Column value is required"); - if (!column.isSetTimestamp()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("Column timestamp is required"); - - CellName cn = scName == null - ? metadata.comparator.cellFromByteBuffer(column.name) - : metadata.comparator.makeCellName(scName, column.name); - try - { - AbstractType validator = metadata.getValueValidator(cn); - if (validator != null) - validator.validate(column.value); - } - catch (MarshalException me) - { - if (logger.isDebugEnabled()) - logger.debug("rejecting invalid value {}", ByteBufferUtil.bytesToHex(summarize(column.value))); - - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("(%s) [%s][%s][%s] failed validation", - me.getMessage(), - metadata.ksName, - metadata.cfName, - (SuperColumns.getComparatorFor(metadata, scName != null)).getString(column.name))); - } - - // Indexed column values cannot be larger than 64K. See CASSANDRA-3057/4240 for more details - if (!Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager.validate(asDBColumn(cn, column))) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Can't index column value of size %d for index %s in CF %s of KS %s", - column.value.remaining(), - metadata.getColumnDefinition(cn).getIndexName(), - metadata.cfName, - metadata.ksName)); - } - - private static Cell asDBColumn(CellName name, Column column) - { - if (column.ttl <= 0) - return new BufferCell(name, column.value, column.timestamp); - else - return new BufferExpiringCell(name, column.value, column.timestamp, column.ttl); - } - - /** - * Return, at most, the first 64K of the buffer. This avoids very large column values being - * logged in their entirety. - */ - private static ByteBuffer summarize(ByteBuffer buffer) - { - int MAX = Short.MAX_VALUE; - if (buffer.remaining() <= MAX) - return buffer; - return (ByteBuffer) buffer.slice().limit(buffer.position() + MAX); - } - - - public static void validatePredicate(CFMetaData metadata, ColumnParent column_parent, SlicePredicate predicate) - throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (predicate.column_names == null && predicate.slice_range == null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("predicate column_names and slice_range may not both be null"); - if (predicate.column_names != null && predicate.slice_range != null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("predicate column_names and slice_range may not both be present"); - - if (predicate.getSlice_range() != null) - validateRange(metadata, column_parent, predicate.slice_range); - else - validateColumnNames(metadata, column_parent, predicate.column_names); - } - - public static void validateKeyRange(CFMetaData metadata, ByteBuffer superColumn, KeyRange range) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if ((range.start_key == null) == (range.start_token == null) - || (range.end_key == null) == (range.end_token == null)) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("exactly one each of {start key, start token} and {end key, end token} must be specified"); - } - - // (key, token) is supported (for wide-row CFRR) but not (token, key) - if (range.start_token != null && range.end_key != null) - throw new org.apache.cassandra.exceptions.InvalidRequestException("start token + end key is not a supported key range"); - - IPartitioner p = StorageService.getPartitioner(); - - if (range.start_key != null && range.end_key != null) - { - Token startToken = p.getToken(range.start_key); - Token endToken = p.getToken(range.end_key); - if (startToken.compareTo(endToken) > 0 && !endToken.isMinimum()) - { - if (p.preservesOrder()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("start key must sort before (or equal to) finish key in your partitioner!"); - else - throw new org.apache.cassandra.exceptions.InvalidRequestException("start key's token sorts after end key's token. this is not allowed; you probably should not specify end key at all except with an ordered partitioner"); - } - } - else if (range.start_key != null && range.end_token != null) - { - // start_token/end_token can wrap, but key/token should not - RowPosition stop = p.getTokenFactory().fromString(range.end_token).maxKeyBound(); - if (RowPosition.ForKey.get(range.start_key, p).compareTo(stop) > 0 && !stop.isMinimum()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("Start key's token sorts after end token"); - } - - validateFilterClauses(metadata, range.row_filter); - - if (!isEmpty(range.row_filter) && superColumn != null) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("super columns are not supported for indexing"); - } - - if (range.count <= 0) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException("maxRows must be positive"); - } - } - - private static boolean isEmpty(List clause) - { - return clause == null || clause.isEmpty(); - } - - public static void validateIndexClauses(CFMetaData metadata, IndexClause index_clause) - throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (index_clause.expressions.isEmpty()) - throw new org.apache.cassandra.exceptions.InvalidRequestException("index clause list may not be empty"); - - if (!validateFilterClauses(metadata, index_clause.expressions)) - throw new org.apache.cassandra.exceptions.InvalidRequestException("No indexed columns present in index clause with operator EQ"); - } - - // return true if index_clause contains an indexed columns with operator EQ - public static boolean validateFilterClauses(CFMetaData metadata, List index_clause) - throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (isEmpty(index_clause)) - // no filter to apply - return false; - - SecondaryIndexManager idxManager = Keyspace.open(metadata.ksName).getColumnFamilyStore(metadata.cfName).indexManager; - AbstractType nameValidator = SuperColumns.getComparatorFor(metadata, null); - - boolean isIndexed = false; - for (IndexExpression expression : index_clause) - { - try - { - nameValidator.validate(expression.column_name); - } - catch (MarshalException me) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("[%s]=[%s] failed name validation (%s)", - ByteBufferUtil.bytesToHex(expression.column_name), - ByteBufferUtil.bytesToHex(expression.value), - me.getMessage())); - } - - if (expression.value.remaining() > 0xFFFF) - throw new org.apache.cassandra.exceptions.InvalidRequestException("Index expression values may not be larger than 64K"); - - CellName name = metadata.comparator.cellFromByteBuffer(expression.column_name); - AbstractType valueValidator = metadata.getValueValidator(name); - try - { - valueValidator.validate(expression.value); - } - catch (MarshalException me) - { - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("[%s]=[%s] failed value validation (%s)", - ByteBufferUtil.bytesToHex(expression.column_name), - ByteBufferUtil.bytesToHex(expression.value), - me.getMessage())); - } - - isIndexed |= (expression.op == IndexOperator.EQ) && idxManager.indexes(name); - } - - return isIndexed; - } - - public static void validateKeyspaceNotYetExisting(String newKsName) throws org.apache.cassandra.exceptions.InvalidRequestException - { - // keyspace names must be unique case-insensitively because the keyspace name becomes the directory - // where we store CF sstables. Names that differ only in case would thus cause problems on - // case-insensitive filesystems (NTFS, most installations of HFS+). - for (String ksName : Schema.instance.getKeyspaces()) - { - if (ksName.equalsIgnoreCase(newKsName)) - throw new org.apache.cassandra.exceptions.InvalidRequestException(String.format("Keyspace names must be case-insensitively unique (\"%s\" conflicts with \"%s\")", - newKsName, - ksName)); - } - } - - public static void validateKeyspaceNotSystem(String modifiedKeyspace) throws org.apache.cassandra.exceptions.InvalidRequestException - { - if (modifiedKeyspace.equalsIgnoreCase(SystemKeyspace.NAME)) - throw new org.apache.cassandra.exceptions.InvalidRequestException("system keyspace is not user-modifiable"); - } - - public static IDiskAtomFilter asIFilter(SlicePredicate sp, CFMetaData metadata, ByteBuffer superColumn) - { - SliceRange sr = sp.slice_range; - IDiskAtomFilter filter; - - CellNameType comparator = metadata.isSuper() - ? new SimpleDenseCellNameType(metadata.comparator.subtype(superColumn == null ? 0 : 1)) - : metadata.comparator; - if (sr == null) - { - - SortedSet ss = new TreeSet(comparator); - for (ByteBuffer bb : sp.column_names) - ss.add(comparator.cellFromByteBuffer(bb)); - filter = new NamesQueryFilter(ss); - } - else - { - filter = new SliceQueryFilter(comparator.fromByteBuffer(sr.start), - comparator.fromByteBuffer(sr.finish), - sr.reversed, - sr.count); - } - - if (metadata.isSuper()) - filter = SuperColumns.fromSCFilter(metadata.comparator, superColumn, filter); - return filter; - } -} -#endif - -} diff --git a/thrift/utils.hh b/thrift/utils.hh deleted file mode 100644 index 6988794c2e..0000000000 --- a/thrift/utils.hh +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2016-present ScyllaDB - */ - -/* - * SPDX-License-Identifier: AGPL-3.0-or-later - */ - -#pragma once - -#include -#include "utils/fmt-compat.hh" - -namespace thrift { - -template -Ex -make_exception(const char* fmt, Args&&... args) { - Ex ex; - ex.why = fmt::format(fmt::runtime(fmt), std::forward(args)...); - return ex; -} - -} diff --git a/timeout_config.hh b/timeout_config.hh index c7e3630840..018ef07b57 100644 --- a/timeout_config.hh +++ b/timeout_config.hh @@ -18,7 +18,7 @@ class updateable_timeout_config; /// timeout_config represents a snapshot of the options stored in it when /// an instance of this class is created. so far this class is only used by -/// client_state and thrift_handler. so either these classes are obliged to +/// client_state. so either these classes are obliged to /// update it by themselves, or they are fine with using the maybe-updated /// options in the lifecycle of a client / connection even if some of these /// options are changed whtn the client / connection is still alive. diff --git a/transport/messages/result_message.cc b/transport/messages/result_message.cc index 013c1e8ff7..bc3c33231f 100644 --- a/transport/messages/result_message.cc +++ b/transport/messages/result_message.cc @@ -34,11 +34,6 @@ std::ostream& operator<<(std::ostream& os, const result_message::set_keyspace& m return os; } -std::ostream& operator<<(std::ostream& os, const result_message::prepared::thrift& msg) { - fmt::print(os, "{{result_message::prepared::thrift {:d}}}", msg.get_id()); - return os; -} - std::ostream& operator<<(std::ostream& os, const result_message::prepared::cql& msg) { fmt::print(os, "{{result_message::prepared::cql {}}}", to_hex(msg.get_id())); return os; @@ -58,7 +53,6 @@ std::ostream& operator<<(std::ostream& os, const result_message& msg) { void visit(const result_message::void_message& m) override { _os << m; }; void visit(const result_message::set_keyspace& m) override { _os << m; }; void visit(const result_message::prepared::cql& m) override { _os << m; }; - void visit(const result_message::prepared::thrift& m) override { _os << m; }; void visit(const result_message::schema_change& m) override { _os << m; }; void visit(const result_message::rows& m) override { fmt::print(_os, "{}", m); }; void visit(const result_message::bounce_to_shard& m) override { _os << m; }; diff --git a/transport/messages/result_message.hh b/transport/messages/result_message.hh index 05c01561be..2d3b477146 100644 --- a/transport/messages/result_message.hh +++ b/transport/messages/result_message.hh @@ -47,7 +47,6 @@ public: } class cql; - class thrift; private: static ::shared_ptr extract_result_metadata(::shared_ptr statement); }; @@ -57,7 +56,6 @@ public: virtual void visit(const result_message::void_message&) = 0; virtual void visit(const result_message::set_keyspace&) = 0; virtual void visit(const result_message::prepared::cql&) = 0; - virtual void visit(const result_message::prepared::thrift&) = 0; virtual void visit(const result_message::schema_change&) = 0; virtual void visit(const result_message::rows&) = 0; virtual void visit(const result_message::bounce_to_shard&) = 0; @@ -69,7 +67,6 @@ public: void visit(const result_message::void_message&) override {}; void visit(const result_message::set_keyspace&) override {}; void visit(const result_message::prepared::cql&) override {}; - void visit(const result_message::prepared::thrift&) override {}; void visit(const result_message::schema_change&) override {}; void visit(const result_message::rows&) override {}; void visit(const result_message::bounce_to_shard&) override { assert(false); }; @@ -187,24 +184,6 @@ public: std::ostream& operator<<(std::ostream& os, const result_message::prepared::cql& msg); -class result_message::prepared::thrift : public result_message::prepared { - int32_t _id; -public: - thrift(int32_t id, cql3::statements::prepared_statement::checked_weak_ptr prepared, bool support_lwt_opt) - : result_message::prepared(std::move(prepared), support_lwt_opt) - , _id{id} - { } - - int32_t get_id() const { - return _id; - } - - virtual void accept(result_message::visitor& v) const override { - v.visit(*this); - } -}; - -std::ostream& operator<<(std::ostream& os, const result_message::prepared::thrift& msg); class result_message::schema_change : public result_message { private: diff --git a/transport/server.cc b/transport/server.cc index 1733f120a0..c388f4feaa 100644 --- a/transport/server.cc +++ b/transport/server.cc @@ -616,15 +616,14 @@ void cql_server::connection::on_connection_close() _server._notifier->unregister_connection(this); } -std::tuple cql_server::connection::make_client_key(const service::client_state& cli_state) { - return std::make_tuple(cli_state.get_client_address().addr(), - cli_state.get_client_port(), - cli_state.is_thrift() ? client_type::thrift : client_type::cql); +std::pair cql_server::connection::make_client_key(const service::client_state& cli_state) { + return {cli_state.get_client_address().addr(), + cli_state.get_client_port()}; } client_data cql_server::connection::make_client_data() const { client_data cd; - std::tie(cd.ip, cd.port, cd.ct) = make_client_key(_client_state); + std::tie(cd.ip, cd.port) = make_client_key(_client_state); cd.shard_id = this_shard_id(); cd.protocol_version = _version; cd.driver_name = _client_state.get_driver_name(); @@ -1064,14 +1063,14 @@ future> cql_server::connection::process_pr return parallel_for_each(cpus.begin(), cpus.end(), [this, query, cpu_id, &client_state] (unsigned int c) mutable { if (c != cpu_id) { return smp::submit_to(c, [this, query, &client_state] () mutable { - return _server._query_processor.local().prepare(std::move(query), client_state, false).discard_result(); + return _server._query_processor.local().prepare(std::move(query), client_state).discard_result(); }); } else { return make_ready_future<>(); } }).then([this, query, stream, &client_state, trace_state] () mutable { tracing::trace(trace_state, "Done preparing on remote shards"); - return _server._query_processor.local().prepare(std::move(query), client_state, false).then([this, stream, trace_state] (auto msg) { + return _server._query_processor.local().prepare(std::move(query), client_state).then([this, stream, trace_state] (auto msg) { tracing::trace(trace_state, "Done preparing on a local shard - preparing a result. ID is [{}]", seastar::value_of([&msg] { return messages::result_message::prepared::cql::get_id(msg); })); diff --git a/transport/server.hh b/transport/server.hh index ec58e13b40..c3a76f04a6 100644 --- a/transport/server.hh +++ b/transport/server.hh @@ -223,7 +223,7 @@ private: future<> process_request() override; void handle_error(future<>&& f) override; void on_connection_close() override; - static std::tuple make_client_key(const service::client_state& cli_state); + static std::pair make_client_key(const service::client_state& cli_state); client_data make_client_data() const; const service::client_state& get_client_state() const { return _client_state; } private: diff --git a/unimplemented.cc b/unimplemented.cc index 583a063108..abfbc0792a 100644 --- a/unimplemented.cc +++ b/unimplemented.cc @@ -35,7 +35,6 @@ std::string_view format_as(cause c) { case cause::LEGACY_COMPOSITE_KEYS: return "LEGACY_COMPOSITE_KEYS"; case cause::COLLECTION_RANGE_TOMBSTONES: return "COLLECTION_RANGE_TOMBSTONES"; case cause::RANGE_DELETES: return "RANGE_DELETES"; - case cause::THRIFT: return "THRIFT"; case cause::VALIDATION: return "VALIDATION"; case cause::REVERSED: return "REVERSED"; case cause::COMPRESSION: return "COMPRESSION"; diff --git a/unimplemented.hh b/unimplemented.hh index 46a5c4b768..c2dfd7f3a4 100644 --- a/unimplemented.hh +++ b/unimplemented.hh @@ -30,7 +30,6 @@ enum class cause { LEGACY_COMPOSITE_KEYS, COLLECTION_RANGE_TOMBSTONES, RANGE_DELETES, - THRIFT, VALIDATION, REVERSED, COMPRESSION,