diff --git a/caching_options.hh b/caching_options.hh new file mode 100644 index 0000000000..c67e19d649 --- /dev/null +++ b/caching_options.hh @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2015 Cloudius Systems, Ltd. + */ + +#pragma once +#include +#include +#include "exceptions/exceptions.hh" +#include "json.hh" + +class schema; + +class caching_options { + // For Origin, the default value for the row is "NONE". However, since our + // row_cache will cache both keys and rows, we will default to ALL. + // + // FIXME: We don't yet make any changes to our caching policies based on + // this (and maybe we shouldn't) + static constexpr auto default_key = "ALL"; + static constexpr auto default_row = "ALL"; + + sstring _key_cache; + sstring _row_cache; + caching_options(sstring k, sstring r) : _key_cache(k), _row_cache(r) { + if ((k != "ALL") && (k != "NONE")) { + throw exceptions::configuration_exception("Invalid key value: " + k); + } + + try { + boost::lexical_cast(r); + } catch (boost::bad_lexical_cast& e) { + if ((r != "ALL") && (r != "NONE")) { + throw exceptions::configuration_exception("Invalid key value: " + k); + } + } + } + + friend class schema; + caching_options() : _key_cache(default_key), _row_cache(default_row) {} +public: + + sstring to_sstring() const { + return json::to_json(std::map({{ "keys", _key_cache }, { "rows_per_partition", _row_cache }})); + } + + static caching_options from_sstring(const sstring& str) { + auto map = json::to_map(str); + if (map.size() > 2) { + throw exceptions::configuration_exception("Invalid map: " + str); + } + sstring k; + sstring r; + if (map.count("keys")) { + k = map.at("keys"); + } else { + k = default_key; + } + + if (map.count("rows_per_partition")) { + r = map.at("rows_per_partition"); + } else { + r = default_row; + } + return caching_options(k, r); + } +}; + + + diff --git a/db/schema_tables.cc b/db/schema_tables.cc index 9b3e6360d8..8dc8d40278 100644 --- a/db/schema_tables.cc +++ b/db/schema_tables.cc @@ -117,10 +117,11 @@ using days = std::chrono::duration>; {"speculative_retry", utf8_type}, {"subcomparator", utf8_type}, {"type", utf8_type}, - // The following 3 columns are only present up until 2.1.8 tables + // The following 4 columns are only present up until 2.1.8 tables {"key_aliases", utf8_type}, {"value_alias", utf8_type}, {"column_aliases", utf8_type}, + {"index_interval", int32_type}, }, // static columns {}, @@ -130,7 +131,7 @@ using days = std::chrono::duration>; "table definitions" ))); builder.set_gc_grace_seconds(std::chrono::duration_cast(days(7)).count()); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return columnfamilies; } @@ -159,7 +160,7 @@ using days = std::chrono::duration>; "column definitions" ))); builder.set_gc_grace_seconds(std::chrono::duration_cast(days(7)).count()); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return columns; } @@ -183,7 +184,7 @@ using days = std::chrono::duration>; "trigger definitions" ))); builder.set_gc_grace_seconds(std::chrono::duration_cast(days(7)).count()); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return triggers; } @@ -208,7 +209,7 @@ using days = std::chrono::duration>; "user defined type definitions" ))); builder.set_gc_grace_seconds(std::chrono::duration_cast(days(7)).count()); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return usertypes; } @@ -237,7 +238,7 @@ using days = std::chrono::duration>; "user defined type definitions" ))); builder.set_gc_grace_seconds(std::chrono::duration_cast(days(7)).count()); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return functions; } @@ -266,7 +267,7 @@ using days = std::chrono::duration>; "user defined aggregate definitions" ))); builder.set_gc_grace_seconds(std::chrono::duration_cast(days(7)).count()); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return aggregates; } @@ -1060,9 +1061,7 @@ future<> save_system_keyspace_schema() { } m.set_clustered_cell(ckey, "bloom_filter_fp_chance", table->bloom_filter_fp_chance(), timestamp); -#if 0 - adder.add("caching", table.getCaching().toString()); -#endif + m.set_clustered_cell(ckey, "caching", table->caching_options().to_sstring(), timestamp); m.set_clustered_cell(ckey, "comment", table->comment(), timestamp); m.set_clustered_cell(ckey, "compaction_strategy_class", sstables::compaction_strategy::name(table->compaction_strategy()), timestamp); @@ -1350,9 +1349,11 @@ future<> save_system_keyspace_schema() { if (table_row.has("memtable_flush_period_in_ms")) { builder.set_memtable_flush_period(table_row.get_nonnull("memtable_flush_period_in_ms")); } -#if 0 - cfm.caching(CachingOptions.fromString(result.getString("caching"))); -#endif + + if (table_row.has("caching")) { + builder.set_caching_options(caching_options::from_sstring(table_row.get_nonnull("caching"))); + } + if (table_row.has("default_time_to_live")) { builder.set_default_time_to_live(gc_clock::duration(table_row.get_nonnull("default_time_to_live"))); } @@ -1376,6 +1377,8 @@ future<> save_system_keyspace_schema() { if (table_row.has("min_index_interval")) { builder.set_min_index_interval(table_row.get_nonnull("min_index_interval")); + } else if (table_row.has("index_interval")) { // compatibility + builder.set_min_index_interval(table_row.get_nonnull("index_interval")); } if (table_row.has("max_index_interval")) { @@ -1423,7 +1426,7 @@ future<> save_system_keyspace_schema() { m.set_clustered_cell(ckey, "validator", column.type->name(), timestamp); m.set_clustered_cell(ckey, "type", serialize_kind(column.kind), timestamp); if (!column.is_on_all_components()) { - m.set_clustered_cell(ckey, "component_index", int32_t(column.position()), timestamp); + m.set_clustered_cell(ckey, "component_index", int32_t(table->position(column)), timestamp); } #if 0 adder.add("index_name", column.getIndexName()); diff --git a/db/system_keyspace.cc b/db/system_keyspace.cc index cc91ce73ec..ba53acbd7d 100644 --- a/db/system_keyspace.cc +++ b/db/system_keyspace.cc @@ -105,7 +105,7 @@ schema_ptr batchlog() { // .compactionStrategyOptions(Collections.singletonMap("min_threshold", "2")) ))); builder.set_gc_grace_seconds(0); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return batchlog; } @@ -129,7 +129,7 @@ schema_ptr batchlog() { // operations on resulting CFMetaData: // .compactionStrategyClass(LeveledCompactionStrategy.class); ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return paxos; } @@ -176,8 +176,8 @@ schema_ptr built_indexes() { {"release_version", utf8_type}, {"schema_version", uuid_type}, {"thrift_version", utf8_type}, - {"tokens", set_type_impl::get_instance(utf8_type, false)}, - {"truncated_at", map_type_impl::get_instance(uuid_type, bytes_type, false)}, + {"tokens", set_type_impl::get_instance(utf8_type, true)}, + {"truncated_at", map_type_impl::get_instance(uuid_type, bytes_type, true)}, // The following 3 columns are only present up until 2.1.8 tables {"rpc_address", inet_addr_type}, {"broadcast_address", inet_addr_type}, @@ -191,7 +191,7 @@ schema_ptr built_indexes() { // comment "information about the local node" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return local; } @@ -212,7 +212,7 @@ schema_ptr built_indexes() { {"release_version", utf8_type}, {"rpc_address", inet_addr_type}, {"schema_version", uuid_type}, - {"tokens", set_type_impl::get_instance(utf8_type, false)}, + {"tokens", set_type_impl::get_instance(utf8_type, true)}, }, // static columns {}, @@ -221,7 +221,7 @@ schema_ptr built_indexes() { // comment "information about known peers in the cluster" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return peers; } @@ -235,7 +235,7 @@ schema_ptr built_indexes() { {}, // regular columns { - {"hints_dropped", map_type_impl::get_instance(uuid_type, int32_type, false)}, + {"hints_dropped", map_type_impl::get_instance(uuid_type, int32_type, true)}, }, // static columns {}, @@ -244,7 +244,7 @@ schema_ptr built_indexes() { // comment "events related to peers" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return peer_events; } @@ -265,7 +265,7 @@ schema_ptr built_indexes() { // comment "ranges requested for transfer" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return range_xfers; } @@ -280,7 +280,7 @@ schema_ptr built_indexes() { // regular columns { {"columnfamily_name", utf8_type}, - {"inputs", set_type_impl::get_instance(int32_type, false)}, + {"inputs", set_type_impl::get_instance(int32_type, true)}, {"keyspace_name", utf8_type}, }, // static columns @@ -290,7 +290,7 @@ schema_ptr built_indexes() { // comment "unfinished compactions" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return compactions_in_progress; } @@ -309,7 +309,7 @@ schema_ptr built_indexes() { {"columnfamily_name", utf8_type}, {"compacted_at", timestamp_type}, {"keyspace_name", utf8_type}, - {"rows_merged", map_type_impl::get_instance(int32_type, long_type, false)}, + {"rows_merged", map_type_impl::get_instance(int32_type, long_type, true)}, }, // static columns {}, @@ -319,7 +319,7 @@ schema_ptr built_indexes() { "week-long compaction history" ))); builder.set_default_time_to_live(std::chrono::duration_cast(days(7))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return compaction_history; } @@ -347,7 +347,7 @@ schema_ptr built_indexes() { // comment "historic sstable read rates" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return sstable_activity; } @@ -372,7 +372,7 @@ schema_ptr size_estimates() { "per-table primary range size estimates" ))); builder.set_gc_grace_seconds(0); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return size_estimates; } diff --git a/schema.cc b/schema.cc index e9ef126e55..875c03bf3a 100644 --- a/schema.cc +++ b/schema.cc @@ -8,6 +8,7 @@ #include "schema_builder.hh" #include #include +#include "version.hh" constexpr int32_t schema::NAME_LENGTH; @@ -76,6 +77,8 @@ schema::schema(const raw_schema& raw) } }; + thrift()._compound = is_compound(); + std::sort( _raw._columns.begin() + column_offset(column_kind::static_column), _raw._columns.begin() @@ -111,6 +114,10 @@ schema::schema(const raw_schema& raw) // Which is true of #pk == 1 def._thrift_bits.is_on_all_components = partition_key_size() == 1; break; + case column_kind::compact_column: + // compact values are alone, so they have no index + def._thrift_bits.is_on_all_components = true; + break; default: // Or any other column where "comparator" is not compound def._thrift_bits.is_on_all_components = !thrift().has_compound_comparator(); @@ -334,6 +341,10 @@ schema_builder& schema_builder::with_column(bytes name, data_type type, index_in return *this; } +schema_ptr schema_builder::build() { + return make_lw_shared(schema(_raw)); +} + schema_ptr schema_builder::build(compact_storage cp) { schema s(_raw); @@ -354,7 +365,7 @@ schema_ptr schema_builder::build(compact_storage cp) { s._raw._is_compound = true; } } - s._thrift._compound = s._raw._is_compound; + if (s._raw._is_dense) { // In Origin, dense CFs always have at least one regular column if (s.regular_columns_count() == 0) { @@ -368,10 +379,8 @@ schema_ptr schema_builder::build(compact_storage cp) { // We need to rebuild the schema in case we added some column. This is way simpler than trying to factor out the relevant code // from the constructor - return make_lw_shared(schema(s._raw)); - } else { - return make_lw_shared(s); } + return make_lw_shared(schema(s._raw)); } // Useful functions to manipulate the schema's comparator field @@ -380,6 +389,11 @@ namespace cell_comparator { static constexpr auto _composite_str = "org.apache.cassandra.db.marshal.CompositeType"; static constexpr auto _collection_str = "org.apache.cassandra.db.marshal.ColumnToCollectionType"; +static bool always_include_default() { + static thread_local bool def = version::version::current() < version::version(2, 2); + return def; +}; + static sstring collection_name(const collection_type& t) { sstring collection_str(_collection_str); collection_str += "(00000000:" + t->name() + ")"; @@ -394,7 +408,8 @@ static sstring compound_name(const schema& s) { for (auto &t : s.clustering_key_columns()) { compound += t.type->name() + ","; } - } else { + } + if (always_include_default() || (s.clustering_key_size() == 0)) { compound += s.regular_column_name_type()->name() + ","; } diff --git a/schema.hh b/schema.hh index 5b7bdda1c5..f5c9c2712b 100644 --- a/schema.hh +++ b/schema.hh @@ -19,6 +19,7 @@ #include "utils/UUID.hh" #include "compress.hh" #include "compaction_strategy.hh" +#include "caching_options.hh" using column_id = uint32_t; @@ -209,7 +210,7 @@ class thrift_schema { bool _compound = true; public: bool has_compound_comparator() const; - friend class schema_builder; + friend class schema; }; bool operator==(const column_definition&, const column_definition&); @@ -255,6 +256,7 @@ private: // we will use by default - when we have the choice. sstables::compaction_strategy_type _compaction_strategy = sstables::compaction_strategy_type::size_tiered; std::map _compaction_strategy_options; + caching_options _caching_options; }; raw_schema _raw; thrift_schema _thrift; @@ -386,6 +388,10 @@ public: return _raw._speculative_retry; } + const ::caching_options& caching_options() const { + return _raw._caching_options; + } + const column_definition* get_column_definition(const bytes& name) const; const_iterator regular_begin() const { return regular_columns().begin(); diff --git a/schema_builder.hh b/schema_builder.hh index 05b4ada3c9..47e2d00949 100644 --- a/schema_builder.hh +++ b/schema_builder.hh @@ -142,6 +142,10 @@ public: _raw._compaction_strategy_options = std::move(options); } + void set_caching_options(caching_options c) { + _raw._caching_options = std::move(c); + } + void set_is_dense(bool is_dense) { _raw._is_dense = is_dense; } @@ -159,5 +163,6 @@ public: void add_default_index_names(database&); enum class compact_storage { no, yes }; - schema_ptr build(compact_storage = compact_storage::no); + schema_ptr build(compact_storage cp); + schema_ptr build(); }; diff --git a/tests/cql_test_env.cc b/tests/cql_test_env.cc index 600956716a..6cc527c63a 100644 --- a/tests/cql_test_env.cc +++ b/tests/cql_test_env.cc @@ -96,7 +96,7 @@ public: return _db->invoke_on_all([schema_maker, id, this] (database& db) { schema_builder builder(make_lw_shared(schema_maker(ks_name))); builder.set_uuid(id); - auto cf_schema = builder.build(); + auto cf_schema = builder.build(schema_builder::compact_storage::no); auto& ks = db.find_keyspace(ks_name); auto cfg = ks.make_column_family_config(*cf_schema); db.add_column_family(std::move(cf_schema), std::move(cfg)); diff --git a/tests/sstable_datafile_test.cc b/tests/sstable_datafile_test.cc index 8ee698c338..77dc602082 100644 --- a/tests/sstable_datafile_test.cc +++ b/tests/sstable_datafile_test.cc @@ -900,7 +900,7 @@ static future<> sstable_compression_test(compressor c, unsigned generation) { // NOTE: set a given compressor algorithm to schema. schema_builder builder(complex_schema()); builder.set_compressor_params(c); - auto s = builder.build(); + auto s = builder.build(schema_builder::compact_storage::no); auto mtp = make_lw_shared(s); diff --git a/tests/sstable_test.hh b/tests/sstable_test.hh index 5839f547ad..2910fd5406 100644 --- a/tests/sstable_test.hh +++ b/tests/sstable_test.hh @@ -103,7 +103,7 @@ inline schema_ptr composite_schema() { // comment "Table with a composite key as pkey" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return s; } @@ -127,7 +127,7 @@ inline schema_ptr set_schema() { // comment "Table with a set as pkeys" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return s; } @@ -151,7 +151,7 @@ inline schema_ptr map_schema() { // comment "Table with a map as pkeys" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return s; } @@ -175,7 +175,7 @@ inline schema_ptr list_schema() { // comment "Table with a list as pkeys" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return s; } @@ -196,7 +196,7 @@ inline schema_ptr uncompressed_schema() { // comment "Uncompressed data" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return uncompressed; } @@ -229,7 +229,7 @@ inline schema_ptr complex_schema() { // comment "Table with a complex schema, including collections and static keys" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return s; } @@ -257,7 +257,7 @@ inline schema_ptr columns_schema() { // comment "column definitions" ))); - return builder.build(); + return builder.build(schema_builder::compact_storage::no); }(); return columns; } diff --git a/version.hh b/version.hh index 64f7490fb3..30f00632c7 100644 --- a/version.hh +++ b/version.hh @@ -1,14 +1,65 @@ #pragma once #include "core/sstring.hh" +#include "core/print.hh" namespace version { +class version { + uint8_t maj; + uint8_t min; + uint16_t rev; + +public: + version(uint8_t x, uint8_t y = 0, uint16_t z = 0): maj(x), min(y), rev(z) {} + + sstring to_sstring() { + return sprint("%d.%d.%d", maj, min, rev); + } + + static version current() { + static version v(2, 1, 8); + return v; + } + + bool operator==(version v) const { + return (maj == v.maj) && (min == v.min) && (rev == v.rev); + } + + bool operator!=(version v) const { + return !(v == *this); + } + + bool operator<(version v) const { + if (maj < v.maj) { + return true; + } else if (maj > v.maj) { + return false; + } + + if (min < v.min) { + return true; + } else if (min > v.min) { + return false; + } + return rev < v.rev; + } + bool operator<=(version v) { + return ((*this < v) || (*this == v)); + } + bool operator>(version v) { + return !(*this <= v); + } + bool operator>=(version v) { + return ((*this == v) || !(*this < v)); + } +}; + inline const int native_protocol() { return 3; } inline const sstring& release() { - static sstring v = "2.1.8"; - return v; + static thread_local auto str_ver = version::current().to_sstring(); + return str_ver; } }