mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-04-26 03:20:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge "Read C* 2.1.8 sstables - and vice versa" from Glauber

Browse Source 
"With the present patchset, we are now able to
- start cassandra-2.1.8
- generate a keyspace and a table
- flush it
- exit it
- start scylla on that same directory

And it works, from the sstable point of view. Note that if we don't flush,
Cassandra-2.1.8 won't do that automatically (it seems to do for 2.2), and
things will be left in the commitlog. We are not replaying their commitlog,
so we won't see any data.

The reverse also works - starting scylla, creating a table, exiting it.
After starting cassandra-2.1.8 in the same directory, everything works fine.

There are still some minor issues left, but they are not showstoppers. I will
open individual issues for each of them."
This commit is contained in:
Avi Kivity
2015-08-07 17:54:29 +03:00
parent ea485fdb0b 92031be642
commit 33a03d4da4
10 changed files with 197 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
caching_options.hh Normal file
View File

@@ -0,0 +1,69 @@
/*
* Copyright (C) 2015 Cloudius Systems, Ltd.
*/
#pragma once
#include <core/sstring.hh>
#include <boost/lexical_cast.hpp>
#include "exceptions/exceptions.hh"
#include "json.hh"
class schema;
class caching_options {
// For Origin, the default value for the row is "NONE". However, since our
// row_cache will cache both keys and rows, we will default to ALL.
//
// FIXME: We don't yet make any changes to our caching policies based on
// this (and maybe we shouldn't)
static constexpr auto default_key = "ALL";
static constexpr auto default_row = "ALL";
sstring _key_cache;
sstring _row_cache;
caching_options(sstring k, sstring r) : _key_cache(k), _row_cache(r) {
if ((k != "ALL") && (k != "NONE")) {
throw exceptions::configuration_exception("Invalid key value: " + k);
}
try {
boost::lexical_cast<unsigned long>(r);
} catch (boost::bad_lexical_cast& e) {
if ((r != "ALL") && (r != "NONE")) {
throw exceptions::configuration_exception("Invalid key value: " + k);
}
}
}
friend class schema;
caching_options() : _key_cache(default_key), _row_cache(default_row) {}
public:
sstring to_sstring() const {
return json::to_json(std::map<sstring, sstring>({{ "keys", _key_cache }, { "rows_per_partition", _row_cache }}));
}
static caching_options from_sstring(const sstring& str) {
auto map = json::to_map(str);
if (map.size() > 2) {
throw exceptions::configuration_exception("Invalid map: " + str);
}
sstring k;
sstring r;
if (map.count("keys")) {
k = map.at("keys");
} else {
k = default_key;
}
if (map.count("rows_per_partition")) {
r = map.at("rows_per_partition");
} else {
r = default_row;
}
return caching_options(k, r);
}
};

31
db/schema_tables.cc
View File

@@ -117,10 +117,11 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
{"speculative_retry", utf8_type},
{"subcomparator", utf8_type},
{"type", utf8_type},
// The following 3 columns are only present up until 2.1.8 tables
// The following 4 columns are only present up until 2.1.8 tables
{"key_aliases", utf8_type},
{"value_alias", utf8_type},
{"column_aliases", utf8_type},
{"index_interval", int32_type},
},
// static columns
{},
@@ -130,7 +131,7 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
"table definitions"
)));
builder.set_gc_grace_seconds(std::chrono::duration_cast<std::chrono::seconds>(days(7)).count());
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return columnfamilies;
}
@@ -159,7 +160,7 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
"column definitions"
)));
builder.set_gc_grace_seconds(std::chrono::duration_cast<std::chrono::seconds>(days(7)).count());
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return columns;
}
@@ -183,7 +184,7 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
"trigger definitions"
)));
builder.set_gc_grace_seconds(std::chrono::duration_cast<std::chrono::seconds>(days(7)).count());
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return triggers;
}
@@ -208,7 +209,7 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
"user defined type definitions"
)));
builder.set_gc_grace_seconds(std::chrono::duration_cast<std::chrono::seconds>(days(7)).count());
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return usertypes;
}
@@ -237,7 +238,7 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
"user defined type definitions"
)));
builder.set_gc_grace_seconds(std::chrono::duration_cast<std::chrono::seconds>(days(7)).count());
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return functions;
}
@@ -266,7 +267,7 @@ using days = std::chrono::duration<int, std::ratio<24 * 3600>>;
"user defined aggregate definitions"
)));
builder.set_gc_grace_seconds(std::chrono::duration_cast<std::chrono::seconds>(days(7)).count());
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return aggregates;
}
@@ -1060,9 +1061,7 @@ future<> save_system_keyspace_schema() {
}
m.set_clustered_cell(ckey, "bloom_filter_fp_chance", table->bloom_filter_fp_chance(), timestamp);
#if 0
adder.add("caching", table.getCaching().toString());
#endif
m.set_clustered_cell(ckey, "caching", table->caching_options().to_sstring(), timestamp);
m.set_clustered_cell(ckey, "comment", table->comment(), timestamp);
m.set_clustered_cell(ckey, "compaction_strategy_class", sstables::compaction_strategy::name(table->compaction_strategy()), timestamp);
@@ -1350,9 +1349,11 @@ future<> save_system_keyspace_schema() {
if (table_row.has("memtable_flush_period_in_ms")) {
builder.set_memtable_flush_period(table_row.get_nonnull<int32_t>("memtable_flush_period_in_ms"));
}
#if 0
cfm.caching(CachingOptions.fromString(result.getString("caching")));
#endif
if (table_row.has("caching")) {
builder.set_caching_options(caching_options::from_sstring(table_row.get_nonnull<sstring>("caching")));
}
if (table_row.has("default_time_to_live")) {
builder.set_default_time_to_live(gc_clock::duration(table_row.get_nonnull<gc_clock::rep>("default_time_to_live")));
}
@@ -1376,6 +1377,8 @@ future<> save_system_keyspace_schema() {
if (table_row.has("min_index_interval")) {
builder.set_min_index_interval(table_row.get_nonnull<int>("min_index_interval"));
} else if (table_row.has("index_interval")) { // compatibility
builder.set_min_index_interval(table_row.get_nonnull<int>("index_interval"));
}
if (table_row.has("max_index_interval")) {
@@ -1423,7 +1426,7 @@ future<> save_system_keyspace_schema() {
m.set_clustered_cell(ckey, "validator", column.type->name(), timestamp);
m.set_clustered_cell(ckey, "type", serialize_kind(column.kind), timestamp);
if (!column.is_on_all_components()) {
m.set_clustered_cell(ckey, "component_index", int32_t(column.position()), timestamp);
m.set_clustered_cell(ckey, "component_index", int32_t(table->position(column)), timestamp);
}
#if 0
adder.add("index_name", column.getIndexName());

32
db/system_keyspace.cc
View File

@@ -105,7 +105,7 @@ schema_ptr batchlog() {
// .compactionStrategyOptions(Collections.singletonMap("min_threshold", "2"))
)));
builder.set_gc_grace_seconds(0);
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return batchlog;
}
@@ -129,7 +129,7 @@ schema_ptr batchlog() {
// operations on resulting CFMetaData:
// .compactionStrategyClass(LeveledCompactionStrategy.class);
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return paxos;
}
@@ -176,8 +176,8 @@ schema_ptr built_indexes() {
{"release_version", utf8_type},
{"schema_version", uuid_type},
{"thrift_version", utf8_type},
{"tokens", set_type_impl::get_instance(utf8_type, false)},
{"truncated_at", map_type_impl::get_instance(uuid_type, bytes_type, false)},
{"tokens", set_type_impl::get_instance(utf8_type, true)},
{"truncated_at", map_type_impl::get_instance(uuid_type, bytes_type, true)},
// The following 3 columns are only present up until 2.1.8 tables
{"rpc_address", inet_addr_type},
{"broadcast_address", inet_addr_type},
@@ -191,7 +191,7 @@ schema_ptr built_indexes() {
// comment
"information about the local node"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return local;
}
@@ -212,7 +212,7 @@ schema_ptr built_indexes() {
{"release_version", utf8_type},
{"rpc_address", inet_addr_type},
{"schema_version", uuid_type},
{"tokens", set_type_impl::get_instance(utf8_type, false)},
{"tokens", set_type_impl::get_instance(utf8_type, true)},
},
// static columns
{},
@@ -221,7 +221,7 @@ schema_ptr built_indexes() {
// comment
"information about known peers in the cluster"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return peers;
}
@@ -235,7 +235,7 @@ schema_ptr built_indexes() {
{},
// regular columns
{
{"hints_dropped", map_type_impl::get_instance(uuid_type, int32_type, false)},
{"hints_dropped", map_type_impl::get_instance(uuid_type, int32_type, true)},
},
// static columns
{},
@@ -244,7 +244,7 @@ schema_ptr built_indexes() {
// comment
"events related to peers"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return peer_events;
}
@@ -265,7 +265,7 @@ schema_ptr built_indexes() {
// comment
"ranges requested for transfer"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return range_xfers;
}
@@ -280,7 +280,7 @@ schema_ptr built_indexes() {
// regular columns
{
{"columnfamily_name", utf8_type},
{"inputs", set_type_impl::get_instance(int32_type, false)},
{"inputs", set_type_impl::get_instance(int32_type, true)},
{"keyspace_name", utf8_type},
},
// static columns
@@ -290,7 +290,7 @@ schema_ptr built_indexes() {
// comment
"unfinished compactions"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return compactions_in_progress;
}
@@ -309,7 +309,7 @@ schema_ptr built_indexes() {
{"columnfamily_name", utf8_type},
{"compacted_at", timestamp_type},
{"keyspace_name", utf8_type},
{"rows_merged", map_type_impl::get_instance(int32_type, long_type, false)},
{"rows_merged", map_type_impl::get_instance(int32_type, long_type, true)},
},
// static columns
{},
@@ -319,7 +319,7 @@ schema_ptr built_indexes() {
"week-long compaction history"
)));
builder.set_default_time_to_live(std::chrono::duration_cast<std::chrono::seconds>(days(7)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return compaction_history;
}
@@ -347,7 +347,7 @@ schema_ptr built_indexes() {
// comment
"historic sstable read rates"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return sstable_activity;
}
@@ -372,7 +372,7 @@ schema_ptr size_estimates() {
"per-table primary range size estimates"
)));
builder.set_gc_grace_seconds(0);
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return size_estimates;
}

25
schema.cc
View File

@@ -8,6 +8,7 @@
#include "schema_builder.hh"
#include <boost/algorithm/cxx11/any_of.hpp>
#include <boost/range/adaptor/transformed.hpp>
#include "version.hh"
constexpr int32_t schema::NAME_LENGTH;
@@ -76,6 +77,8 @@ schema::schema(const raw_schema& raw)
}
};
thrift()._compound = is_compound();
std::sort(
_raw._columns.begin() + column_offset(column_kind::static_column),
_raw._columns.begin()
@@ -111,6 +114,10 @@ schema::schema(const raw_schema& raw)
// Which is true of #pk == 1
def._thrift_bits.is_on_all_components = partition_key_size() == 1;
break;
case column_kind::compact_column:
// compact values are alone, so they have no index
def._thrift_bits.is_on_all_components = true;
break;
default:
// Or any other column where "comparator" is not compound
def._thrift_bits.is_on_all_components = !thrift().has_compound_comparator();
@@ -334,6 +341,10 @@ schema_builder& schema_builder::with_column(bytes name, data_type type, index_in
return *this;
}
schema_ptr schema_builder::build() {
return make_lw_shared<schema>(schema(_raw));
}
schema_ptr schema_builder::build(compact_storage cp) {
schema s(_raw);
@@ -354,7 +365,7 @@ schema_ptr schema_builder::build(compact_storage cp) {
s._raw._is_compound = true;
}
}
s._thrift._compound = s._raw._is_compound;
if (s._raw._is_dense) {
// In Origin, dense CFs always have at least one regular column
if (s.regular_columns_count() == 0) {
@@ -368,10 +379,8 @@ schema_ptr schema_builder::build(compact_storage cp) {
// We need to rebuild the schema in case we added some column. This is way simpler than trying to factor out the relevant code
// from the constructor
return make_lw_shared<schema>(schema(s._raw));
} else {
return make_lw_shared<schema>(s);
}
return make_lw_shared<schema>(schema(s._raw));
}
// Useful functions to manipulate the schema's comparator field
@@ -380,6 +389,11 @@ namespace cell_comparator {
static constexpr auto _composite_str = "org.apache.cassandra.db.marshal.CompositeType";
static constexpr auto _collection_str = "org.apache.cassandra.db.marshal.ColumnToCollectionType";
static bool always_include_default() {
static thread_local bool def = version::version::current() < version::version(2, 2);
return def;
};
static sstring collection_name(const collection_type& t) {
sstring collection_str(_collection_str);
collection_str += "(00000000:" + t->name() + ")";
@@ -394,7 +408,8 @@ static sstring compound_name(const schema& s) {
for (auto &t : s.clustering_key_columns()) {
compound += t.type->name() + ",";
}
} else {
}
if (always_include_default() || (s.clustering_key_size() == 0)) {
compound += s.regular_column_name_type()->name() + ",";
}

8
schema.hh
View File

@@ -19,6 +19,7 @@
#include "utils/UUID.hh"
#include "compress.hh"
#include "compaction_strategy.hh"
#include "caching_options.hh"
using column_id = uint32_t;
@@ -209,7 +210,7 @@ class thrift_schema {
bool _compound = true;
public:
bool has_compound_comparator() const;
friend class schema_builder;
friend class schema;
};
bool operator==(const column_definition&, const column_definition&);
@@ -255,6 +256,7 @@ private:
// we will use by default - when we have the choice.
sstables::compaction_strategy_type _compaction_strategy = sstables::compaction_strategy_type::size_tiered;
std::map<sstring, sstring> _compaction_strategy_options;
caching_options _caching_options;
};
raw_schema _raw;
thrift_schema _thrift;
@@ -386,6 +388,10 @@ public:
return _raw._speculative_retry;
}
const ::caching_options& caching_options() const {
return _raw._caching_options;
}
const column_definition* get_column_definition(const bytes& name) const;
const_iterator regular_begin() const {
return regular_columns().begin();

7
schema_builder.hh
View File

@@ -142,6 +142,10 @@ public:
_raw._compaction_strategy_options = std::move(options);
}
void set_caching_options(caching_options c) {
_raw._caching_options = std::move(c);
}
void set_is_dense(bool is_dense) {
_raw._is_dense = is_dense;
}
@@ -159,5 +163,6 @@ public:
void add_default_index_names(database&);
enum class compact_storage { no, yes };
schema_ptr build(compact_storage = compact_storage::no);
schema_ptr build(compact_storage cp);
schema_ptr build();
};

2
tests/cql_test_env.cc
View File

@@ -96,7 +96,7 @@ public:
return _db->invoke_on_all([schema_maker, id, this] (database& db) {
schema_builder builder(make_lw_shared(schema_maker(ks_name)));
builder.set_uuid(id);
auto cf_schema = builder.build();
auto cf_schema = builder.build(schema_builder::compact_storage::no);
auto& ks = db.find_keyspace(ks_name);
auto cfg = ks.make_column_family_config(*cf_schema);
db.add_column_family(std::move(cf_schema), std::move(cfg));

2
tests/sstable_datafile_test.cc
View File

@@ -900,7 +900,7 @@ static future<> sstable_compression_test(compressor c, unsigned generation) {
// NOTE: set a given compressor algorithm to schema.
schema_builder builder(complex_schema());
builder.set_compressor_params(c);
auto s = builder.build();
auto s = builder.build(schema_builder::compact_storage::no);
auto mtp = make_lw_shared<memtable>(s);

14
tests/sstable_test.hh
View File

@@ -103,7 +103,7 @@ inline schema_ptr composite_schema() {
// comment
"Table with a composite key as pkey"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return s;
}
@@ -127,7 +127,7 @@ inline schema_ptr set_schema() {
// comment
"Table with a set as pkeys"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return s;
}
@@ -151,7 +151,7 @@ inline schema_ptr map_schema() {
// comment
"Table with a map as pkeys"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return s;
}
@@ -175,7 +175,7 @@ inline schema_ptr list_schema() {
// comment
"Table with a list as pkeys"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return s;
}
@@ -196,7 +196,7 @@ inline schema_ptr uncompressed_schema() {
// comment
"Uncompressed data"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return uncompressed;
}
@@ -229,7 +229,7 @@ inline schema_ptr complex_schema() {
// comment
"Table with a complex schema, including collections and static keys"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return s;
}
@@ -257,7 +257,7 @@ inline schema_ptr columns_schema() {
// comment
"column definitions"
)));
return builder.build();
return builder.build(schema_builder::compact_storage::no);
}();
return columns;
}

55
version.hh
View File

@@ -1,14 +1,65 @@
#pragma once
#include "core/sstring.hh"
#include "core/print.hh"
namespace version {
class version {
uint8_t maj;
uint8_t min;
uint16_t rev;
public:
version(uint8_t x, uint8_t y = 0, uint16_t z = 0): maj(x), min(y), rev(z) {}
sstring to_sstring() {
return sprint("%d.%d.%d", maj, min, rev);
}
static version current() {
static version v(2, 1, 8);
return v;
}
bool operator==(version v) const {
return (maj == v.maj) && (min == v.min) && (rev == v.rev);
}
bool operator!=(version v) const {
return !(v == *this);
}
bool operator<(version v) const {
if (maj < v.maj) {
return true;
} else if (maj > v.maj) {
return false;
}
if (min < v.min) {
return true;
} else if (min > v.min) {
return false;
}
return rev < v.rev;
}
bool operator<=(version v) {
return ((*this < v) || (*this == v));
}
bool operator>(version v) {
return !(*this <= v);
}
bool operator>=(version v) {
return ((*this == v) || !(*this < v));
}
};
inline const int native_protocol() {
return 3;
}
inline const sstring& release() {
static sstring v = "2.1.8";
return v;
static thread_local auto str_ver = version::current().to_sstring();
return str_ver;
}
}