diff --git a/configure.py b/configure.py index bfcd591882..e5ae7c464f 100755 --- a/configure.py +++ b/configure.py @@ -396,6 +396,7 @@ scylla_core = (['database.cc', 'counters.cc', 'compress.cc', 'sstables/sstables.cc', + 'sstables/sstable_version.cc', 'sstables/compress.cc', 'sstables/row.cc', 'sstables/partition.cc', diff --git a/sstables/sstable_version.cc b/sstables/sstable_version.cc new file mode 100644 index 0000000000..951515b423 --- /dev/null +++ b/sstables/sstable_version.cc @@ -0,0 +1,79 @@ +/* + * Copyright (C) 2018 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#include "sstable_version.hh" +#include "sstable_version_k_l.hh" +#include "sstable_version_m.hh" + +namespace sstables { + +const sstring sstable_version_constants::TOC_SUFFIX = "TOC.txt"; +const sstring sstable_version_constants::TEMPORARY_TOC_SUFFIX = "TOC.txt.tmp"; + +sstable_version_constants::component_map_t sstable_version_constants::create_component_map() { + return { + { component_type::Index, "Index.db"}, + { component_type::CompressionInfo, "CompressionInfo.db" }, + { component_type::Data, "Data.db" }, + { component_type::TOC, TOC_SUFFIX }, + { component_type::Summary, "Summary.db" }, + { component_type::CRC, "CRC.db" }, + { component_type::Filter, "Filter.db" }, + { component_type::Statistics, "Statistics.db" }, + { component_type::Scylla, "Scylla.db" }, + { component_type::TemporaryTOC, TEMPORARY_TOC_SUFFIX }, + { component_type::TemporaryStatistics, "Statistics.db.tmp" }, + }; +} + +const sstable_version_constants::component_map_t& +sstable_version_constants::get_component_map(sstable_version_types version) { + switch (version) { + case sstable_version_types::ka: + case sstable_version_types::la: + return sstable_version_constants_k_l::_component_map; + case sstable_version_types::mc: + return sstable_version_constants_m::_component_map; + } + // Should never reach this. + // Compiler should complain if the switch above does no cover all sstable_version_types values. + throw std::invalid_argument("Invalid sstable format version"); +} + +const sstable_version_constants::component_map_t sstable_version_constants_k_l::create_component_map() { + auto result = sstable_version_constants::create_component_map(); + result.emplace(component_type::Digest, "Digest.sha1"); + return result; +} + +const sstable_version_constants::component_map_t sstable_version_constants_k_l::_component_map = + sstable_version_constants_k_l::create_component_map(); + +const sstable_version_constants::component_map_t sstable_version_constants_m::create_component_map() { + auto result = sstable_version_constants::create_component_map(); + result.emplace(component_type::Digest, "Digest.crc32"); + return result; +} + +const sstable_version_constants::component_map_t sstable_version_constants_m::_component_map = + sstable_version_constants_m::create_component_map(); + +} diff --git a/sstables/sstable_version.hh b/sstables/sstable_version.hh new file mode 100644 index 0000000000..43fa32fa76 --- /dev/null +++ b/sstables/sstable_version.hh @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include "types.hh" +#include "version.hh" +#include "component_type.hh" + +namespace sstables { + +class sstable_version_constants { +public: + using component_map_t = std::unordered_map>; + static const sstring TOC_SUFFIX; + static const sstring TEMPORARY_TOC_SUFFIX; + static const component_map_t& get_component_map(sstable_version_types); + sstable_version_constants() = delete; +protected: + static component_map_t create_component_map(); +}; + +} diff --git a/sstables/sstable_version_k_l.hh b/sstables/sstable_version_k_l.hh new file mode 100644 index 0000000000..9901f365d4 --- /dev/null +++ b/sstables/sstable_version_k_l.hh @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include "sstable_version.hh" +#include "types.hh" + +namespace sstables { + +class sstable_version_constants_k_l final : public sstable_version_constants { + static const sstable_version_constants::component_map_t create_component_map(); +public: + sstable_version_constants_k_l() = delete; + static const sstable_version_constants::component_map_t _component_map; +}; + +} diff --git a/sstables/sstable_version_m.hh b/sstables/sstable_version_m.hh new file mode 100644 index 0000000000..434d0cd65b --- /dev/null +++ b/sstables/sstable_version_m.hh @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2018 ScyllaDB + * + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include "sstable_version.hh" +#include "types.hh" + +namespace sstables { + +class sstable_version_constants_m final : public sstable_version_constants { + static const sstable_version_constants::component_map_t create_component_map(); +public: + sstable_version_constants_m() = delete; + static const sstable_version_constants::component_map_t _component_map; +}; + +} diff --git a/sstables/sstables.cc b/sstables/sstables.cc index 1db0a4fd36..c0dac7b922 100644 --- a/sstables/sstables.cc +++ b/sstables/sstables.cc @@ -198,25 +198,6 @@ std::unordered_map> sstable::_component_map = { - { component_type::Index, "Index.db"}, - { component_type::CompressionInfo, "CompressionInfo.db" }, - { component_type::Data, "Data.db" }, - { component_type::TOC, TOC_SUFFIX }, - { component_type::Summary, "Summary.db" }, - { component_type::Digest, "Digest.sha1" }, - { component_type::CRC, "CRC.db" }, - { component_type::Filter, "Filter.db" }, - { component_type::Statistics, "Statistics.db" }, - { component_type::Scylla, "Scylla.db" }, - { component_type::TemporaryTOC, TEMPORARY_TOC_SUFFIX }, - { component_type::TemporaryStatistics, "Statistics.db.tmp" }, -}; - // This assumes that the mappings are small enough, and called unfrequent // enough. If that changes, it would be adviseable to create a full static // reverse mapping, even if it is done at runtime. @@ -984,7 +965,7 @@ future<> sstable::read_toc() { continue; } try { - _recognized_components.insert(reverse_map(c, _component_map)); + _recognized_components.insert(reverse_map(c, sstable_version_constants::get_component_map(_version))); } catch (std::out_of_range& oor) { _unrecognized_components.push_back(c); sstlog.info("Unrecognized TOC component was found: {} in sstable {}", c, file_path); @@ -1054,7 +1035,7 @@ void sstable::write_toc(const io_priority_class& pc) { for (auto&& key : _recognized_components) { // new line character is appended to the end of each component name. - auto value = _component_map[key] + "\n"; + auto value = sstable_version_constants::get_component_map(_version).at(key) + "\n"; bytes b = bytes(reinterpret_cast(value.c_str()), value.size()); write(w, b); } @@ -1129,7 +1110,7 @@ template future<> sstable::read_simple(T& component, const io_priority_class& pc) { auto file_path = filename(Type); - sstlog.debug(("Reading " + _component_map[Type] + " file {} ").c_str(), file_path); + sstlog.debug(("Reading " + sstable_version_constants::get_component_map(_version).at(Type) + " file {} ").c_str(), file_path); return open_file_dma(file_path, open_flags::ro).then([this, &component] (file fi) { auto fut = fi.size(); return fut.then([this, &component, fi = std::move(fi)] (uint64_t size) { @@ -1155,7 +1136,7 @@ future<> sstable::read_simple(T& component, const io_priority_class& pc) { template void sstable::write_simple(const T& component, const io_priority_class& pc) { auto file_path = filename(Type); - sstlog.debug(("Writing " + _component_map[Type] + " file {} ").c_str(), file_path); + sstlog.debug(("Writing " + sstable_version_constants::get_component_map(_version).at(Type) + " file {} ").c_str(), file_path); file f = new_sstable_component_file(_write_error_handler, file_path, open_flags::wo | open_flags::create | open_flags::exclusive).get0(); file_output_stream_options options; @@ -2556,7 +2537,7 @@ const sstring sstable::filename(component_type f) const { std::vector sstable::component_filenames() const { std::vector res; - for (auto c : _component_map | boost::adaptors::map_keys) { + for (auto c : sstable_version_constants::get_component_map(_version) | boost::adaptors::map_keys) { if (has_component(c)) { res.emplace_back(filename(c)); } @@ -2573,13 +2554,16 @@ const sstring sstable::filename(sstring dir, sstring ks, sstring cf, version_typ static std::unordered_map, enum_hash> strmap = { { sstable::version_types::ka, [] (entry_descriptor d) { - return d.ks + "-" + d.cf + "-" + _version_string.at(d.version) + "-" + to_sstring(d.generation) + "-" + _component_map.at(d.component); } + return d.ks + "-" + d.cf + "-" + _version_string.at(d.version) + "-" + to_sstring(d.generation) + "-" + + sstable_version_constants::get_component_map(d.version).at(d.component); } }, { sstable::version_types::la, [] (entry_descriptor d) { - return _version_string.at(d.version) + "-" + to_sstring(d.generation) + "-" + _format_string.at(d.format) + "-" + _component_map.at(d.component); } + return _version_string.at(d.version) + "-" + to_sstring(d.generation) + "-" + _format_string.at(d.format) + "-" + + sstable_version_constants::get_component_map(d.version).at(d.component); } }, { sstable::version_types::mc, [] (entry_descriptor d) { - return _version_string.at(d.version) + "-" + to_sstring(d.generation) + "-" + _format_string.at(d.format) + "-" + _component_map.at(d.component); } + return _version_string.at(d.version) + "-" + to_sstring(d.generation) + "-" + _format_string.at(d.format) + "-" + + sstable_version_constants::get_component_map(d.version).at(d.component); } }, }; @@ -2600,7 +2584,7 @@ std::vector> sstable::all_components() const std::vector> all; all.reserve(_recognized_components.size() + _unrecognized_components.size()); for (auto& c : _recognized_components) { - all.push_back(std::make_pair(c, _component_map.at(c))); + all.push_back(std::make_pair(c, sstable_version_constants::get_component_map(_version).at(c))); } for (auto& c : _unrecognized_components) { all.push_back(std::make_pair(component_type::Unknown, c)); @@ -2696,7 +2680,7 @@ entry_descriptor entry_descriptor::make_descriptor(sstring sstdir, sstring fname } else { throw malformed_sstable_exception(seastar::sprint("invalid version for file %s. Name doesn't match any known version.", fname)); } - return entry_descriptor(ks, cf, version, boost::lexical_cast(generation), sstable::format_from_sstring(format), sstable::component_from_sstring(component)); + return entry_descriptor(ks, cf, version, boost::lexical_cast(generation), sstable::format_from_sstring(format), sstable::component_from_sstring(version, component)); } sstable::version_types sstable::version_from_sstring(sstring &s) { @@ -2715,9 +2699,9 @@ sstable::format_types sstable::format_from_sstring(sstring &s) { } } -component_type sstable::component_from_sstring(sstring &s) { +component_type sstable::component_from_sstring(version_types v, sstring &s) { try { - return reverse_map(s, _component_map); + return reverse_map(s, sstable_version_constants::get_component_map(v)); } catch (std::out_of_range&) { return component_type::Unknown; } @@ -2918,8 +2902,8 @@ fsync_directory(const io_error_handler& error_handler, sstring fname) { future<> remove_by_toc_name(sstring sstable_toc_name, const io_error_handler& error_handler) { return seastar::async([sstable_toc_name, &error_handler] () mutable { - sstring prefix = sstable_toc_name.substr(0, sstable_toc_name.size() - TOC_SUFFIX.size()); - auto new_toc_name = prefix + TEMPORARY_TOC_SUFFIX; + sstring prefix = sstable_toc_name.substr(0, sstable_toc_name.size() - sstable_version_constants::TOC_SUFFIX.size()); + auto new_toc_name = prefix + sstable_version_constants::TEMPORARY_TOC_SUFFIX; sstring dir; if (sstable_io_check(error_handler, file_exists, sstable_toc_name).get0()) { @@ -2946,7 +2930,7 @@ remove_by_toc_name(sstring sstable_toc_name, const io_error_handler& error_handl // eof return make_ready_future<>(); } - if (component == TOC_SUFFIX) { + if (component == sstable_version_constants::TOC_SUFFIX) { // already deleted return make_ready_future<>(); } @@ -2984,7 +2968,7 @@ sstable::remove_sstable_with_temp_toc(sstring ks, sstring cf, sstring dir, int64 // assert that temporary toc exists for this sstable. assert(tmptoc == true); - for (auto& entry : sstable::_component_map) { + for (auto& entry : sstable_version_constants::get_component_map(v)) { // Skipping TemporaryTOC because it must be the last component to // be deleted, and unordered map doesn't guarantee ordering. // This is needed because we may end up with a partial delete in diff --git a/sstables/sstables.hh b/sstables/sstables.hh index 4395bb6641..1051798f79 100644 --- a/sstables/sstables.hh +++ b/sstables/sstables.hh @@ -57,6 +57,7 @@ #include "flat_mutation_reader.hh" #include "utils/phased_barrier.hh" #include "component_type.hh" +#include "sstable_version.hh" #include @@ -225,7 +226,7 @@ public: // Like data_consume_rows() with bounds, but iterates over whole range data_consume_context data_consume_rows(row_consumer& consumer); - static component_type component_from_sstring(sstring& s); + static component_type component_from_sstring(version_types v, sstring& s); static version_types version_from_sstring(sstring& s); static format_types format_from_sstring(sstring& s); static const sstring filename(sstring dir, sstring ks, sstring cf, version_types version, int64_t generation, @@ -449,7 +450,6 @@ private: static std::unordered_map> _version_string; static std::unordered_map> _format_string; - static std::unordered_map> _component_map; std::unordered_set> _recognized_components; std::vector _unrecognized_components;