sstables: scylla_metadata: add schema member

To store the most important schema fields, like id, version, keyspace
name, table name and the list of all columns, along with their kind,
name and type. This will serve as alternative schema source to the one
stored in statistics component. This latter one doesn't store any of the
metatada and neither does primary key names (just the types), so it is
leads to confusion when it is used as schema source for scylla-sstable.

This new schema stored in the scylla-metadata component is not intended
to be a full-schema, equivalent to the one stored in the schema tables,
it is intended to be good enough for scylla-sstable being able to parse
sstables in a self-sufficient manner.

sstables/sstables.cc

@@ -1860,6 +1860,16 @@ sstable::read_scylla_metadata() noexcept {
     });
 }
 
+static sstable_column_kind to_sstable_column_kind(column_kind k) {
+    switch (k) {
+        case column_kind::partition_key: return sstable_column_kind::partition_key;
+        case column_kind::clustering_key: return sstable_column_kind::clustering_key;
+        case column_kind::static_column: return sstable_column_kind::static_column;
+        case column_kind::regular_column: return sstable_column_kind::regular_column;
+    }
+    on_internal_error(sstlog, format("to_sstable_column_kind(): unknown column kind {}", static_cast<std::underlying_type_t<column_kind>>(k)));
+}
+
 void
 sstable::write_scylla_metadata(shard_id shard, struct run_identifier identifier,
         std::optional<scylla_metadata::large_data_stats> ld_stats, std::optional<scylla_metadata::ext_timestamp_stats> ts_stats) {
@@ -1926,6 +1936,16 @@ sstable::write_scylla_metadata(shard_id shard, struct run_identifier identifier,
     }
     _components->scylla_metadata->data.set<scylla_metadata_type::SSTableIdentifier>(scylla_metadata::sstable_identifier{sid});
 
+    sstable_schema_type sstable_schema;
+    sstable_schema.id = _schema->id();
+    sstable_schema.version = _schema->version();
+    sstable_schema.keyspace_name.value = to_bytes(_schema->ks_name());
+    sstable_schema.table_name.value = to_bytes(_schema->cf_name());
+    for (const auto& col : _schema->all_columns()) {
+        sstable_schema.columns.elements.push_back(sstable_column_description{to_sstable_column_kind(col.kind), {col.name()}, {to_bytes(col.type->name())}});
+    }
+    _components->scylla_metadata->data.set<scylla_metadata_type::Schema>(std::move(sstable_schema));
+
     write_simple<component_type::Scylla>(*_components->scylla_metadata);
 }
 

sstables/types.hh

@@ -27,6 +27,7 @@
 #include "version.hh"
 #include "encoding_stats.hh"
 #include "types_fwd.hh"
+#include "schema/schema_fwd.hh"
 
 // While the sstable code works with char, bytes_view works with int8_t
 // (signed char). Rather than change all the code, let's do a cast.
@@ -542,6 +543,7 @@ enum class scylla_metadata_type : uint32_t {
     ScyllaVersion = 8,
     ExtTimestampStats = 9,
     SSTableIdentifier = 10,
+    Schema = 11,
 };
 
 // UUID is used for uniqueness across nodes, such that an imported sstable
@@ -598,6 +600,38 @@ enum class ext_timestamp_stats_type : uint32_t {
     min_live_row_marker_timestamp = 2,
 };
 
+// Mirrors column_kind from schema.hh
+// Not reusing said enum because this enum is ABI, it must have a defined
+// integer storage type and defined values for each member. This kind of
+// restrictions are hard to enforce on an enum in a seemingly unrelated part
+// of the code.
+enum class sstable_column_kind : uint8_t {
+    partition_key = 1,
+    clustering_key = 2,
+    static_column = 3,
+    regular_column = 4,
+};
+
+struct sstable_column_description {
+    sstable_column_kind kind;
+    disk_string<uint32_t> name;
+    disk_string<uint32_t> type;
+
+    template <typename Describer>
+    auto describe_type(sstable_version_types v, Describer f) { return f(kind, name, type); }
+};
+
+struct sstable_schema_type {
+    table_id id;
+    table_schema_version version;
+    disk_string<uint32_t> keyspace_name;
+    disk_string<uint32_t> table_name;
+    disk_array<uint32_t, sstable_column_description> columns;
+
+    template <typename Describer>
+    auto describe_type(sstable_version_types v, Describer f) { return f(id, version, keyspace_name, table_name, columns); }
+};
+
 struct scylla_metadata {
     using extension_attributes = disk_hash<uint32_t, disk_string<uint32_t>, disk_string<uint32_t>>;
     using large_data_stats = disk_hash<uint32_t, large_data_type, large_data_stats_entry>;
@@ -606,6 +640,7 @@ struct scylla_metadata {
     using scylla_version = disk_string<uint32_t>;
     using ext_timestamp_stats = disk_hash<uint32_t, ext_timestamp_stats_type, int64_t>;
     using sstable_identifier = sstable_identifier_type;
+    using sstable_schema = sstable_schema_type;
 
     disk_set_of_tagged_union<scylla_metadata_type,
             disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::Sharding, sharding_metadata>,
@@ -617,7 +652,8 @@ struct scylla_metadata {
             disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::ScyllaBuildId, scylla_build_id>,
             disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::ScyllaVersion, scylla_version>,
             disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::ExtTimestampStats, ext_timestamp_stats>,
-            disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::SSTableIdentifier, sstable_identifier>
+            disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::SSTableIdentifier, sstable_identifier>,
+            disk_tagged_union_member<scylla_metadata_type, scylla_metadata_type::Schema, sstable_schema>
             > data;
 
     sstable_enabled_features get_features() const {

tools/scylla-sstable.cc

@@ -1320,6 +1320,7 @@ const char* to_string(sstables::scylla_metadata_type t) {
         case sstables::scylla_metadata_type::ScyllaBuildId: return "scylla_build_id";
         case sstables::scylla_metadata_type::ExtTimestampStats: return "ext_timestamp_stats";
         case sstables::scylla_metadata_type::SSTableIdentifier: return "sstable_identifier";
+        case sstables::scylla_metadata_type::Schema: return "schema";
     }
     std::abort();
 }
@@ -1442,9 +1443,54 @@ public:
         (*this)(m.value);
     }
 
+    template <typename Size, typename Members>
+    void operator()(const sstables::disk_array<Size, Members>& a) const {
+        _writer.StartArray();
+        for (const auto& element : a.elements) {
+            (*this)(element);
+        }
+        _writer.EndArray();
+    }
+
     void operator()(const sstables::scylla_metadata::sstable_identifier& sid) const {
         _writer.AsString(sid.value);
     }
+
+    void operator()(const sstables::sstable_column_description& cd) const {
+        _writer.StartObject();
+
+        _writer.Key("kind");
+        _writer.Int64(static_cast<uint32_t>(cd.kind));
+
+        _writer.Key("name");
+        _writer.String(disk_string_to_string(cd.name));
+
+        _writer.Key("type");
+        _writer.String(disk_string_to_string(cd.type));
+
+        _writer.EndObject();
+    }
+
+    void operator()(const sstables::scylla_metadata::sstable_schema& s) const {
+        _writer.StartObject();
+
+        _writer.Key("id");
+        _writer.String(fmt::to_string(s.id));
+
+        _writer.Key("version");
+        _writer.String(fmt::to_string(s.version));
+
+        _writer.Key("keyspace_name");
+        _writer.String(disk_string_to_string(s.keyspace_name));
+
+        _writer.Key("table_name");
+        _writer.String(disk_string_to_string(s.table_name));
+
+        _writer.Key("columns");
+        (*this)(s.columns);
+
+        _writer.EndObject();
+    }
 };
 
 void dump_scylla_metadata_operation(schema_ptr schema, reader_permit permit, const std::vector<sstables::shared_sstable>& sstables,