lsa: disable constant_time_size in binomial_heap implementation

Corrupts heap on boost < 1.60, and not needed.

Fixes #698.

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "seastar"]
 	path = seastar
-	url = ../seastar
+	url = ../scylla-seastar
 	ignore = dirty
 [submodule "swagger-ui"]
 	path = swagger-ui

README.md

@@ -15,13 +15,13 @@ git submodule update --recursive
 * Installing required packages:
 
 ```
-sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel
+sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan
 ```
 
 * Build Scylla
 ```
 ./configure.py --mode=release --with=scylla --disable-xen
-ninja-build build/release/scylla -j2 # you can use more cpus if you have tons of RAM
+ninja build/release/scylla -j2 # you can use more cpus if you have tons of RAM
 
 ```
 

SCYLLA-VERSION-GEN

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-VERSION=0.15
+VERSION=0.13.2
 
 if test -f version
 then

api/api-doc/messaging_service.json

@@ -233,27 +233,46 @@
             "verb":{
                "type":"string",
                "enum":[
-                 "CLIENT_ID",
-                 "ECHO",
-                 "MUTATION",
-                 "MUTATION_DONE",
-                 "READ_DATA",
-                 "READ_MUTATION_DATA",
-                 "READ_DIGEST",
-                 "GOSSIP_DIGEST_SYN",
-                 "GOSSIP_DIGEST_ACK2",
-                 "GOSSIP_SHUTDOWN",
-                 "DEFINITIONS_UPDATE",
-                 "TRUNCATE",
-                 "REPLICATION_FINISHED",
-                 "MIGRATION_REQUEST",
-                 "STREAM_INIT_MESSAGE",
-                 "PREPARE_MESSAGE",
-                 "PREPARE_DONE_MESSAGE",
-                 "STREAM_MUTATION",
-                 "STREAM_MUTATION_DONE",
-                 "COMPLETE_MESSAGE",
-                 "LAST"
+                  "MUTATION",
+                  "BINARY",
+                  "READ_REPAIR",
+                  "READ",
+                  "REQUEST_RESPONSE",
+                  "STREAM_INITIATE",
+                  "STREAM_INITIATE_DONE",
+                  "STREAM_REPLY",
+                  "STREAM_REQUEST",
+                  "RANGE_SLICE",
+                  "BOOTSTRAP_TOKEN",
+                  "TREE_REQUEST",
+                  "TREE_RESPONSE",
+                  "JOIN",
+                  "GOSSIP_DIGEST_SYN",
+                  "GOSSIP_DIGEST_ACK",
+                  "GOSSIP_DIGEST_ACK2",
+                  "DEFINITIONS_ANNOUNCE",
+                  "DEFINITIONS_UPDATE",
+                  "TRUNCATE",
+                  "SCHEMA_CHECK",
+                  "INDEX_SCAN",
+                  "REPLICATION_FINISHED",
+                  "INTERNAL_RESPONSE",
+                  "COUNTER_MUTATION",
+                  "STREAMING_REPAIR_REQUEST",
+                  "STREAMING_REPAIR_RESPONSE",
+                  "SNAPSHOT",
+                  "MIGRATION_REQUEST",
+                  "GOSSIP_SHUTDOWN",
+                  "_TRACE",
+                  "ECHO",
+                  "REPAIR_MESSAGE",
+                  "PAXOS_PREPARE",
+                  "PAXOS_PROPOSE",
+                  "PAXOS_COMMIT",
+                  "PAGED_RANGE",
+                  "UNUSED_1",
+                  "UNUSED_2",
+                  "UNUSED_3"
                ]
             }
          }

api/api-doc/storage_service.json

@@ -425,7 +425,7 @@
                "summary":"load value. Keys are IP addresses",
                "type":"array",
                "items":{
-                  "type":"map_string_double"
+                  "type":"mapper"
                },
                "nickname":"get_load_map",
                "produces":[
@@ -797,72 +797,8 @@
                      "paramType":"path"
                   },
                   {
-                     "name":"primaryRange",
-                     "description":"If the value is the string 'true' with any capitalization, repair only the first range returned by the partitioner.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"parallelism",
-                     "description":"Repair parallelism, can be 0 (sequential), 1 (parallel) or 2 (datacenter-aware).",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"incremental",
-                     "description":"If the value is the string 'true' with any capitalization, perform incremental repair.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"jobThreads",
-                     "description":"An integer specifying the parallelism on each node.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"ranges",
-                     "description":"An explicit list of ranges to repair, overriding the default choice. Each range is expressed as token1:token2, and multiple ranges can be given as a comma separated list.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"columnFamilies",
-                     "description":"Which column families to repair in the given keyspace. Multiple columns families can be named separated by commas. If this option is missing, all column families in the keyspace are repaired.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"dataCenters",
-                     "description":"Which data centers are to participate in this repair. Multiple data centers can be listed separated by commas.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"hosts",
-                     "description":"Which hosts are to participate in this repair. Multiple hosts can be listed separated by commas.",
-                     "required":false,
-                     "allowMultiple":false,
-                     "type":"string",
-                     "paramType":"query"
-                  },
-                  {
-                     "name":"trace",
-                     "description":"If the value is the string 'true' with any capitalization, enable tracing of the repair.",
+                     "name":"options",
+                     "description":"Options for the repair",
                      "required":false,
                      "allowMultiple":false,
                      "type":"string",
@@ -2028,20 +1964,6 @@
             }
          }
       },
-      "map_string_double":{
-         "id":"map_string_double",
-         "description":"A key value mapping between a string and a double",
-         "properties":{
-            "key":{
-               "type":"string",
-               "description":"The key"
-            },
-            "value":{
-               "type":"double",
-               "description":"The value"
-            }
-         }
-      },
       "maplist_mapper":{
          "id":"maplist_mapper",
          "description":"A key value mapping, where key and value are list",

api/column_family.cc

@@ -64,21 +64,21 @@ future<> foreach_column_family(http_context& ctx, const sstring& name, function<
 
 future<json::json_return_type>  get_cf_stats(http_context& ctx, const sstring& name,
         int64_t column_family::stats::*f) {
-    return map_reduce_cf(ctx, name, int64_t(0), [f](const column_family& cf) {
+    return map_reduce_cf(ctx, name, 0, [f](const column_family& cf) {
         return cf.get_stats().*f;
     }, std::plus<int64_t>());
 }
 
 future<json::json_return_type>  get_cf_stats(http_context& ctx,
         int64_t column_family::stats::*f) {
-    return map_reduce_cf(ctx, int64_t(0), [f](const column_family& cf) {
+    return map_reduce_cf(ctx, 0, [f](const column_family& cf) {
         return cf.get_stats().*f;
     }, std::plus<int64_t>());
 }
 
 static future<json::json_return_type>  get_cf_stats_count(http_context& ctx, const sstring& name,
         utils::ihistogram column_family::stats::*f) {
-    return map_reduce_cf(ctx, name, int64_t(0), [f](const column_family& cf) {
+    return map_reduce_cf(ctx, name, 0, [f](const column_family& cf) {
         return (cf.get_stats().*f).count;
     }, std::plus<int64_t>());
 }
@@ -101,7 +101,7 @@ static future<json::json_return_type>  get_cf_stats_sum(http_context& ctx, const
 
 static future<json::json_return_type>  get_cf_stats_count(http_context& ctx,
         utils::ihistogram column_family::stats::*f) {
-    return map_reduce_cf(ctx, int64_t(0), [f](const column_family& cf) {
+    return map_reduce_cf(ctx, 0, [f](const column_family& cf) {
         return (cf.get_stats().*f).count;
     }, std::plus<int64_t>());
 }
@@ -133,7 +133,7 @@ static future<json::json_return_type> get_cf_histogram(http_context& ctx, utils:
 }
 
 static future<json::json_return_type> get_cf_unleveled_sstables(http_context& ctx, const sstring& name) {
-    return map_reduce_cf(ctx, name, int64_t(0), [](const column_family& cf) {
+    return map_reduce_cf(ctx, name, 0, [](const column_family& cf) {
         return cf.get_unleveled_sstables();
     }, std::plus<int64_t>());
 }
@@ -223,25 +223,25 @@ void set_column_family(http_context& ctx, routes& r) {
     });
 
     cf::get_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](column_family& cf) {
             return cf.active_memtable().region().occupancy().total_space();
         }, std::plus<int64_t>());
     });
 
     cf::get_all_memtable_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, 0, [](column_family& cf) {
             return cf.active_memtable().region().occupancy().total_space();
         }, std::plus<int64_t>());
     });
 
     cf::get_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](column_family& cf) {
             return cf.active_memtable().region().occupancy().used_space();
         }, std::plus<int64_t>());
     });
 
     cf::get_all_memtable_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, 0, [](column_family& cf) {
             return cf.active_memtable().region().occupancy().used_space();
         }, std::plus<int64_t>());
     });
@@ -256,7 +256,7 @@ void set_column_family(http_context& ctx, routes& r) {
 
     cf::get_cf_all_memtables_off_heap_size.set(r, [&ctx] (std::unique_ptr<request> req) {
         warn(unimplemented::cause::INDEXES);
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](column_family& cf) {
             return cf.occupancy().total_space();
         }, std::plus<int64_t>());
     });
@@ -265,21 +265,21 @@ void set_column_family(http_context& ctx, routes& r) {
         warn(unimplemented::cause::INDEXES);
         return ctx.db.map_reduce0([](const database& db){
             return db.dirty_memory_region_group().memory_used();
-        }, int64_t(0), std::plus<int64_t>()).then([](int res) {
+        }, 0, std::plus<int64_t>()).then([](int res) {
             return make_ready_future<json::json_return_type>(res);
         });
     });
 
     cf::get_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
         warn(unimplemented::cause::INDEXES);
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](column_family& cf) {
             return cf.occupancy().used_space();
         }, std::plus<int64_t>());
     });
 
     cf::get_all_cf_all_memtables_live_data_size.set(r, [&ctx] (std::unique_ptr<request> req) {
         warn(unimplemented::cause::INDEXES);
-        return map_reduce_cf(ctx, int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, 0, [](column_family& cf) {
             return cf.active_memtable().region().occupancy().used_space();
         }, std::plus<int64_t>());
     });
@@ -304,7 +304,7 @@ void set_column_family(http_context& ctx, routes& r) {
     });
 
     cf::get_estimated_row_count.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](column_family& cf) {
             uint64_t res = 0;
             for (auto i: *cf.get_sstables() ) {
                 res += i.second->get_stats_metadata().estimated_row_size.count();
@@ -424,11 +424,11 @@ void set_column_family(http_context& ctx, routes& r) {
     });
 
     cf::get_max_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), max_row_size, max_int64);
+        return map_reduce_cf(ctx, req->param["name"], 0, max_row_size, max_int64);
     });
 
     cf::get_all_max_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, int64_t(0), max_row_size, max_int64);
+        return map_reduce_cf(ctx, 0, max_row_size, max_int64);
     });
 
     cf::get_mean_row_size.set(r, [&ctx] (std::unique_ptr<request> req) {
@@ -539,20 +539,20 @@ void set_column_family(http_context& ctx, routes& r) {
         }, std::plus<uint64_t>());
     });
 
-    cf::get_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], uint64_t(0), [] (column_family& cf) {
-            return std::accumulate(cf.get_sstables()->begin(), cf.get_sstables()->end(), uint64_t(0), [](uint64_t s, auto& sst) {
-                return sst.second->get_summary().memory_footprint();
-            });
-        }, std::plus<uint64_t>());
+    cf::get_index_summary_off_heap_memory_used.set(r, [] (std::unique_ptr<request> req) {
+        //TBD
+        // FIXME
+        // We are missing the off heap memory calculation
+        // Return 0 is the wrong value. It's a work around
+        // until the memory calculation will be available
+        //auto id = get_uuid(req->param["name"], ctx.db.local());
+        return make_ready_future<json::json_return_type>(0);
     });
 
-    cf::get_all_index_summary_off_heap_memory_used.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, uint64_t(0), [] (column_family& cf) {
-            return std::accumulate(cf.get_sstables()->begin(), cf.get_sstables()->end(), uint64_t(0), [](uint64_t s, auto& sst) {
-                return sst.second->get_summary().memory_footprint();
-            });
-        }, std::plus<uint64_t>());
+    cf::get_all_index_summary_off_heap_memory_used.set(r, [] (std::unique_ptr<request> req) {
+        //TBD
+        unimplemented();
+        return make_ready_future<json::json_return_type>(0);
     });
 
     cf::get_compression_metadata_off_heap_memory_used.set(r, [] (std::unique_ptr<request> req) {
@@ -623,25 +623,25 @@ void set_column_family(http_context& ctx, routes& r) {
     });
 
     cf::get_row_cache_hit.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](const column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](const column_family& cf) {
             return cf.get_row_cache().stats().hits;
         }, std::plus<int64_t>());
     });
 
     cf::get_all_row_cache_hit.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, int64_t(0), [](const column_family& cf) {
+        return map_reduce_cf(ctx, 0, [](const column_family& cf) {
             return cf.get_row_cache().stats().hits;
         }, std::plus<int64_t>());
     });
 
     cf::get_row_cache_miss.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, req->param["name"], int64_t(0), [](const column_family& cf) {
+        return map_reduce_cf(ctx, req->param["name"], 0, [](const column_family& cf) {
             return cf.get_row_cache().stats().misses;
         }, std::plus<int64_t>());
     });
 
     cf::get_all_row_cache_miss.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return map_reduce_cf(ctx, int64_t(0), [](const column_family& cf) {
+        return map_reduce_cf(ctx, 0, [](const column_family& cf) {
             return cf.get_row_cache().stats().misses;
         }, std::plus<int64_t>());
 

api/compaction_manager.cc

@@ -21,7 +21,6 @@
 
 #include "compaction_manager.hh"
 #include "api/api-doc/compaction_manager.json.hh"
-#include "db/system_keyspace.hh"
 
 namespace api {
 
@@ -39,25 +38,12 @@ static future<json::json_return_type> get_cm_stats(http_context& ctx,
 }
 
 void set_compaction_manager(http_context& ctx, routes& r) {
-    cm::get_compactions.set(r, [&ctx] (std::unique_ptr<request> req) {
-        return ctx.db.map_reduce0([](database& db) {
-            std::vector<cm::summary> summaries;
-            const compaction_manager& cm = db.get_compaction_manager();
-
-            for (const auto& c : cm.get_compactions()) {
-                cm::summary s;
-                s.ks = c->ks;
-                s.cf = c->cf;
-                s.unit = "keys";
-                s.task_type = "compaction";
-                s.completed = c->total_keys_written;
-                s.total = c->total_partitions;
-                summaries.push_back(std::move(s));
-            }
-            return summaries;
-        }, std::vector<cm::summary>(), concat<cm::summary>).then([](const std::vector<cm::summary>& res) {
-            return make_ready_future<json::json_return_type>(res);
-        });
+    cm::get_compactions.set(r, [] (std::unique_ptr<request> req) {
+        //TBD
+        // FIXME
+        warn(unimplemented::cause::API);
+        std::vector<cm::summary> map;
+        return make_ready_future<json::json_return_type>(map);
     });
 
     cm::force_user_defined_compaction.set(r, [] (std::unique_ptr<request> req) {
@@ -97,29 +83,11 @@ void set_compaction_manager(http_context& ctx, routes& r) {
     });
 
     cm::get_compaction_history.set(r, [] (std::unique_ptr<request> req) {
-        return db::system_keyspace::get_compaction_history().then([] (std::vector<db::system_keyspace::compaction_history_entry> history) {
-            std::vector<cm::history> res;
-            res.reserve(history.size());
-
-            for (auto& entry : history) {
-                cm::history h;
-                h.id = entry.id.to_sstring();
-                h.ks = std::move(entry.ks);
-                h.cf = std::move(entry.cf);
-                h.compacted_at = entry.compacted_at;
-                h.bytes_in = entry.bytes_in;
-                h.bytes_out =  entry.bytes_out;
-                for (auto it : entry.rows_merged) {
-                    httpd::compaction_manager_json::row_merged e;
-                    e.key = it.first;
-                    e.value = it.second;
-                    h.rows_merged.push(std::move(e));
-                }
-                res.push_back(std::move(h));
-            }
-
-            return make_ready_future<json::json_return_type>(res);
-        });
+        //TBD
+        // FIXME
+        warn(unimplemented::cause::API);
+        std::vector<cm::history> res;
+        return make_ready_future<json::json_return_type>(res);
     });
 
     cm::get_compaction_info.set(r, [] (std::unique_ptr<request> req) {

api/messaging_service.cc

@@ -34,7 +34,7 @@ namespace api {
 using shard_info = messaging_service::shard_info;
 using shard_id = messaging_service::shard_id;
 
-static const int32_t num_verb = static_cast<int32_t>(messaging_verb::LAST) + 1;
+static const int32_t num_verb = static_cast<int32_t>(messaging_verb::UNUSED_3) + 1;
 
 std::vector<message_counter> map_to_message_counters(
         const std::unordered_map<gms::inet_address, unsigned long>& map) {

api/storage_service.cc

@@ -89,7 +89,7 @@ void set_storage_service(http_context& ctx, routes& r) {
     });
 
     ss::get_token_endpoint.set(r, [] (const_req req) {
-        auto token_to_ep = service::get_local_storage_service().get_token_to_endpoint_map();
+        auto token_to_ep = service::get_local_storage_service().get_token_metadata().get_token_to_endpoint();
         std::vector<storage_service_json::mapper> res;
         return map_to_key_value(token_to_ep, res);
     });
@@ -169,14 +169,8 @@ void set_storage_service(http_context& ctx, routes& r) {
 
     ss::get_load_map.set(r, [] (std::unique_ptr<request> req) {
         return service::get_local_storage_service().get_load_map().then([] (auto&& load_map) {
-            std::vector<ss::map_string_double> res;
-            for (auto i : load_map) {
-                ss::map_string_double val;
-                val.key = i.first;
-                val.value = i.second;
-                res.push_back(val);
-            }
-            return make_ready_future<json::json_return_type>(res);
+            std::vector<ss::mapper> res;
+            return make_ready_future<json::json_return_type>(map_to_key_value(load_map, res));
         });
     });
 
@@ -318,14 +312,18 @@ void set_storage_service(http_context& ctx, routes& r) {
 
 
     ss::repair_async.set(r, [&ctx](std::unique_ptr<request> req) {
-        static std::vector<sstring> options = {"primaryRange", "parallelism", "incremental",
-                "jobThreads", "ranges", "columnFamilies", "dataCenters", "hosts", "trace"};
+        // Currently, we get all the repair options encoded in a single
+        // "options" option, and split it to a map using the "," and ":"
+        // delimiters. TODO: consider if it doesn't make more sense to just
+        // take all the query parameters as this map and pass it to the repair
+        // function.
         std::unordered_map<sstring, sstring> options_map;
-        for (auto o : options) {
-            auto s = req->get_query_param(o);
-            if (s != "") {
-                options_map[o] = s;
+        for (auto s : split(req->get_query_param("options"), ",")) {
+            auto kv = split(s, ":");
+            if (kv.size() != 2) {
+                throw httpd::bad_param_exception("malformed async repair options");
             }
+            options_map.emplace(std::move(kv[0]), std::move(kv[1]));
         }
 
         // The repair process is asynchronous: repair_start only starts it and
@@ -417,18 +415,15 @@ void set_storage_service(http_context& ctx, routes& r) {
     });
 
     ss::get_drain_progress.set(r, [](std::unique_ptr<request> req) {
-        return service::get_storage_service().map_reduce(adder<service::storage_service::drain_progress>(), [] (auto& ss) {
-            return ss.get_drain_progress();
-        }).then([] (auto&& progress) {
-            auto progress_str = sprint("Drained %s/%s ColumnFamilies", progress.remaining_cfs, progress.total_cfs);
-            return make_ready_future<json::json_return_type>(std::move(progress_str));
-        });
+        //TBD
+        unimplemented();
+        return make_ready_future<json::json_return_type>("");
     });
 
     ss::drain.set(r, [](std::unique_ptr<request> req) {
-        return service::get_local_storage_service().drain().then([] {
-            return make_ready_future<json::json_return_type>(json_void());
-        });
+        //TBD
+        unimplemented();
+        return make_ready_future<json::json_return_type>(json_void());
     });
     ss::truncate.set(r, [&ctx](std::unique_ptr<request> req) {
         //TBD
@@ -542,9 +537,10 @@ void set_storage_service(http_context& ctx, routes& r) {
         return make_ready_future<json::json_return_type>(0);
     });
 
-    ss::get_compaction_throughput_mb_per_sec.set(r, [&ctx](std::unique_ptr<request> req) {
-        int value = ctx.db.local().get_config().compaction_throughput_mb_per_sec();
-        return make_ready_future<json::json_return_type>(value);
+    ss::get_compaction_throughput_mb_per_sec.set(r, [](std::unique_ptr<request> req) {
+        //TBD
+        unimplemented();
+        return make_ready_future<json::json_return_type>(0);
     });
 
     ss::set_compaction_throughput_mb_per_sec.set(r, [](std::unique_ptr<request> req) {

api/stream_manager.cc

@@ -72,12 +72,11 @@ static hs::stream_state get_state(
         si.peer = boost::lexical_cast<std::string>(info.peer);
         si.session_index = info.session_index;
         si.state = info.state;
-        si.connecting = si.peer;
+        si.connecting = boost::lexical_cast<std::string>(info.connecting);
         set_summaries(info.receiving_summaries, si.receiving_summaries);
         set_summaries(info.sending_summaries, si.sending_summaries);
         set_files(info.receiving_files, si.receiving_files);
         set_files(info.sending_files, si.sending_files);
-        state.sessions.push(si);
     }
     return state;
 }

atomic_cell.hh

@@ -302,12 +302,6 @@ public:
     bool operator==(const atomic_cell_or_collection& other) const {
         return _data == other._data;
     }
-    void linearize() {
-        _data.linearize();
-    }
-    void unlinearize() {
-        _data.scatter();
-    }
     friend std::ostream& operator<<(std::ostream&, const atomic_cell_or_collection&);
 };
 

caching_options.hh

@@ -82,12 +82,6 @@ public:
         }
         return caching_options(k, r);
     }
-    bool operator==(const caching_options& other) const {
-        return _key_cache == other._key_cache && _row_cache == other._row_cache;
-    }
-    bool operator!=(const caching_options& other) const {
-        return !(*this == other);
-    }
 };
 
 

compound.hh

@@ -68,7 +68,7 @@ public:
         , _byte_order_equal(std::all_of(_types.begin(), _types.end(), [] (auto t) {
                 return t->is_byte_order_equal();
             }))
-        , _byte_order_comparable(!is_prefixable && _types.size() == 1 && _types[0]->is_byte_order_comparable())
+        , _byte_order_comparable(_types.size() == 1 && _types[0]->is_byte_order_comparable())
         , _is_reversed(_types.size() == 1 && _types[0]->is_reversed())
     { }
 
@@ -278,10 +278,10 @@ public:
             });
     }
     bytes from_string(sstring_view s) {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
     }
     sstring to_string(const bytes& b) {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
     }
     // Retruns true iff given prefix has no missing components
     bool is_full(bytes_view v) const {

compress.hh

@@ -114,14 +114,6 @@ public:
         }
         return opts;
     }
-    bool operator==(const compression_parameters& other) const {
-        return _compressor == other._compressor
-               && _chunk_length == other._chunk_length
-               && _crc_check_chance == other._crc_check_chance;
-    }
-    bool operator!=(const compression_parameters& other) const {
-        return !(*this == other);
-    }
 private:
     void validate_options(const std::map<sstring, sstring>& options) {
         // currently, there are no options specific to a particular compressor

conf/scylla.yaml

@@ -782,25 +782,40 @@ commitlog_total_space_in_mb: -1
 # the request scheduling. Currently the only valid option is keyspace.
 # request_scheduler_id: keyspace
 
-# Enable or disable inter-node encryption. 
-# You must also generate keys and provide the appropriate key and trust store locations and passwords. 
-# No custom encryption options are currently enabled. The available options are:
-#
+# Enable or disable inter-node encryption
+# Default settings are TLS v1, RSA 1024-bit keys (it is imperative that
+# users generate their own keys) TLS_RSA_WITH_AES_128_CBC_SHA as the cipher
+# suite for authentication, key exchange and encryption of the actual data transfers.
+# Use the DHE/ECDHE ciphers if running in FIPS 140 compliant mode.
+# NOTE: No custom encryption options are enabled at the moment
 # The available internode options are : all, none, dc, rack
-# If set to dc scylla  will encrypt the traffic between the DCs
-# If set to rack scylla  will encrypt the traffic between the racks
+#
+# If set to dc cassandra will encrypt the traffic between the DCs
+# If set to rack cassandra will encrypt the traffic between the racks
+#
+# The passwords used in these options must match the passwords used when generating
+# the keystore and truststore.  For instructions on generating these files, see:
+# http://download.oracle.com/javase/6/docs/technotes/guides/security/jsse/JSSERefGuide.html#CreateKeystore
 #
 # server_encryption_options:
 #    internode_encryption: none
-#    certificate: conf/scylla.crt
-#    keyfile: conf/scylla.key
-#    truststore: <none, use system trust>
+#    keystore: conf/.keystore
+#    keystore_password: cassandra
+#    truststore: conf/.truststore
+#    truststore_password: cassandra
+
+    # More advanced defaults below:
+    # protocol: TLS
+    # algorithm: SunX509
+    # store_type: JKS
+    # cipher_suites: [TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_DHE_RSA_WITH_AES_128_CBC_SHA,TLS_DHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]
+    # require_client_auth: false
 
 # enable or disable client/server encryption.
 # client_encryption_options:
 #    enabled: false
-#    certificate: conf/scylla.crt
-#    keyfile: conf/scylla.key
+#    keystore: conf/.keystore
+#    keystore_password: cassandra
 
     # require_client_auth: false
     # Set trustore and truststore_password if require_client_auth is true
@@ -824,17 +839,3 @@ commitlog_total_space_in_mb: -1
 # reducing overhead from the TCP protocol itself, at the cost of increasing
 # latency if you block for cross-datacenter responses.
 # inter_dc_tcp_nodelay: false
-
-# Relaxation of environment checks.
-#
-# Scylla places certain requirements on its environment.  If these requirements are
-# not met, performance and reliability can be degraded.
-#
-# These requirements include:
-#    - A filesystem with good support for aysnchronous I/O (AIO). Currently,
-#      this means XFS.
-#
-# false: strict environment checks are in place; do not start if they are not met.
-# true: relaxed environment checks; performance and reliability may degraade.
-#
-# developer_mode: false

configure.py

@@ -183,7 +183,6 @@ scylla_tests = [
     'tests/managed_vector_test',
     'tests/crc_test',
     'tests/flush_queue_test',
-    'tests/dynamic_bitset_test',
 ]
 
 apps = [
@@ -281,8 +280,6 @@ scylla_core = (['database.cc',
                  'cql3/statements/schema_altering_statement.cc',
                  'cql3/statements/ks_prop_defs.cc',
                  'cql3/statements/modification_statement.cc',
-                 'cql3/statements/parsed_statement.cc',
-                 'cql3/statements/property_definitions.cc',
                  'cql3/statements/update_statement.cc',
                  'cql3/statements/delete_statement.cc',
                  'cql3/statements/batch_statement.cc',
@@ -292,7 +289,6 @@ scylla_core = (['database.cc',
                  'cql3/statements/index_target.cc',
                  'cql3/statements/create_index_statement.cc',
                  'cql3/statements/truncate_statement.cc',
-                 'cql3/statements/alter_table_statement.cc',
                  'cql3/update_parameters.cc',
                  'cql3/ut_name.cc',
                  'thrift/handler.cc',
@@ -343,7 +339,6 @@ scylla_core = (['database.cc',
                  'utils/rate_limiter.cc',
                  'utils/compaction_manager.cc',
                  'utils/file_lock.cc',
-                 'utils/dynamic_bitset.cc',
                  'gms/version_generator.cc',
                  'gms/versioned_value.cc',
                  'gms/gossiper.cc',
@@ -487,7 +482,6 @@ tests_not_using_seastar_test_framework = set([
     'tests/crc_test',
     'tests/perf/perf_sstable',
     'tests/managed_vector_test',
-    'tests/dynamic_bitset_test',
 ])
 
 for t in tests_not_using_seastar_test_framework:
@@ -504,7 +498,7 @@ deps['tests/sstable_test'] += ['tests/sstable_datafile_test.cc']
 deps['tests/bytes_ostream_test'] = ['tests/bytes_ostream_test.cc']
 deps['tests/UUID_test'] = ['utils/UUID_gen.cc', 'tests/UUID_test.cc']
 deps['tests/murmur_hash_test'] = ['bytes.cc', 'utils/murmur_hash.cc', 'tests/murmur_hash_test.cc']
-deps['tests/allocation_strategy_test'] = ['tests/allocation_strategy_test.cc', 'utils/logalloc.cc', 'log.cc', 'utils/dynamic_bitset.cc']
+deps['tests/allocation_strategy_test'] = ['tests/allocation_strategy_test.cc', 'utils/logalloc.cc', 'log.cc']
 
 warnings = [
     '-Wno-mismatched-tags',  # clang-only

cql3/Cql.g

@@ -31,7 +31,6 @@ options {
 
 @parser::includes {
 #include "cql3/selection/writetime_or_ttl.hh"
-#include "cql3/statements/alter_table_statement.hh"
 #include "cql3/statements/create_keyspace_statement.hh"
 #include "cql3/statements/drop_keyspace_statement.hh"
 #include "cql3/statements/create_index_statement.hh"
@@ -270,9 +269,7 @@ cqlStatement returns [shared_ptr<parsed_statement> stmt]
     | st12=dropTableStatement          { $stmt = st12; }
 #if 0
     | st13=dropIndexStatement          { $stmt = st13; }
-#endif
     | st14=alterTableStatement         { $stmt = st14; }
-#if 0
     | st15=alterKeyspaceStatement      { $stmt = st15; }
     | st16=grantStatement              { $stmt = st16; }
     | st17=revokeStatement             { $stmt = st17; }
@@ -771,7 +768,7 @@ alterKeyspaceStatement returns [AlterKeyspaceStatement expr]
     : K_ALTER K_KEYSPACE ks=keyspaceName
         K_WITH properties[attrs] { $expr = new AlterKeyspaceStatement(ks, attrs); }
     ;
-#endif
+
 
 /**
  * ALTER COLUMN FAMILY <CF> ALTER <column> TYPE <newtype>;
@@ -780,29 +777,27 @@ alterKeyspaceStatement returns [AlterKeyspaceStatement expr]
  * ALTER COLUMN FAMILY <CF> WITH <property> = <value>;
  * ALTER COLUMN FAMILY <CF> RENAME <column> TO <column>;
  */
-alterTableStatement returns [shared_ptr<alter_table_statement> expr]
+alterTableStatement returns [AlterTableStatement expr]
     @init {
-        alter_table_statement::type type;
-        auto props = make_shared<cql3::statements::cf_prop_defs>();;
-        std::vector<std::pair<shared_ptr<cql3::column_identifier::raw>, shared_ptr<cql3::column_identifier::raw>>> renames;
-        bool is_static = false;
+        AlterTableStatement.Type type = null;
+        CFPropDefs props = new CFPropDefs();
+        Map<ColumnIdentifier.Raw, ColumnIdentifier.Raw> renames = new HashMap<ColumnIdentifier.Raw, ColumnIdentifier.Raw>();
+        boolean isStatic = false;
     }
     : K_ALTER K_COLUMNFAMILY cf=columnFamilyName
-          ( K_ALTER id=cident K_TYPE v=comparatorType { type = alter_table_statement::type::alter; }
-          | K_ADD   id=cident v=comparatorType ({ is_static=true; } K_STATIC)? { type = alter_table_statement::type::add; }
-          | K_DROP  id=cident                         { type = alter_table_statement::type::drop; }
-          | K_WITH  properties[props]                 { type = alter_table_statement::type::opts; }
-          | K_RENAME                                  { type = alter_table_statement::type::rename; }
-               id1=cident K_TO toId1=cident { renames.emplace_back(id1, toId1); }
-               ( K_AND idn=cident K_TO toIdn=cident { renames.emplace_back(idn, toIdn); } )*
+          ( K_ALTER id=cident K_TYPE v=comparatorType { type = AlterTableStatement.Type.ALTER; }
+          | K_ADD   id=cident v=comparatorType ({ isStatic=true; } K_STATIC)? { type = AlterTableStatement.Type.ADD; }
+          | K_DROP  id=cident                         { type = AlterTableStatement.Type.DROP; }
+          | K_WITH  properties[props]                 { type = AlterTableStatement.Type.OPTS; }
+          | K_RENAME                                  { type = AlterTableStatement.Type.RENAME; }
+               id1=cident K_TO toId1=cident { renames.put(id1, toId1); }
+               ( K_AND idn=cident K_TO toIdn=cident { renames.put(idn, toIdn); } )*
           )
     {
-        $expr = ::make_shared<alter_table_statement>(std::move(cf), type, std::move(id),
-            std::move(v), std::move(props), std::move(renames), is_static);
+        $expr = new AlterTableStatement(cf, type, id, v, props, renames, isStatic);
     }
     ;
 
-#if 0
 /**
  * ALTER TYPE <name> ALTER <field> TYPE <newtype>;
  * ALTER TYPE <name> ADD <field> <newtype>;
@@ -1248,7 +1243,6 @@ relationType returns [const cql3::operator_type* op = nullptr]
     ;
 
 relation[std::vector<cql3::relation_ptr>& clauses]
-    @init{ const cql3::operator_type* rt = nullptr; }
     : name=cident type=relationType t=term { $clauses.emplace_back(::make_shared<cql3::single_column_relation>(std::move(name), *type, std::move(t))); }
 
     | K_TOKEN l=tupleOfIdentifiers type=relationType t=term
@@ -1258,9 +1252,11 @@ relation[std::vector<cql3::relation_ptr>& clauses]
         { $clauses.emplace_back(make_shared<cql3::single_column_relation>(std::move(name), cql3::operator_type::IN, std::move(marker))); }
     | name=cident K_IN in_values=singleColumnInValues
         { $clauses.emplace_back(cql3::single_column_relation::create_in_relation(std::move(name), std::move(in_values))); }
-    | name=cident K_CONTAINS { rt = &cql3::operator_type::CONTAINS; } (K_KEY { rt = &cql3::operator_type::CONTAINS_KEY; })?
-        t=term { $clauses.emplace_back(make_shared<cql3::single_column_relation>(std::move(name), *rt, std::move(t))); }
-    | name=cident '[' key=term ']' type=relationType t=term { $clauses.emplace_back(make_shared<cql3::single_column_relation>(std::move(name), std::move(key), *type, std::move(t))); }
+#if 0
+    | name=cident K_CONTAINS { Operator rt = Operator.CONTAINS; } (K_KEY { rt = Operator.CONTAINS_KEY; })?
+        t=term { $clauses.add(new SingleColumnRelation(name, rt, t)); }
+    | name=cident '[' key=term ']' type=relationType t=term { $clauses.add(new SingleColumnRelation(name, key, type, t)); }
+#endif
     | ids=tupleOfIdentifiers
       ( K_IN
           ( '(' ')'

cql3/column_identifier.hh

@@ -55,11 +55,14 @@ namespace cql3 {
  * Represents an identifer for a CQL column definition.
  * TODO : should support light-weight mode without text representation for when not interned
  */
-class column_identifier final : public selection::selectable {
+class column_identifier final : public selection::selectable /* implements IMeasurableMemory*/ {
 public:
     bytes bytes_;
 private:
     sstring _text;
+#if 0
+    private static final long EMPTY_SIZE = ObjectSizes.measure(new ColumnIdentifier("", true));
+#endif
 public:
     column_identifier(sstring raw_text, bool keep_case);
 
@@ -80,6 +83,20 @@ public:
     }
 
 #if 0
+    public long unsharedHeapSize()
+    {
+        return EMPTY_SIZE
+             + ObjectSizes.sizeOnHeapOf(bytes)
+             + ObjectSizes.sizeOf(text);
+    }
+
+    public long unsharedHeapSizeExcludingData()
+    {
+        return EMPTY_SIZE
+             + ObjectSizes.sizeOnHeapExcludingData(bytes)
+             + ObjectSizes.sizeOf(text);
+    }
+
     public ColumnIdentifier clone(AbstractAllocator allocator)
     {
         return new ColumnIdentifier(allocator.clone(bytes), text);

cql3/maps.cc

@@ -114,26 +114,30 @@ maps::literal::validate_assignable_to(database& db, const sstring& keyspace, col
 
 assignment_testable::test_result
 maps::literal::test_assignment(database& db, const sstring& keyspace, ::shared_ptr<column_specification> receiver) {
-    if (!dynamic_pointer_cast<const map_type_impl>(receiver->type)) {
-        return assignment_testable::test_result::NOT_ASSIGNABLE;
-    }
+    throw std::runtime_error("not implemented");
+#if 0
+    if (!(receiver.type instanceof MapType))
+        return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+
     // If there is no elements, we can't say it's an exact match (an empty map if fundamentally polymorphic).
-    if (entries.empty()) {
-        return assignment_testable::test_result::WEAKLY_ASSIGNABLE;
-    }
-    auto key_spec = maps::key_spec_of(*receiver);
-    auto value_spec = maps::value_spec_of(*receiver);
+    if (entries.isEmpty())
+        return AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
+
+    ColumnSpecification keySpec = Maps.keySpecOf(receiver);
+    ColumnSpecification valueSpec = Maps.valueSpecOf(receiver);
     // It's an exact match if all are exact match, but is not assignable as soon as any is non assignable.
-    auto res = assignment_testable::test_result::EXACT_MATCH;
-    for (auto entry : entries) {
-        auto t1 = entry.first->test_assignment(db, keyspace, key_spec);
-        auto t2 = entry.second->test_assignment(db, keyspace, value_spec);
-        if (t1 == assignment_testable::test_result::NOT_ASSIGNABLE || t2 == assignment_testable::test_result::NOT_ASSIGNABLE)
-            return assignment_testable::test_result::NOT_ASSIGNABLE;
-        if (t1 != assignment_testable::test_result::EXACT_MATCH || t2 != assignment_testable::test_result::EXACT_MATCH)
-            res = assignment_testable::test_result::WEAKLY_ASSIGNABLE;
+    AssignmentTestable.TestResult res = AssignmentTestable.TestResult.EXACT_MATCH;
+    for (Pair<Term.Raw, Term.Raw> entry : entries)
+    {
+        AssignmentTestable.TestResult t1 = entry.left.testAssignment(keyspace, keySpec);
+        AssignmentTestable.TestResult t2 = entry.right.testAssignment(keyspace, valueSpec);
+        if (t1 == AssignmentTestable.TestResult.NOT_ASSIGNABLE || t2 == AssignmentTestable.TestResult.NOT_ASSIGNABLE)
+            return AssignmentTestable.TestResult.NOT_ASSIGNABLE;
+        if (t1 != AssignmentTestable.TestResult.EXACT_MATCH || t2 != AssignmentTestable.TestResult.EXACT_MATCH)
+            res = AssignmentTestable.TestResult.WEAKLY_ASSIGNABLE;
     }
     return res;
+#endif
 }
 
 sstring

cql3/operation.hh

@@ -199,7 +199,13 @@ public:
         }
 
         virtual shared_ptr<operation> prepare(database& db, const sstring& keyspace, const column_definition& receiver);
+#if 0
+        protected String toString(ColumnSpecification column)
+        {
+            return String.format("%s[%s] = %s", column.name, selector, value);
+        }
 
+#endif
         virtual bool is_compatible_with(shared_ptr<raw_update> other) override;
     };
 
@@ -212,6 +218,13 @@ public:
 
         virtual shared_ptr<operation> prepare(database& db, const sstring& keyspace, const column_definition& receiver) override;
 
+#if 0
+        protected String toString(ColumnSpecification column)
+        {
+            return String.format("%s = %s + %s", column.name, column.name, value);
+        }
+#endif
+
         virtual bool is_compatible_with(shared_ptr<raw_update> other) override;
     };
 
@@ -224,6 +237,13 @@ public:
 
         virtual shared_ptr<operation> prepare(database& db, const sstring& keyspace, const column_definition& receiver) override;
 
+#if 0
+        protected String toString(ColumnSpecification column)
+        {
+            return String.format("%s = %s - %s", column.name, column.name, value);
+        }
+#endif
+
         virtual bool is_compatible_with(shared_ptr<raw_update> other) override;
     };
 
@@ -236,6 +256,12 @@ public:
 
         virtual shared_ptr<operation> prepare(database& db, const sstring& keyspace, const column_definition& receiver) override;
 
+#if 0
+        protected String toString(ColumnSpecification column)
+        {
+            return String.format("%s = %s - %s", column.name, value, column.name);
+        }
+#endif
         virtual bool is_compatible_with(shared_ptr<raw_update> other) override;
     };
 

cql3/query_processor.cc

@@ -178,7 +178,7 @@ query_processor::prepare(const std::experimental::string_view& query_string, con
 query_processor::get_stored_prepared_statement(const std::experimental::string_view& query_string, const sstring& keyspace, bool for_thrift)
 {
     if (for_thrift) {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
 #if 0
         Integer thriftStatementId = computeThriftId(queryString, keyspace);
         ParsedStatement.Prepared existing = thriftPreparedStatements.get(thriftStatementId);
@@ -209,7 +209,7 @@ query_processor::store_prepared_statement(const std::experimental::string_view&
                                                         MAX_CACHE_PREPARED_MEMORY));
 #endif
     if (for_thrift) {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
 #if 0
         Integer statementId = computeThriftId(queryString, keyspace);
         thriftPreparedStatements.put(statementId, prepared);
@@ -334,9 +334,6 @@ query_options query_processor::make_internal_options(
 future<::shared_ptr<untyped_result_set>> query_processor::execute_internal(
         const std::experimental::string_view& query_string,
         const std::initializer_list<data_value>& values) {
-    if (log.is_enabled(logging::log_level::trace)) {
-        log.trace("execute_internal: \"{}\" ({})", query_string, ::join(", ", values));
-    }
     auto p = prepare_internal(query_string);
     auto opts = make_internal_options(p, values);
     return do_with(std::move(opts),

cql3/restrictions/multi_column_restriction.hh

@@ -374,7 +374,7 @@ public:
     }
 
     virtual std::vector<bytes_opt> bounds(statements::bound b, const query_options& options) const override {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
 #if 0
         return Composites.toByteBuffers(boundsAsComposites(b, options));
 #endif

cql3/restrictions/statement_restrictions.cc

@@ -41,13 +41,13 @@ public:
 
     ::shared_ptr<primary_key_restrictions<T>> do_merge_to(schema_ptr schema, ::shared_ptr<restriction> restriction) const {
         if (restriction->is_multi_column()) {
-            throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+            throw std::runtime_error("not implemented");
         }
         return ::make_shared<single_column_primary_key_restrictions<T>>(schema)->merge_to(schema, restriction);
     }
     ::shared_ptr<primary_key_restrictions<T>> merge_to(schema_ptr schema, ::shared_ptr<restriction> restriction) override {
         if (restriction->is_multi_column()) {
-            throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+            throw std::runtime_error("not implemented");
         }
         if (restriction->is_on_token()) {
             return static_pointer_cast<token_restriction>(restriction);

cql3/selection/selection.cc

@@ -384,11 +384,7 @@ void result_set_builder::visitor::accept_new_row(
             _builder.add(_partition_key[def->component_index()]);
             break;
         case column_kind::clustering_key:
-            if (_clustering_key.size() > def->component_index()) {
-                _builder.add(_clustering_key[def->component_index()]);
-            } else {
-                _builder.add({});
-            }
+            _builder.add(_clustering_key[def->component_index()]);
             break;
         case column_kind::regular_column:
             add_value(*def, row_iterator);

cql3/selection/selection.hh

@@ -134,7 +134,7 @@ public:
      * @return <code>true</code> if this selection contains a collection, <code>false</code> otherwise.
      */
     bool contains_a_collection() const {
-        if (!_schema->has_multi_cell_collections()) {
+        if (!_schema->has_collections()) {
             return false;
         }
 

cql3/single_column_relation.hh

@@ -159,7 +159,7 @@ protected:
     virtual shared_ptr<restrictions::restriction> new_contains_restriction(database& db, schema_ptr schema,
                                                  ::shared_ptr<variable_specifications> bound_names,
                                                  bool is_key) override {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
 #if 0
         ColumnDefinition columnDef = toColumnDefinition(schema, entity);
         Term term = toTerm(toReceivers(schema, columnDef), value, schema.ksName, bound_names);

cql3/statements/alter_table_statement.cc

@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Copyright 2015 ScyllaDB
- *
- * Modified by ScyllaDB
- */
-
-/*
- * This file is part of Scylla.
- *
- * Scylla is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * Scylla is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cql3/statements/alter_table_statement.hh"
-#include "service/migration_manager.hh"
-#include "validation.hh"
-
-namespace cql3 {
-
-namespace statements {
-
-alter_table_statement::alter_table_statement(shared_ptr<cf_name> name,
-                                             type t,
-                                             shared_ptr<column_identifier::raw> column_name,
-                                             shared_ptr<cql3_type::raw> validator,
-                                             shared_ptr<cf_prop_defs> properties,
-                                             renames_type renames,
-                                             bool is_static)
-    : schema_altering_statement(std::move(name))
-    , _type(t)
-    , _raw_column_name(std::move(column_name))
-    , _validator(std::move(validator))
-    , _properties(std::move(properties))
-    , _renames(std::move(renames))
-    , _is_static(is_static)
-{
-}
-
-void alter_table_statement::check_access(const service::client_state& state)
-{
-    warn(unimplemented::cause::PERMISSIONS);
-#if 0
-    state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.ALTER);
-#endif
-}
-
-void alter_table_statement::validate(distributed<service::storage_proxy>& proxy, const service::client_state& state)
-{
-    // validated in announce_migration()
-}
-
-future<bool> alter_table_statement::announce_migration(distributed<service::storage_proxy>& proxy, bool is_local_only)
-{
-    throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
-#if 0
-    CFMetaData meta = validateColumnFamily(keyspace(), columnFamily());
-    CFMetaData cfm = meta.copy();
-
-    CQL3Type validator = this.validator == null ? null : this.validator.prepare(keyspace());
-    ColumnIdentifier columnName = null;
-    ColumnDefinition def = null;
-    if (rawColumnName != null)
-    {
-        columnName = rawColumnName.prepare(cfm);
-        def = cfm.getColumnDefinition(columnName);
-    }
-
-    switch (oType)
-    {
-        case ADD:
-            assert columnName != null;
-            if (cfm.comparator.isDense())
-                throw new InvalidRequestException("Cannot add new column to a COMPACT STORAGE table");
-
-            if (isStatic)
-            {
-                if (!cfm.comparator.isCompound())
-                    throw new InvalidRequestException("Static columns are not allowed in COMPACT STORAGE tables");
-                if (cfm.clusteringColumns().isEmpty())
-                    throw new InvalidRequestException("Static columns are only useful (and thus allowed) if the table has at least one clustering column");
-            }
-
-            if (def != null)
-            {
-                switch (def.kind)
-                {
-                    case PARTITION_KEY:
-                    case CLUSTERING_COLUMN:
-                        throw new InvalidRequestException(String.format("Invalid column name %s because it conflicts with a PRIMARY KEY part", columnName));
-                    default:
-                        throw new InvalidRequestException(String.format("Invalid column name %s because it conflicts with an existing column", columnName));
-                }
-            }
-
-            // Cannot re-add a dropped counter column. See #7831.
-            if (meta.isCounter() && meta.getDroppedColumns().containsKey(columnName))
-                throw new InvalidRequestException(String.format("Cannot re-add previously dropped counter column %s", columnName));
-
-            AbstractType<?> type = validator.getType();
-            if (type.isCollection() && type.isMultiCell())
-            {
-                if (!cfm.comparator.supportCollections())
-                    throw new InvalidRequestException("Cannot use non-frozen collections with a non-composite PRIMARY KEY");
-                if (cfm.isSuper())
-                    throw new InvalidRequestException("Cannot use non-frozen collections with super column families");
-
-                // If there used to be a collection column with the same name (that has been dropped), it will
-                // still be appear in the ColumnToCollectionType because or reasons explained on #6276. The same
-                // reason mean that we can't allow adding a new collection with that name (see the ticket for details).
-                if (cfm.comparator.hasCollections())
-                {
-                    CollectionType previous = cfm.comparator.collectionType() == null ? null : cfm.comparator.collectionType().defined.get(columnName.bytes);
-                    if (previous != null && !type.isCompatibleWith(previous))
-                        throw new InvalidRequestException(String.format("Cannot add a collection with the name %s " +
-                                    "because a collection with the same name and a different type has already been used in the past", columnName));
-                }
-
-                cfm.comparator = cfm.comparator.addOrUpdateCollection(columnName, (CollectionType)type);
-            }
-
-            Integer componentIndex = cfm.comparator.isCompound() ? cfm.comparator.clusteringPrefixSize() : null;
-            cfm.addColumnDefinition(isStatic
-                                    ? ColumnDefinition.staticDef(cfm, columnName.bytes, type, componentIndex)
-                                    : ColumnDefinition.regularDef(cfm, columnName.bytes, type, componentIndex));
-            break;
-
-        case ALTER:
-            assert columnName != null;
-            if (def == null)
-                throw new InvalidRequestException(String.format("Column %s was not found in table %s", columnName, columnFamily()));
-
-            AbstractType<?> validatorType = validator.getType();
-            switch (def.kind)
-            {
-                case PARTITION_KEY:
-                    if (validatorType instanceof CounterColumnType)
-                        throw new InvalidRequestException(String.format("counter type is not supported for PRIMARY KEY part %s", columnName));
-                    if (cfm.getKeyValidator() instanceof CompositeType)
-                    {
-                        List<AbstractType<?>> oldTypes = ((CompositeType) cfm.getKeyValidator()).types;
-                        if (!validatorType.isValueCompatibleWith(oldTypes.get(def.position())))
-                            throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.",
-                                                                           columnName,
-                                                                           oldTypes.get(def.position()).asCQL3Type(),
-                                                                           validator));
-
-                        List<AbstractType<?>> newTypes = new ArrayList<AbstractType<?>>(oldTypes);
-                        newTypes.set(def.position(), validatorType);
-                        cfm.keyValidator(CompositeType.getInstance(newTypes));
-                    }
-                    else
-                    {
-                        if (!validatorType.isValueCompatibleWith(cfm.getKeyValidator()))
-                            throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.",
-                                                                           columnName,
-                                                                           cfm.getKeyValidator().asCQL3Type(),
-                                                                           validator));
-                        cfm.keyValidator(validatorType);
-                    }
-                    break;
-                case CLUSTERING_COLUMN:
-                    if (!cfm.isCQL3Table())
-                        throw new InvalidRequestException(String.format("Cannot alter clustering column %s in a non-CQL3 table", columnName));
-
-                    AbstractType<?> oldType = cfm.comparator.subtype(def.position());
-                    // Note that CFMetaData.validateCompatibility already validate the change we're about to do. However, the error message it
-                    // sends is a bit cryptic for a CQL3 user, so validating here for a sake of returning a better error message
-                    // Do note that we need isCompatibleWith here, not just isValueCompatibleWith.
-                    if (!validatorType.isCompatibleWith(oldType))
-                        throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are not order-compatible.",
-                                                                       columnName,
-                                                                       oldType.asCQL3Type(),
-                                                                       validator));
-
-                    cfm.comparator = cfm.comparator.setSubtype(def.position(), validatorType);
-                    break;
-                case COMPACT_VALUE:
-                    // See below
-                    if (!validatorType.isValueCompatibleWith(cfm.getDefaultValidator()))
-                        throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.",
-                                                                       columnName,
-                                                                       cfm.getDefaultValidator().asCQL3Type(),
-                                                                       validator));
-                    cfm.defaultValidator(validatorType);
-                    break;
-                case REGULAR:
-                case STATIC:
-                    // Thrift allows to change a column validator so CFMetaData.validateCompatibility will let it slide
-                    // if we change to an incompatible type (contrarily to the comparator case). But we don't want to
-                    // allow it for CQL3 (see #5882) so validating it explicitly here. We only care about value compatibility
-                    // though since we won't compare values (except when there is an index, but that is validated by
-                    // ColumnDefinition already).
-                    if (!validatorType.isValueCompatibleWith(def.type))
-                        throw new ConfigurationException(String.format("Cannot change %s from type %s to type %s: types are incompatible.",
-                                                                       columnName,
-                                                                       def.type.asCQL3Type(),
-                                                                       validator));
-
-                    // For collections, if we alter the type, we need to update the comparator too since it includes
-                    // the type too (note that isValueCompatibleWith above has validated that the new type doesn't
-                    // change the underlying sorting order, but we still don't want to have a discrepancy between the type
-                    // in the comparator and the one in the ColumnDefinition as that would be dodgy).
-                    if (validatorType.isCollection() && validatorType.isMultiCell())
-                        cfm.comparator = cfm.comparator.addOrUpdateCollection(def.name, (CollectionType)validatorType);
-
-                    break;
-            }
-            // In any case, we update the column definition
-            cfm.addOrReplaceColumnDefinition(def.withNewType(validatorType));
-            break;
-
-        case DROP:
-            assert columnName != null;
-            if (!cfm.isCQL3Table())
-                throw new InvalidRequestException("Cannot drop columns from a non-CQL3 table");
-            if (def == null)
-                throw new InvalidRequestException(String.format("Column %s was not found in table %s", columnName, columnFamily()));
-
-            switch (def.kind)
-            {
-                case PARTITION_KEY:
-                case CLUSTERING_COLUMN:
-                    throw new InvalidRequestException(String.format("Cannot drop PRIMARY KEY part %s", columnName));
-                case REGULAR:
-                case STATIC:
-                    ColumnDefinition toDelete = null;
-                    for (ColumnDefinition columnDef : cfm.regularAndStaticColumns())
-                    {
-                        if (columnDef.name.equals(columnName))
-                            toDelete = columnDef;
-                    }
-                    assert toDelete != null;
-                    cfm.removeColumnDefinition(toDelete);
-                    cfm.recordColumnDrop(toDelete);
-                    break;
-            }
-            break;
-        case OPTS:
-            if (cfProps == null)
-                throw new InvalidRequestException(String.format("ALTER COLUMNFAMILY WITH invoked, but no parameters found"));
-
-            cfProps.validate();
-
-            if (meta.isCounter() && cfProps.getDefaultTimeToLive() > 0)
-                throw new InvalidRequestException("Cannot set default_time_to_live on a table with counters");
-
-            cfProps.applyToCFMetadata(cfm);
-            break;
-        case RENAME:
-            for (Map.Entry<ColumnIdentifier.Raw, ColumnIdentifier.Raw> entry : renames.entrySet())
-            {
-                ColumnIdentifier from = entry.getKey().prepare(cfm);
-                ColumnIdentifier to = entry.getValue().prepare(cfm);
-                cfm.renameColumn(from, to);
-            }
-            break;
-    }
-
-    MigrationManager.announceColumnFamilyUpdate(cfm, false, isLocalOnly);
-    return true;
-#endif
-}
-
-shared_ptr<transport::event::schema_change> alter_table_statement::change_event()
-{
-    return make_shared<transport::event::schema_change>(transport::event::schema_change::change_type::UPDATED,
-        transport::event::schema_change::target_type::TABLE, keyspace(), column_family());
-}
-
-}
-
-}

cql3/statements/alter_table_statement.hh

@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Copyright 2015 ScyllaDB
- *
- * Modified by ScyllaDB
- */
-
-/*
- * This file is part of Scylla.
- *
- * Scylla is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * Scylla is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#pragma once
-
-#include "cql3/statements/schema_altering_statement.hh"
-#include "cql3/statements/cf_prop_defs.hh"
-#include "cql3/cql3_type.hh"
-
-namespace cql3 {
-
-namespace statements {
-
-class alter_table_statement : public schema_altering_statement {
-public:
-    enum class type {
-        add,
-        alter,
-        drop,
-        opts,
-        rename,
-    };
-    using renames_type = std::vector<std::pair<shared_ptr<column_identifier::raw>,
-                                               shared_ptr<column_identifier::raw>>>;
-private:
-    const type _type;
-    const shared_ptr<column_identifier::raw> _raw_column_name;
-    const shared_ptr<cql3_type::raw> _validator;
-    const shared_ptr<cf_prop_defs> _properties;
-    const renames_type _renames;
-    const bool _is_static;
-public:
-    alter_table_statement(shared_ptr<cf_name> name,
-                          type t,
-                          shared_ptr<column_identifier::raw> column_name,
-                          shared_ptr<cql3_type::raw> validator,
-                          shared_ptr<cf_prop_defs> properties,
-                          renames_type renames,
-                          bool is_static);
-
-    virtual void check_access(const service::client_state& state) override;
-    virtual void validate(distributed<service::storage_proxy>& proxy, const service::client_state& state) override;
-    virtual future<bool> announce_migration(distributed<service::storage_proxy>& proxy, bool is_local_only) override;
-    virtual shared_ptr<transport::event::schema_change> change_event() override;
-};
-
-}
-
-}

cql3/statements/batch_statement.hh

@@ -322,7 +322,7 @@ public:
     virtual future<shared_ptr<transport::messages::result_message>> execute_internal(
             distributed<service::storage_proxy>& proxy,
             service::query_state& query_state, const query_options& options) override {
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw "not implemented";
 #if 0
         assert !hasConditions;
         for (IMutation mutation : getMutations(BatchQueryOptions.withoutPerStatementVariables(options), true, queryState.getTimestamp()))

cql3/statements/cf_prop_defs.cc

@@ -139,11 +139,6 @@ std::map<sstring, sstring> cf_prop_defs::get_compression_options() const {
     return std::map<sstring, sstring>{};
 }
 
-int32_t cf_prop_defs::get_default_time_to_live() const
-{
-    return get_int(KW_DEFAULT_TIME_TO_LIVE, 0);
-}
-
 void cf_prop_defs::apply_to_builder(schema_builder& builder) {
     if (has_property(KW_COMMENT)) {
         builder.set_comment(get_string(KW_COMMENT, ""));

cql3/statements/cf_prop_defs.hh

@@ -100,8 +100,6 @@ public:
         return options;
     }
 #endif
-    int32_t get_default_time_to_live() const;
-
     void apply_to_builder(schema_builder& builder);
     void validate_minimum_int(const sstring& field, int32_t minimum_value, int32_t default_value) const;
 };

cql3/statements/delete_statement.cc

@@ -45,14 +45,6 @@ namespace cql3 {
 
 namespace statements {
 
-delete_statement::delete_statement(statement_type type, uint32_t bound_terms, schema_ptr s, std::unique_ptr<attributes> attrs)
-        : modification_statement{type, bound_terms, std::move(s), std::move(attrs)}
-{ }
-
-bool delete_statement::require_full_clustering_key() const {
-    return false;
-}
-
 void delete_statement::add_update_for_key(mutation& m, const exploded_clustering_prefix& prefix, const update_parameters& params) {
     if (_column_operations.empty()) {
         m.partition().apply_delete(*s, prefix, params.make_tombstone());
@@ -104,17 +96,5 @@ delete_statement::parsed::prepare_internal(database& db, schema_ptr schema, ::sh
     return stmt;
 }
 
-delete_statement::parsed::parsed(::shared_ptr<cf_name> name,
-                                 ::shared_ptr<attributes::raw> attrs,
-                                 std::vector<::shared_ptr<operation::raw_deletion>> deletions,
-                                 std::vector<::shared_ptr<relation>> where_clause,
-                                 conditions_vector conditions,
-                                 bool if_exists)
-    : modification_statement::parsed(std::move(name), std::move(attrs), std::move(conditions), false, if_exists)
-    , _deletions(std::move(deletions))
-    , _where_clause(std::move(where_clause))
-{ }
-
 }
-
 }

cql3/statements/delete_statement.hh

@@ -55,9 +55,13 @@ namespace statements {
 */
 class delete_statement : public modification_statement {
 public:
-    delete_statement(statement_type type, uint32_t bound_terms, schema_ptr s, std::unique_ptr<attributes> attrs);
+    delete_statement(statement_type type, uint32_t bound_terms, schema_ptr s, std::unique_ptr<attributes> attrs)
+            : modification_statement{type, bound_terms, std::move(s), std::move(attrs)}
+    { }
 
-    virtual bool require_full_clustering_key() const override;
+    virtual bool require_full_clustering_key() const override {
+        return false;
+    }
 
     virtual void add_update_for_key(mutation& m, const exploded_clustering_prefix& prefix, const update_parameters& params) override;
 
@@ -90,7 +94,11 @@ public:
                std::vector<::shared_ptr<operation::raw_deletion>> deletions,
                std::vector<::shared_ptr<relation>> where_clause,
                conditions_vector conditions,
-               bool if_exists);
+               bool if_exists)
+            : modification_statement::parsed(std::move(name), std::move(attrs), std::move(conditions), false, if_exists)
+            , _deletions(std::move(deletions))
+            , _where_clause(std::move(where_clause))
+        { }
     protected:
         virtual ::shared_ptr<modification_statement> prepare_internal(database& db, schema_ptr schema,
             ::shared_ptr<variable_specifications> bound_names, std::unique_ptr<attributes> attrs);

cql3/statements/modification_statement.cc

@@ -71,81 +71,6 @@ operator<<(std::ostream& out, modification_statement::statement_type t) {
     return out;
 }
 
-modification_statement::modification_statement(statement_type type_, uint32_t bound_terms, schema_ptr schema_, std::unique_ptr<attributes> attrs_)
-    : type{type_}
-    , _bound_terms{bound_terms}
-    , s{schema_}
-    , attrs{std::move(attrs_)}
-    , _column_operations{}
-{ }
-
-bool modification_statement::uses_function(const sstring& ks_name, const sstring& function_name) const {
-    if (attrs->uses_function(ks_name, function_name)) {
-        return true;
-    }
-    for (auto&& e : _processed_keys) {
-        auto r = e.second;
-        if (r && r->uses_function(ks_name, function_name)) {
-            return true;
-        }
-    }
-    for (auto&& operation : _column_operations) {
-        if (operation && operation->uses_function(ks_name, function_name)) {
-            return true;
-        }
-    }
-    for (auto&& condition : _column_conditions) {
-        if (condition && condition->uses_function(ks_name, function_name)) {
-            return true;
-        }
-    }
-    for (auto&& condition : _static_conditions) {
-        if (condition && condition->uses_function(ks_name, function_name)) {
-            return true;
-        }
-    }
-    return false;
-}
-
-uint32_t modification_statement::get_bound_terms() {
-    return _bound_terms;
-}
-
-sstring modification_statement::keyspace() const {
-    return s->ks_name();
-}
-
-sstring modification_statement::column_family() const {
-    return s->cf_name();
-}
-
-bool modification_statement::is_counter() const {
-    return s->is_counter();
-}
-
-int64_t modification_statement::get_timestamp(int64_t now, const query_options& options) const {
-    return attrs->get_timestamp(now, options);
-}
-
-bool modification_statement::is_timestamp_set() const {
-    return attrs->is_timestamp_set();
-}
-
-gc_clock::duration modification_statement::get_time_to_live(const query_options& options) const {
-    return gc_clock::duration(attrs->get_time_to_live(options));
-}
-
-void modification_statement::check_access(const service::client_state& state) {
-    warn(unimplemented::cause::PERMISSIONS);
-#if 0
-    state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.MODIFY);
-
-    // CAS updates can be used to simulate a SELECT query, so should require Permission.SELECT as well.
-    if (hasConditions())
-        state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.SELECT);
-#endif
-}
-
 future<std::vector<mutation>>
 modification_statement::get_mutations(distributed<service::storage_proxy>& proxy, const query_options& options, bool local, int64_t now) {
     auto keys = make_lw_shared(build_partition_keys(options));
@@ -624,63 +549,6 @@ bool modification_statement::depends_on_column_family(const sstring& cf_name) co
     return column_family() == cf_name;
 }
 
-void modification_statement::add_operation(::shared_ptr<operation> op) {
-    if (op->column.is_static()) {
-        _sets_static_columns = true;
-    } else {
-        _sets_regular_columns = true;
-    }
-    _column_operations.push_back(std::move(op));
-}
-
-void modification_statement::add_condition(::shared_ptr<column_condition> cond) {
-    if (cond->column.is_static()) {
-        _sets_static_columns = true;
-        _static_conditions.emplace_back(std::move(cond));
-    } else {
-        _sets_regular_columns = true;
-        _column_conditions.emplace_back(std::move(cond));
-    }
-}
-
-void modification_statement::set_if_not_exist_condition() {
-    _if_not_exists = true;
-}
-
-bool modification_statement::has_if_not_exist_condition() const {
-    return _if_not_exists;
-}
-
-void modification_statement::set_if_exist_condition() {
-    _if_exists = true;
-}
-
-bool modification_statement::has_if_exist_condition() const {
-    return _if_exists;
-}
-
-bool modification_statement::requires_read() {
-    return std::any_of(_column_operations.begin(), _column_operations.end(), [] (auto&& op) {
-        return op->requires_read();
-    });
-}
-
-bool modification_statement::has_conditions() {
-    return _if_not_exists || _if_exists || !_column_conditions.empty() || !_static_conditions.empty();
-}
-
-void modification_statement::validate_where_clause_for_conditions() {
-    //  no-op by default
-}
-
-modification_statement::parsed::parsed(::shared_ptr<cf_name> name, ::shared_ptr<attributes::raw> attrs, conditions_vector conditions, bool if_not_exists, bool if_exists)
-    : cf_statement{std::move(name)}
-    , _attrs{std::move(attrs)}
-    , _conditions{std::move(conditions)}
-    , _if_not_exists{if_not_exists}
-    , _if_exists{if_exists}
-{ }
-
 }
 
 }

cql3/statements/modification_statement.hh

@@ -107,29 +107,84 @@ private:
         };
 
 public:
-    modification_statement(statement_type type_, uint32_t bound_terms, schema_ptr schema_, std::unique_ptr<attributes> attrs_);
+    modification_statement(statement_type type_, uint32_t bound_terms, schema_ptr schema_, std::unique_ptr<attributes> attrs_)
+        : type{type_}
+        , _bound_terms{bound_terms}
+        , s{schema_}
+        , attrs{std::move(attrs_)}
+        , _column_operations{}
+    { }
 
-    virtual bool uses_function(const sstring& ks_name, const sstring& function_name) const override;
+    virtual bool uses_function(const sstring& ks_name, const sstring& function_name) const override {
+        if (attrs->uses_function(ks_name, function_name)) {
+            return true;
+        }
+        for (auto&& e : _processed_keys) {
+            auto r = e.second;
+            if (r && r->uses_function(ks_name, function_name)) {
+                return true;
+            }
+        }
+        for (auto&& operation : _column_operations) {
+            if (operation && operation->uses_function(ks_name, function_name)) {
+                return true;
+            }
+        }
+        for (auto&& condition : _column_conditions) {
+            if (condition && condition->uses_function(ks_name, function_name)) {
+                return true;
+            }
+        }
+        for (auto&& condition : _static_conditions) {
+            if (condition && condition->uses_function(ks_name, function_name)) {
+                return true;
+            }
+        }
+        return false;
+    }
 
     virtual bool require_full_clustering_key() const = 0;
 
     virtual void add_update_for_key(mutation& m, const exploded_clustering_prefix& prefix, const update_parameters& params) = 0;
 
-    virtual uint32_t get_bound_terms() override;
+    virtual uint32_t get_bound_terms() override {
+        return _bound_terms;
+    }
 
-    virtual sstring keyspace() const;
+    virtual sstring keyspace() const {
+        return s->ks_name();
+    }
 
-    virtual sstring column_family() const;
+    virtual sstring column_family() const {
+        return s->cf_name();
+    }
 
-    virtual bool is_counter() const;
+    virtual bool is_counter() const {
+        return s->is_counter();
+    }
 
-    int64_t get_timestamp(int64_t now, const query_options& options) const;
+    int64_t get_timestamp(int64_t now, const query_options& options) const {
+        return attrs->get_timestamp(now, options);
+    }
 
-    bool is_timestamp_set() const;
+    bool is_timestamp_set() const {
+        return attrs->is_timestamp_set();
+    }
 
-    gc_clock::duration get_time_to_live(const query_options& options) const;
+    gc_clock::duration get_time_to_live(const query_options& options) const {
+        return gc_clock::duration(attrs->get_time_to_live(options));
+    }
 
-    virtual void check_access(const service::client_state& state) override;
+    virtual void check_access(const service::client_state& state) override {
+        warn(unimplemented::cause::PERMISSIONS);
+#if 0
+        state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.MODIFY);
+
+        // CAS updates can be used to simulate a SELECT query, so should require Permission.SELECT as well.
+        if (hasConditions())
+            state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.SELECT);
+#endif
+    }
 
     void validate(distributed<service::storage_proxy>&, const service::client_state& state) override;
 
@@ -137,7 +192,14 @@ public:
 
     virtual bool depends_on_column_family(const sstring& cf_name) const override;
 
-    void add_operation(::shared_ptr<operation> op);
+    void add_operation(::shared_ptr<operation> op) {
+        if (op->column.is_static()) {
+            _sets_static_columns = true;
+        } else {
+            _sets_regular_columns = true;
+        }
+        _column_operations.push_back(std::move(op));
+    }
 
 #if 0
     public Iterable<ColumnDefinition> getColumnsWithConditions()
@@ -150,15 +212,31 @@ public:
     }
 #endif
 public:
-    void add_condition(::shared_ptr<column_condition> cond);
+    void add_condition(::shared_ptr<column_condition> cond) {
+        if (cond->column.is_static()) {
+            _sets_static_columns = true;
+            _static_conditions.emplace_back(std::move(cond));
+        } else {
+            _sets_regular_columns = true;
+            _column_conditions.emplace_back(std::move(cond));
+        }
+    }
 
-    void set_if_not_exist_condition();
+    void set_if_not_exist_condition() {
+        _if_not_exists = true;
+    }
 
-    bool has_if_not_exist_condition() const;
+    bool has_if_not_exist_condition() const {
+        return _if_not_exists;
+    }
 
-    void set_if_exist_condition();
+    void set_if_exist_condition() {
+        _if_exists = true;
+    }
 
-    bool has_if_exist_condition() const;
+    bool has_if_exist_condition() const {
+        return _if_exists;
+    }
 
 private:
     void add_key_values(const column_definition& def, ::shared_ptr<restrictions::restriction> values);
@@ -176,7 +254,11 @@ protected:
     const column_definition* get_first_empty_key();
 
 public:
-    bool requires_read();
+    bool requires_read() {
+        return std::any_of(_column_operations.begin(), _column_operations.end(), [] (auto&& op) {
+            return op->requires_read();
+        });
+    }
 
 protected:
     future<update_parameters::prefetched_rows_type> read_required_rows(
@@ -187,7 +269,9 @@ protected:
                 db::consistency_level cl);
 
 public:
-    bool has_conditions();
+    bool has_conditions() {
+        return _if_not_exists || _if_exists || !_column_conditions.empty() || !_static_conditions.empty();
+    }
 
     virtual future<::shared_ptr<transport::messages::result_message>>
     execute(distributed<service::storage_proxy>& proxy, service::query_state& qs, const query_options& options) override;
@@ -344,7 +428,9 @@ protected:
      * processed to check that they are compatible.
      * @throws InvalidRequestException
      */
-    virtual void validate_where_clause_for_conditions();
+    virtual void validate_where_clause_for_conditions() {
+        //  no-op by default
+    }
 
 public:
     class parsed : public cf_statement {
@@ -357,7 +443,13 @@ public:
         const bool _if_not_exists;
         const bool _if_exists;
     protected:
-        parsed(::shared_ptr<cf_name> name, ::shared_ptr<attributes::raw> attrs, conditions_vector conditions, bool if_not_exists, bool if_exists);
+        parsed(::shared_ptr<cf_name> name, ::shared_ptr<attributes::raw> attrs, conditions_vector conditions, bool if_not_exists, bool if_exists)
+            : cf_statement{std::move(name)}
+            , _attrs{std::move(attrs)}
+            , _conditions{std::move(conditions)}
+            , _if_not_exists{if_not_exists}
+            , _if_exists{if_exists}
+        { }
 
     public:
         virtual ::shared_ptr<parsed_statement::prepared> prepare(database& db) override;

cql3/statements/parsed_statement.cc

@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Copyright 2014 Cloudius Systems
- *
- * Modified by Cloudius Systems
- */
-
-/*
- * This file is part of Scylla.
- *
- * Scylla is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * Scylla is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cql3/statements/parsed_statement.hh"
-
-namespace cql3 {
-
-namespace statements {
-
-parsed_statement::~parsed_statement()
-{ }
-
-shared_ptr<variable_specifications> parsed_statement::get_bound_variables() {
-    return _variables;
-}
-
-// Used by the parser and preparable statement
-void parsed_statement::set_bound_variables(const std::vector<::shared_ptr<column_identifier>>& bound_names) {
-    _variables = ::make_shared<variable_specifications>(bound_names);
-}
-
-bool parsed_statement::uses_function(const sstring& ks_name, const sstring& function_name) const {
-    return false;
-}
-
-parsed_statement::prepared::prepared(::shared_ptr<cql_statement> statement_, std::vector<::shared_ptr<column_specification>> bound_names_)
-    : statement(std::move(statement_))
-    , bound_names(std::move(bound_names_))
-{ }
-
-parsed_statement::prepared::prepared(::shared_ptr<cql_statement> statement_, const variable_specifications& names)
-    : prepared(statement_, names.get_specifications())
-{ }
-
-parsed_statement::prepared::prepared(::shared_ptr<cql_statement> statement_, variable_specifications&& names)
-    : prepared(statement_, std::move(names).get_specifications())
-{ }
-
-parsed_statement::prepared::prepared(::shared_ptr<cql_statement>&& statement_)
-    : prepared(statement_, std::vector<::shared_ptr<column_specification>>())
-{ }
-
-}
-
-}

cql3/statements/parsed_statement.hh

@@ -60,29 +60,47 @@ private:
     ::shared_ptr<variable_specifications> _variables;
 
 public:
-    virtual ~parsed_statement();
+    virtual ~parsed_statement()
+    { }
 
-    shared_ptr<variable_specifications> get_bound_variables();
+    shared_ptr<variable_specifications> get_bound_variables() {
+        return _variables;
+    }
 
-    void set_bound_variables(const std::vector<::shared_ptr<column_identifier>>& bound_names);
+    // Used by the parser and preparable statement
+    void set_bound_variables(const std::vector<::shared_ptr<column_identifier>>& bound_names)
+    {
+        _variables = ::make_shared<variable_specifications>(bound_names);
+    }
 
     class prepared {
     public:
         const ::shared_ptr<cql_statement> statement;
         const std::vector<::shared_ptr<column_specification>> bound_names;
 
-        prepared(::shared_ptr<cql_statement> statement_, std::vector<::shared_ptr<column_specification>> bound_names_);
+        prepared(::shared_ptr<cql_statement> statement_, std::vector<::shared_ptr<column_specification>> bound_names_)
+            : statement(std::move(statement_))
+            , bound_names(std::move(bound_names_))
+        { }
 
-        prepared(::shared_ptr<cql_statement> statement_, const variable_specifications& names);
+        prepared(::shared_ptr<cql_statement> statement_, const variable_specifications& names)
+            : prepared(statement_, names.get_specifications())
+        { }
 
-        prepared(::shared_ptr<cql_statement> statement_, variable_specifications&& names);
+        prepared(::shared_ptr<cql_statement> statement_, variable_specifications&& names)
+            : prepared(statement_, std::move(names).get_specifications())
+        { }
 
-        prepared(::shared_ptr<cql_statement>&& statement_);
+        prepared(::shared_ptr<cql_statement>&& statement_)
+            : prepared(statement_, std::vector<::shared_ptr<column_specification>>())
+        { }
     };
 
     virtual ::shared_ptr<prepared> prepare(database& db) = 0;
 
-    virtual bool uses_function(const sstring& ks_name, const sstring& function_name) const;
+    virtual bool uses_function(const sstring& ks_name, const sstring& function_name) const {
+        return false;
+    }
 };
 
 }

cql3/statements/property_definitions.cc

@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- * Copyright 2015 Cloudius Systems
- *
- * Modified by Cloudius Systems
- */
-
-/*
- * This file is part of Scylla.
- *
- * Scylla is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * Scylla is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cql3/statements/property_definitions.hh"
-
-namespace cql3 {
-
-namespace statements {
-
-property_definitions::property_definitions()
-    : _properties{}
-{ }
-
-void property_definitions::add_property(const sstring& name, sstring value) {
-    auto it = _properties.find(name);
-    if (it != _properties.end()) {
-        throw exceptions::syntax_exception(sprint("Multiple definition for property '%s'", name));
-    }
-    _properties.emplace(name, value);
-}
-
-void property_definitions::add_property(const sstring& name, const std::map<sstring, sstring>& value) {
-    auto it = _properties.find(name);
-    if (it != _properties.end()) {
-        throw exceptions::syntax_exception(sprint("Multiple definition for property '%s'", name));
-    }
-    _properties.emplace(name, value);
-}
-
-void property_definitions::validate(const std::set<sstring>& keywords, const std::set<sstring>& obsolete) {
-    for (auto&& kv : _properties) {
-        auto&& name = kv.first;
-        if (keywords.count(name)) {
-            continue;
-        }
-        if (obsolete.count(name)) {
-#if 0
-            logger.warn("Ignoring obsolete property {}", name);
-#endif
-        } else {
-            throw exceptions::syntax_exception(sprint("Unknown property '%s'", name));
-        }
-    }
-}
-
-std::experimental::optional<sstring> property_definitions::get_simple(const sstring& name) const {
-    auto it = _properties.find(name);
-    if (it == _properties.end()) {
-        return std::experimental::nullopt;
-    }
-    try {
-        return boost::any_cast<sstring>(it->second);
-    } catch (const boost::bad_any_cast& e) {
-        throw exceptions::syntax_exception(sprint("Invalid value for property '%s'. It should be a string", name));
-    }
-}
-
-std::experimental::optional<std::map<sstring, sstring>> property_definitions::get_map(const sstring& name) const {
-    auto it = _properties.find(name);
-    if (it == _properties.end()) {
-        return std::experimental::nullopt;
-    }
-    try {
-        return boost::any_cast<std::map<sstring, sstring>>(it->second);
-    } catch (const boost::bad_any_cast& e) {
-        throw exceptions::syntax_exception(sprint("Invalid value for property '%s'. It should be a map.", name));
-    }
-}
-
-bool property_definitions::has_property(const sstring& name) const {
-    return _properties.find(name) != _properties.end();
-}
-
-sstring property_definitions::get_string(sstring key, sstring default_value) const {
-    auto value = get_simple(key);
-    if (value) {
-        return value.value();
-    } else {
-        return default_value;
-    }
-}
-
-// Return a property value, typed as a Boolean
-bool property_definitions::get_boolean(sstring key, bool default_value) const {
-    auto value = get_simple(key);
-    if (value) {
-        std::string s{value.value()};
-        std::transform(s.begin(), s.end(), s.begin(), ::tolower);
-        return s == "1" || s == "true" || s == "yes";
-    } else {
-        return default_value;
-    }
-}
-
-// Return a property value, typed as a double
-double property_definitions::get_double(sstring key, double default_value) const {
-    auto value = get_simple(key);
-    return to_double(key, value, default_value);
-}
-
-double property_definitions::to_double(sstring key, std::experimental::optional<sstring> value, double default_value) {
-    if (value) {
-        auto val = value.value();
-        try {
-            return std::stod(val);
-        } catch (const std::exception& e) {
-            throw exceptions::syntax_exception(sprint("Invalid double value %s for '%s'", val, key));
-        }
-    } else {
-        return default_value;
-    }
-}
-
-// Return a property value, typed as an Integer
-int32_t property_definitions::get_int(sstring key, int32_t default_value) const {
-    auto value = get_simple(key);
-    return to_int(key, value, default_value);
-}
-
-int32_t property_definitions::to_int(sstring key, std::experimental::optional<sstring> value, int32_t default_value) {
-    if (value) {
-        auto val = value.value();
-        try {
-            return std::stoi(val);
-        } catch (const std::exception& e) {
-            throw exceptions::syntax_exception(sprint("Invalid integer value %s for '%s'", val, key));
-        }
-    } else {
-        return default_value;
-    }
-}
-
-long property_definitions::to_long(sstring key, std::experimental::optional<sstring> value, long default_value) {
-    if (value) {
-        auto val = value.value();
-        try {
-            return std::stol(val);
-        } catch (const std::exception& e) {
-            throw exceptions::syntax_exception(sprint("Invalid long value %s for '%s'", val, key));
-        }
-    } else {
-        return default_value;
-    }
-}
-
-}
-
-}

cql3/statements/property_definitions.hh

@@ -66,38 +66,141 @@ protected:
 #endif
     std::unordered_map<sstring, boost::any> _properties;
 
-    property_definitions();
+    property_definitions()
+        : _properties{}
+    { }
 public:
-    void add_property(const sstring& name, sstring value);
+    void add_property(const sstring& name, sstring value) {
+        auto it = _properties.find(name);
+        if (it != _properties.end()) {
+            throw exceptions::syntax_exception(sprint("Multiple definition for property '%s'", name));
+        }
+        _properties.emplace(name, value);
+    }
 
-    void add_property(const sstring& name, const std::map<sstring, sstring>& value);
-
-    void validate(const std::set<sstring>& keywords, const std::set<sstring>& obsolete);
+    void add_property(const sstring& name, const std::map<sstring, sstring>& value) {
+        auto it = _properties.find(name);
+        if (it != _properties.end()) {
+            throw exceptions::syntax_exception(sprint("Multiple definition for property '%s'", name));
+        }
+        _properties.emplace(name, value);
+    }
 
+    void validate(const std::set<sstring>& keywords, const std::set<sstring>& obsolete) {
+        for (auto&& kv : _properties) {
+            auto&& name = kv.first;
+            if (keywords.count(name)) {
+                continue;
+            }
+            if (obsolete.count(name)) {
+#if 0
+                logger.warn("Ignoring obsolete property {}", name);
+#endif
+            } else {
+                throw exceptions::syntax_exception(sprint("Unknown property '%s'", name));
+            }
+        }
+    }
 protected:
-    std::experimental::optional<sstring> get_simple(const sstring& name) const;
-
-    std::experimental::optional<std::map<sstring, sstring>> get_map(const sstring& name) const;
+    std::experimental::optional<sstring> get_simple(const sstring& name) const {
+        auto it = _properties.find(name);
+        if (it == _properties.end()) {
+            return std::experimental::nullopt;
+        }
+        try {
+            return boost::any_cast<sstring>(it->second);
+        } catch (const boost::bad_any_cast& e) {
+            throw exceptions::syntax_exception(sprint("Invalid value for property '%s'. It should be a string", name));
+        }
+    }
 
+    std::experimental::optional<std::map<sstring, sstring>> get_map(const sstring& name) const {
+        auto it = _properties.find(name);
+        if (it == _properties.end()) {
+            return std::experimental::nullopt;
+        }
+        try {
+            return boost::any_cast<std::map<sstring, sstring>>(it->second);
+        } catch (const boost::bad_any_cast& e) {
+            throw exceptions::syntax_exception(sprint("Invalid value for property '%s'. It should be a map.", name));
+        }
+    }
 public:
-    bool has_property(const sstring& name) const;
+    bool has_property(const sstring& name) const {
+        return _properties.find(name) != _properties.end();
+    }
 
-    sstring get_string(sstring key, sstring default_value) const;
+    sstring get_string(sstring key, sstring default_value) const {
+        auto value = get_simple(key);
+        if (value) {
+            return value.value();
+        } else {
+            return default_value;
+        }
+    }
 
     // Return a property value, typed as a Boolean
-    bool get_boolean(sstring key, bool default_value) const;
+    bool get_boolean(sstring key, bool default_value) const {
+        auto value = get_simple(key);
+        if (value) {
+            std::string s{value.value()};
+            std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+            return s == "1" || s == "true" || s == "yes";
+        } else {
+            return default_value;
+        }
+    }
 
     // Return a property value, typed as a double
-    double get_double(sstring key, double default_value) const;
+    double get_double(sstring key, double default_value) const {
+        auto value = get_simple(key);
+        return to_double(key, value, default_value);
+    }
 
-    static double to_double(sstring key, std::experimental::optional<sstring> value, double default_value);
+    static double to_double(sstring key, std::experimental::optional<sstring> value, double default_value) {
+        if (value) {
+            auto val = value.value();
+            try {
+                return std::stod(val);
+            } catch (const std::exception& e) {
+                throw exceptions::syntax_exception(sprint("Invalid double value %s for '%s'", val, key));
+            }
+        } else {
+            return default_value;
+        }
+    }
 
     // Return a property value, typed as an Integer
-    int32_t get_int(sstring key, int32_t default_value) const;
+    int32_t get_int(sstring key, int32_t default_value) const {
+        auto value = get_simple(key);
+        return to_int(key, value, default_value);
+    }
 
-    static int32_t to_int(sstring key, std::experimental::optional<sstring> value, int32_t default_value);
+    static int32_t to_int(sstring key, std::experimental::optional<sstring> value, int32_t default_value) {
+        if (value) {
+            auto val = value.value();
+            try {
+                return std::stoi(val);
+            } catch (const std::exception& e) {
+                throw exceptions::syntax_exception(sprint("Invalid integer value %s for '%s'", val, key));
+            }
+        } else {
+            return default_value;
+        }
+    }
 
-    static long to_long(sstring key, std::experimental::optional<sstring> value, long default_value);
+    static long to_long(sstring key, std::experimental::optional<sstring> value, long default_value) {
+        if (value) {
+            auto val = value.value();
+            try {
+                return std::stol(val);
+            } catch (const std::exception& e) {
+                throw exceptions::syntax_exception(sprint("Invalid long value %s for '%s'", val, key));
+            }
+        } else {
+            return default_value;
+        }
+    }
 };
 
 }

cql3/statements/select_statement.cc

@@ -54,31 +54,6 @@ namespace statements {
 
 thread_local const shared_ptr<select_statement::parameters> select_statement::_default_parameters = ::make_shared<select_statement::parameters>();
 
-select_statement::parameters::parameters()
-    : _is_distinct{false}
-    , _allow_filtering{false}
-{ }
-
-select_statement::parameters::parameters(orderings_type orderings,
-    bool is_distinct,
-    bool allow_filtering)
-    : _orderings{std::move(orderings)}
-    , _is_distinct{is_distinct}
-    , _allow_filtering{allow_filtering}
-{ }
-
-bool select_statement::parameters::is_distinct() {
-    return _is_distinct;
-}
-
-bool select_statement::parameters::allow_filtering() {
-    return _allow_filtering;
-}
-
-select_statement::parameters::orderings_type const& select_statement::parameters::orderings() {
-    return _orderings;
-}
-
 select_statement::select_statement(schema_ptr schema,
     uint32_t bound_terms,
     ::shared_ptr<parameters> parameters,
@@ -140,14 +115,6 @@ bool select_statement::depends_on_column_family(const sstring& cf_name) const {
     return column_family() == cf_name;
 }
 
-const sstring& select_statement::keyspace() const {
-    return _schema->ks_name();
-}
-
-const sstring& select_statement::column_family() const {
-    return _schema->cf_name();
-}
-
 query::partition_slice
 select_statement::make_partition_slice(const query_options& options) {
     std::vector<column_id> static_columns;
@@ -351,18 +318,6 @@ shared_ptr<transport::messages::result_message> select_statement::process_result
     return ::make_shared<transport::messages::result_message::rows>(std::move(rs));
 }
 
-select_statement::raw_statement::raw_statement(::shared_ptr<cf_name> cf_name,
-                                               ::shared_ptr<parameters> parameters,
-                                               std::vector<::shared_ptr<selection::raw_selector>> select_clause,
-                                               std::vector<::shared_ptr<relation>> where_clause,
-                                               ::shared_ptr<term::raw> limit)
-    : cf_statement(std::move(cf_name))
-    , _parameters(std::move(parameters))
-    , _select_clause(std::move(select_clause))
-    , _where_clause(std::move(where_clause))
-    , _limit(std::move(limit))
-{ }
-
 ::shared_ptr<parsed_statement::prepared>
 select_statement::raw_statement::prepare(database& db) {
     schema_ptr schema = validation::validate_column_family(db, keyspace(), column_family());

cql3/statements/select_statement.hh

@@ -72,13 +72,20 @@ public:
         const bool _is_distinct;
         const bool _allow_filtering;
     public:
-        parameters();
+        parameters()
+            : _is_distinct{false}
+            , _allow_filtering{false}
+        { }
         parameters(orderings_type orderings,
             bool is_distinct,
-            bool allow_filtering);
-        bool is_distinct();
-        bool allow_filtering();
-        orderings_type const& orderings();
+            bool allow_filtering)
+            : _orderings{std::move(orderings)}
+            , _is_distinct{is_distinct}
+            , _allow_filtering{allow_filtering}
+        { }
+        bool is_distinct() { return _is_distinct; }
+        bool allow_filtering() { return _allow_filtering; }
+        orderings_type const& orderings() { return _orderings; }
     };
 private:
     static constexpr int DEFAULT_COUNT_PAGE_SIZE = 10000;
@@ -188,9 +195,13 @@ public:
     }
 #endif
 
-    const sstring& keyspace() const;
+    const sstring& keyspace() const {
+        return _schema->ks_name();
+    }
 
-    const sstring& column_family() const;
+    const sstring& column_family() const {
+        return _schema->cf_name();
+    }
 
     query::partition_slice make_partition_slice(const query_options& options);
 
@@ -446,7 +457,13 @@ public:
             ::shared_ptr<parameters> parameters,
             std::vector<::shared_ptr<selection::raw_selector>> select_clause,
             std::vector<::shared_ptr<relation>> where_clause,
-            ::shared_ptr<term::raw> limit);
+            ::shared_ptr<term::raw> limit)
+        : cf_statement(std::move(cf_name))
+        , _parameters(std::move(parameters))
+        , _select_clause(std::move(select_clause))
+        , _where_clause(std::move(where_clause))
+        , _limit(std::move(limit))
+    { }
 
     virtual ::shared_ptr<prepared> prepare(database& db) override;
 private:

cql3/statements/update_statement.cc

@@ -48,14 +48,6 @@ namespace cql3 {
 
 namespace statements {
 
-update_statement::update_statement(statement_type type, uint32_t bound_terms, schema_ptr s, std::unique_ptr<attributes> attrs)
-    : modification_statement{type, bound_terms, std::move(s), std::move(attrs)}
-{ }
-
-bool update_statement::require_full_clustering_key() const {
-    return true;
-}
-
 void update_statement::add_update_for_key(mutation& m, const exploded_clustering_prefix& prefix, const update_parameters& params) {
     if (s->is_dense()) {
         if (!prefix || (prefix.size() == 1 && prefix.components().front().empty())) {
@@ -108,16 +100,6 @@ void update_statement::add_update_for_key(mutation& m, const exploded_clustering
 #endif
 }
 
-update_statement::parsed_insert::parsed_insert(::shared_ptr<cf_name> name,
-                                               ::shared_ptr<attributes::raw> attrs,
-                                               std::vector<::shared_ptr<column_identifier::raw>> column_names,
-                                               std::vector<::shared_ptr<term::raw>> column_values,
-                                               bool if_not_exists)
-    : modification_statement::parsed{std::move(name), std::move(attrs), conditions_vector{}, if_not_exists, false}
-    , _column_names{std::move(column_names)}
-    , _column_values{std::move(column_values)}
-{ }
-
 ::shared_ptr<modification_statement>
 update_statement::parsed_insert::prepare_internal(database& db, schema_ptr schema,
     ::shared_ptr<variable_specifications> bound_names, std::unique_ptr<attributes> attrs)
@@ -166,16 +148,6 @@ update_statement::parsed_insert::prepare_internal(database& db, schema_ptr schem
     return stmt;
 }
 
-update_statement::parsed_update::parsed_update(::shared_ptr<cf_name> name,
-                                               ::shared_ptr<attributes::raw> attrs,
-                                               std::vector<std::pair<::shared_ptr<column_identifier::raw>, ::shared_ptr<operation::raw_update>>> updates,
-                                               std::vector<relation_ptr> where_clause,
-                                               conditions_vector conditions)
-    : modification_statement::parsed(std::move(name), std::move(attrs), std::move(conditions), false, false)
-    , _updates(std::move(updates))
-    , _where_clause(std::move(where_clause))
-{ }
-
 ::shared_ptr<modification_statement>
 update_statement::parsed_update::prepare_internal(database& db, schema_ptr schema,
     ::shared_ptr<variable_specifications> bound_names, std::unique_ptr<attributes> attrs)

cql3/statements/update_statement.hh

@@ -64,9 +64,14 @@ public:
     private static final Constants.Value EMPTY = new Constants.Value(ByteBufferUtil.EMPTY_BYTE_BUFFER);
 #endif
 
-    update_statement(statement_type type, uint32_t bound_terms, schema_ptr s, std::unique_ptr<attributes> attrs);
+    update_statement(statement_type type, uint32_t bound_terms, schema_ptr s, std::unique_ptr<attributes> attrs)
+        : modification_statement{type, bound_terms, std::move(s), std::move(attrs)}
+    { }
+
 private:
-    virtual bool require_full_clustering_key() const override;
+    virtual bool require_full_clustering_key() const override {
+        return true;
+    }
 
     virtual void add_update_for_key(mutation& m, const exploded_clustering_prefix& prefix, const update_parameters& params) override;
 public:
@@ -87,7 +92,11 @@ public:
                       ::shared_ptr<attributes::raw> attrs,
                       std::vector<::shared_ptr<column_identifier::raw>> column_names,
                       std::vector<::shared_ptr<term::raw>> column_values,
-                      bool if_not_exists);
+                      bool if_not_exists)
+            : modification_statement::parsed{std::move(name), std::move(attrs), conditions_vector{}, if_not_exists, false}
+            , _column_names{std::move(column_names)}
+            , _column_values{std::move(column_values)}
+        { }
 
         virtual ::shared_ptr<modification_statement> prepare_internal(database& db, schema_ptr schema,
                     ::shared_ptr<variable_specifications> bound_names, std::unique_ptr<attributes> attrs) override;
@@ -113,7 +122,11 @@ public:
             ::shared_ptr<attributes::raw> attrs,
             std::vector<std::pair<::shared_ptr<column_identifier::raw>, ::shared_ptr<operation::raw_update>>> updates,
             std::vector<relation_ptr> where_clause,
-            conditions_vector conditions);
+            conditions_vector conditions)
+                : modification_statement::parsed(std::move(name), std::move(attrs), std::move(conditions), false, false)
+                , _updates(std::move(updates))
+                , _where_clause(std::move(where_clause))
+        { }
     protected:
         virtual ::shared_ptr<modification_statement> prepare_internal(database& db, schema_ptr schema,
                     ::shared_ptr<variable_specifications> bound_names, std::unique_ptr<attributes> attrs);

cql3/tuples.hh

@@ -224,6 +224,14 @@ public:
             // We don't "need" that override but it saves us the allocation of a Value object if used
             return options.make_temporary(_type->build_value(bind_internal(options)));
         }
+
+#if 0
+        @Override
+        public String toString()
+        {
+            return tupleToString(elements);
+        }
+#endif
     };
 
     /**

cql3/variable_specifications.hh

@@ -88,6 +88,14 @@ public:
         }
         _specs[bind_index] = spec;
     }
+
+#if 0
+    @Override
+    public String toString()
+    {
+        return Arrays.toString(specs);
+    }
+#endif
 };
 
 }

database.cc

@@ -358,32 +358,23 @@ column_family::for_all_partitions_slow(std::function<bool (const dht::decorated_
 class lister {
 public:
     using dir_entry_types = std::unordered_set<directory_entry_type, enum_hash<directory_entry_type>>;
-    using walker_type = std::function<future<> (directory_entry)>;
-    using filter_type = std::function<bool (const sstring&)>;
 private:
     file _f;
-    walker_type _walker;
-    filter_type _filter;
+    std::function<future<> (directory_entry de)> _walker;
     dir_entry_types _expected_type;
     subscription<directory_entry> _listing;
     sstring _dirname;
 
 public:
-    lister(file f, dir_entry_types type, walker_type walker, sstring dirname)
+    lister(file f, dir_entry_types type, std::function<future<> (directory_entry)> walker, sstring dirname)
             : _f(std::move(f))
             , _walker(std::move(walker))
-            , _filter([] (const sstring& fname) { return true; })
             , _expected_type(type)
             , _listing(_f.list_directory([this] (directory_entry de) { return _visit(de); }))
             , _dirname(dirname) {
     }
 
-    lister(file f, dir_entry_types type, walker_type walker, filter_type filter, sstring dirname)
-            : lister(std::move(f), type, std::move(walker), dirname) {
-        _filter = std::move(filter);
-    }
-
-    static future<> scan_dir(sstring name, dir_entry_types type, walker_type walker, filter_type filter = [] (const sstring& fname) { return true; });
+    static future<> scan_dir(sstring name, dir_entry_types type, std::function<future<> (directory_entry)> walker);
 protected:
     future<> _visit(directory_entry de) {
 
@@ -392,12 +383,6 @@ protected:
             if ((!_expected_type.count(*(de.type))) || (de.name[0] == '.')) {
                 return make_ready_future<>();
             }
-
-            // apply a filter
-            if (!_filter(_dirname + "/" + de.name)) {
-                return make_ready_future<>();
-            }
-
             return _walker(de);
         });
 
@@ -418,9 +403,9 @@ private:
 };
 
 
-future<> lister::scan_dir(sstring name, lister::dir_entry_types type, walker_type walker, filter_type filter) {
-    return engine().open_directory(name).then([type, walker = std::move(walker), filter = std::move(filter), name] (file f) {
-        auto l = make_lw_shared<lister>(std::move(f), type, walker, filter, name);
+future<> lister::scan_dir(sstring name, lister::dir_entry_types type, std::function<future<> (directory_entry)> walker) {
+    return engine().open_directory(name).then([type, walker = std::move(walker), name] (file f) {
+        auto l = make_lw_shared<lister>(std::move(f), type, walker, name);
         return l->done().then([l] { });
     });
 }
@@ -468,9 +453,6 @@ future<sstables::entry_descriptor> column_family::probe_file(sstring sstdir, sst
     return std::move(fut).then([this, sstdir = std::move(sstdir), comps] (range<partition_key> r) {
         // Checks whether or not sstable belongs to current shard.
         if (!belongs_to_current_shard(*_schema, std::move(r))) {
-            dblog.debug("sstable {} not relevant for this shard, ignoring",
-                    sstables::sstable::filename(sstdir, _schema->ks_name(), _schema->cf_name(), comps.version, comps.generation, comps.format,
-                            sstables::sstable::component_type::Data));
             sstable::mark_sstable_for_deletion(_schema->ks_name(), _schema->cf_name(), sstdir, comps.generation, comps.version, comps.format);
             return make_ready_future<>();
         }
@@ -488,8 +470,7 @@ future<sstables::entry_descriptor> column_family::probe_file(sstring sstdir, sst
             dblog.error("malformed sstable {}: {}. Refusing to boot", fname, e.what());
             throw;
         } catch(...) {
-            dblog.error("Unrecognized error while processing {}: {}. Refusing to boot",
-                    fname, std::current_exception());
+            dblog.error("Unrecognized error while processing {}: Refusing to boot", fname);
             throw;
         }
         return make_ready_future<entry_descriptor>(std::move(comps));
@@ -690,7 +671,7 @@ column_family::reshuffle_sstables(int64_t start) {
             // Those SSTables are not known by anyone in the system. So we don't have any kind of
             // object describing them. There isn't too much of a choice.
             return work.sstables[comps.generation]->read_toc();
-        }, &manifest_json_filter).then([&work] {
+        }).then([&work] {
             // Note: cannot be parallel because we will be shuffling things around at this stage. Can't race.
             return do_for_each(work.sstables, [&work] (auto& pair) {
                 auto&& comps = std::move(work.descriptors.at(pair.first));
@@ -750,22 +731,21 @@ column_family::compact_sstables(sstables::compaction_descriptor descriptor) {
             std::unordered_set<sstables::shared_sstable> s(
                     sstables_to_compact->begin(), sstables_to_compact->end());
             for (const auto& oldtab : *current_sstables) {
-                // Checks if oldtab is a sstable not being compacted.
                 if (!s.count(oldtab.second)) {
                     update_stats_for_new_sstable(oldtab.second->data_size());
                     _sstables->emplace(oldtab.first, oldtab.second);
                 }
-            }
 
-            for (const auto& newtab : *new_tables) {
-                // FIXME: rename the new sstable(s). Verify a rename doesn't cause
-                // problems for the sstable object.
-                update_stats_for_new_sstable(newtab.second->data_size());
-                _sstables->emplace(newtab.first, newtab.second);
-            }
+                for (const auto& newtab : *new_tables) {
+                    // FIXME: rename the new sstable(s). Verify a rename doesn't cause
+                    // problems for the sstable object.
+                    update_stats_for_new_sstable(newtab.second->data_size());
+                    _sstables->emplace(newtab.first, newtab.second);
+                }
 
-            for (const auto& oldtab : *sstables_to_compact) {
-                oldtab->mark_for_deletion();
+                for (const auto& oldtab : *sstables_to_compact) {
+                    oldtab->mark_for_deletion();
+                }
             }
         });
     });
@@ -782,8 +762,6 @@ column_family::load_new_sstables(std::vector<sstables::entry_descriptor> new_tab
             auto last = sst->get_last_partition_key(*_schema);
             if (belongs_to_current_shard(*_schema, first, last)) {
                 this->add_sstable(sst);
-            } else {
-                sst->mark_for_deletion();
             }
             return make_ready_future<>();
         });
@@ -856,17 +834,6 @@ lw_shared_ptr<sstable_list> column_family::get_sstables() {
     return _sstables;
 }
 
-inline bool column_family::manifest_json_filter(const sstring& fname) {
-    using namespace boost::filesystem;
-
-    path entry_path(fname);
-    if (!is_directory(status(entry_path)) && entry_path.filename() == path("manifest.json")) {
-        return false;
-    }
-
-    return true;
-}
-
 future<> column_family::populate(sstring sstdir) {
     // We can catch most errors when we try to load an sstable. But if the TOC
     // file is the one missing, we won't try to load the sstable at all. This
@@ -887,79 +854,79 @@ future<> column_family::populate(sstring sstdir) {
     auto verifier = make_lw_shared<std::unordered_map<unsigned long, status>>();
     auto descriptor = make_lw_shared<sstable_descriptor>();
 
-    return do_with(std::vector<future<>>(), [this, sstdir, verifier, descriptor] (std::vector<future<>>& futures) {
-        return lister::scan_dir(sstdir, { directory_entry_type::regular }, [this, sstdir, verifier, descriptor, &futures] (directory_entry de) {
-            // FIXME: The secondary indexes are in this level, but with a directory type, (starting with ".")
-            auto f = probe_file(sstdir, de.name).then([verifier, descriptor] (auto entry) {
-                if (verifier->count(entry.generation)) {
-                    if (verifier->at(entry.generation) == status::has_toc_file) {
-                        if (entry.component == sstables::sstable::component_type::TOC) {
-                            throw sstables::malformed_sstable_exception("Invalid State encountered. TOC file already processed");
-                        } else if (entry.component == sstables::sstable::component_type::TemporaryTOC) {
-                            throw sstables::malformed_sstable_exception("Invalid State encountered. Temporary TOC file found after TOC file was processed");
-                        }
-                    } else if (entry.component == sstables::sstable::component_type::TOC) {
-                        verifier->at(entry.generation) = status::has_toc_file;
-                    } else if (entry.component == sstables::sstable::component_type::TemporaryTOC) {
-                        verifier->at(entry.generation) = status::has_temporary_toc_file;
-                    }
-                } else {
+  return do_with(std::vector<future<>>(), [this, sstdir, verifier, descriptor] (std::vector<future<>>& futures) {
+    return lister::scan_dir(sstdir, { directory_entry_type::regular }, [this, sstdir, verifier, descriptor, &futures] (directory_entry de) {
+        // FIXME: The secondary indexes are in this level, but with a directory type, (starting with ".")
+        auto f = probe_file(sstdir, de.name).then([verifier, descriptor] (auto entry) {
+            if (verifier->count(entry.generation)) {
+                if (verifier->at(entry.generation) == status::has_toc_file) {
                     if (entry.component == sstables::sstable::component_type::TOC) {
-                        verifier->emplace(entry.generation, status::has_toc_file);
+                        throw sstables::malformed_sstable_exception("Invalid State encountered. TOC file already processed");
                     } else if (entry.component == sstables::sstable::component_type::TemporaryTOC) {
-                        verifier->emplace(entry.generation, status::has_temporary_toc_file);
-                    } else {
-                        verifier->emplace(entry.generation, status::has_some_file);
+                        throw sstables::malformed_sstable_exception("Invalid State encountered. Temporary TOC file found after TOC file was processed");
                     }
+                } else if (entry.component == sstables::sstable::component_type::TOC) {
+                    verifier->at(entry.generation) = status::has_toc_file;
+                } else if (entry.component == sstables::sstable::component_type::TemporaryTOC) {
+                    verifier->at(entry.generation) = status::has_temporary_toc_file;
                 }
-
-                // Retrieve both version and format used for this column family.
-                if (!descriptor->version) {
-                    descriptor->version = entry.version;
+            } else {
+                if (entry.component == sstables::sstable::component_type::TOC) {
+                    verifier->emplace(entry.generation, status::has_toc_file);
+                } else if (entry.component == sstables::sstable::component_type::TemporaryTOC) {
+                    verifier->emplace(entry.generation, status::has_temporary_toc_file);
+                } else {
+                    verifier->emplace(entry.generation, status::has_some_file);
                 }
-                if (!descriptor->format) {
-                    descriptor->format = entry.format;
+            }
+
+            // Retrieve both version and format used for this column family.
+            if (!descriptor->version) {
+                descriptor->version = entry.version;
+            }
+            if (!descriptor->format) {
+                descriptor->format = entry.format;
+            }
+        });
+
+        // push future returned by probe_file into an array of futures,
+        // so that the supplied callback will not block scan_dir() from
+        // reading the next entry in the directory.
+        futures.push_back(std::move(f));
+
+        return make_ready_future<>();
+    }).then([&futures] {
+        return when_all(futures.begin(), futures.end()).then([] (std::vector<future<>> ret) {
+            try {
+                for (auto& f : ret) {
+                    f.get();
                 }
-            });
-
-            // push future returned by probe_file into an array of futures,
-            // so that the supplied callback will not block scan_dir() from
-            // reading the next entry in the directory.
-            futures.push_back(std::move(f));
+            } catch(...) {
+                throw;
+            }
+        });
+    }).then([verifier, sstdir, descriptor, this] {
+        return parallel_for_each(*verifier, [sstdir = std::move(sstdir), descriptor, this] (auto v) {
+            if (v.second == status::has_temporary_toc_file) {
+                unsigned long gen = v.first;
+                assert(descriptor->version);
+                sstables::sstable::version_types version = descriptor->version.value();
+                assert(descriptor->format);
+                sstables::sstable::format_types format = descriptor->format.value();
 
+                if (engine().cpu_id() != 0) {
+                    dblog.info("At directory: {}, partial SSTable with generation {} not relevant for this shard, ignoring", sstdir, v.first);
+                    return make_ready_future<>();
+                }
+                // shard 0 is the responsible for removing a partial sstable.
+                return sstables::sstable::remove_sstable_with_temp_toc(_schema->ks_name(), _schema->cf_name(), sstdir, gen, version, format);
+            } else if (v.second != status::has_toc_file) {
+                throw sstables::malformed_sstable_exception(sprint("At directory: %s: no TOC found for SSTable with generation %d!. Refusing to boot", sstdir, v.first));
+            }
             return make_ready_future<>();
-        }, &manifest_json_filter).then([&futures] {
-            return when_all(futures.begin(), futures.end()).then([] (std::vector<future<>> ret) {
-                try {
-                    for (auto& f : ret) {
-                        f.get();
-                    }
-                } catch(...) {
-                    throw;
-                }
-            });
-        }).then([verifier, sstdir, descriptor, this] {
-            return parallel_for_each(*verifier, [sstdir = std::move(sstdir), descriptor, this] (auto v) {
-                if (v.second == status::has_temporary_toc_file) {
-                    unsigned long gen = v.first;
-                    assert(descriptor->version);
-                    sstables::sstable::version_types version = descriptor->version.value();
-                    assert(descriptor->format);
-                    sstables::sstable::format_types format = descriptor->format.value();
-
-                    if (engine().cpu_id() != 0) {
-                        dblog.debug("At directory: {}, partial SSTable with generation {} not relevant for this shard, ignoring", sstdir, v.first);
-                        return make_ready_future<>();
-                    }
-                    // shard 0 is the responsible for removing a partial sstable.
-                    return sstables::sstable::remove_sstable_with_temp_toc(_schema->ks_name(), _schema->cf_name(), sstdir, gen, version, format);
-                } else if (v.second != status::has_toc_file) {
-                    throw sstables::malformed_sstable_exception(sprint("At directory: %s: no TOC found for SSTable with generation %d!. Refusing to boot", sstdir, v.first));
-                }
-                return make_ready_future<>();
-            });
         });
     });
+  });
 }
 
 utils::UUID database::empty_version = utils::UUID_gen::get_name_UUID(bytes{});
@@ -1176,7 +1143,7 @@ void database::add_keyspace(sstring name, keyspace k) {
 }
 
 void database::update_keyspace(const sstring& name) {
-    throw std::runtime_error("update keyspace not implemented");
+    throw std::runtime_error("not implemented");
 }
 
 void database::drop_keyspace(const sstring& name) {
@@ -1827,36 +1794,6 @@ const sstring& database::get_snitch_name() const {
     return _cfg->endpoint_snitch();
 }
 
-// For the filesystem operations, this code will assume that all keyspaces are visible in all shards
-// (as we have been doing for a lot of the other operations, like the snapshot itself).
-future<> database::clear_snapshot(sstring tag, std::vector<sstring> keyspace_names) {
-    std::vector<std::reference_wrapper<keyspace>> keyspaces;
-
-    if (keyspace_names.empty()) {
-        // if keyspace names are not given - apply to all existing local keyspaces
-        for (auto& ks: _keyspaces) {
-            keyspaces.push_back(std::reference_wrapper<keyspace>(ks.second));
-        }
-    } else {
-        for (auto& ksname: keyspace_names) {
-            try {
-                keyspaces.push_back(std::reference_wrapper<keyspace>(find_keyspace(ksname)));
-            } catch (no_such_keyspace& e) {
-                return make_exception_future(std::current_exception());
-            }
-        }
-    }
-
-    return parallel_for_each(keyspaces, [this, tag] (auto& ks) {
-        return parallel_for_each(ks.get().metadata()->cf_meta_data(), [this, tag] (auto& pair) {
-            auto& cf = this->find_column_family(pair.second);
-            return cf.clear_snapshot(tag);
-         }).then_wrapped([] (future<> f) {
-            dblog.debug("Cleared out snapshot directories");
-         });
-    });
-}
-
 future<> update_schema_version_and_announce(distributed<service::storage_proxy>& proxy)
 {
     return db::schema_tables::calculate_schema_digest(proxy).then([&proxy] (utils::UUID uuid) {
@@ -1921,7 +1858,7 @@ seal_snapshot(sstring jsondir) {
     dblog.debug("Storing manifest {}", jsonfile);
 
     return recursive_touch_directory(jsondir).then([jsonfile, json = std::move(json)] {
-        return open_file_dma(jsonfile, open_flags::wo | open_flags::create | open_flags::truncate).then([json](file f) {
+        return engine().open_file_dma(jsonfile, open_flags::wo | open_flags::create | open_flags::truncate).then([json](file f) {
             return do_with(make_file_output_stream(std::move(f)), [json] (output_stream<char>& out) {
                 return out.write(json.c_str(), json.size()).then([&out] {
                    return out.flush();
@@ -2008,7 +1945,7 @@ future<> column_family::snapshot(sstring name) {
 }
 
 future<bool> column_family::snapshot_exists(sstring tag) {
-    sstring jsondir = _config.datadir + "/snapshots/" + tag;
+    sstring jsondir = _config.datadir + "/snapshots/";
     return engine().open_directory(std::move(jsondir)).then_wrapped([] (future<file> f) {
         try {
             f.get0();

database.hh

@@ -194,7 +194,8 @@ private:
     mutation_source sstables_as_mutation_source();
     key_source sstables_as_key_source() const;
     partition_presence_checker make_partition_presence_checker(lw_shared_ptr<sstable_list> old_sstables);
-    std::chrono::steady_clock::time_point _sstable_writes_disabled_at;
+    // We will use highres because hopefully it won't take more than a few usecs
+    std::chrono::high_resolution_clock::time_point _sstable_writes_disabled_at;
 public:
     // Creates a mutation reader which covers all data sources for this column family.
     // Caller needs to ensure that column_family remains live (FIXME: relax this).
@@ -215,10 +216,6 @@ public:
         return _cache;
     }
 
-    row_cache& get_row_cache() {
-        return _cache;
-    }
-
     logalloc::occupancy_stats occupancy() const;
 public:
     column_family(schema_ptr schema, config cfg, db::commitlog& cl, compaction_manager&);
@@ -250,7 +247,7 @@ public:
     // to call this separately in all shards first, to guarantee that none of them are writing
     // new data before you can safely assume that the whole node is disabled.
     future<int64_t> disable_sstable_write() {
-        _sstable_writes_disabled_at = std::chrono::steady_clock::now();
+        _sstable_writes_disabled_at = std::chrono::high_resolution_clock::now();
         return _sstables_lock.write_lock().then([this] {
             return make_ready_future<int64_t>((*_sstables->end()).first);
         });
@@ -258,10 +255,10 @@ public:
 
     // SSTable writes are now allowed again, and generation is updated to new_generation
     // returns the amount of microseconds elapsed since we disabled writes.
-    std::chrono::steady_clock::duration enable_sstable_write(int64_t new_generation) {
+    std::chrono::high_resolution_clock::duration enable_sstable_write(int64_t new_generation) {
         update_sstables_known_generation(new_generation);
         _sstables_lock.write_unlock();
-        return std::chrono::steady_clock::now() - _sstable_writes_disabled_at;
+        return std::chrono::high_resolution_clock::now() - _sstable_writes_disabled_at;
     }
 
     // Make sure the generation numbers are sequential, starting from "start".
@@ -324,10 +321,6 @@ public:
         return _stats;
     }
 
-    compaction_manager& get_compaction_manager() const {
-        return _compaction_manager;
-    }
-
     template<typename Func, typename Result = futurize_t<std::result_of_t<Func()>>>
     Result run_with_compaction_disabled(Func && func) {
         ++_compaction_disabled;
@@ -351,9 +344,6 @@ private:
     // one are also complete
     future<> seal_active_memtable();
 
-    // filter manifest.json files out
-    static bool manifest_json_filter(const sstring& fname);
-
     seastar::gate _in_flight_seals;
 
     // Iterate over all partitions.  Protocol is the same as std::all_of(),
@@ -572,9 +562,6 @@ public:
         return _commitlog.get();
     }
 
-    compaction_manager& get_compaction_manager() {
-        return _compaction_manager;
-    }
     const compaction_manager& get_compaction_manager() const {
         return _compaction_manager;
     }
@@ -624,7 +611,6 @@ public:
     future<> apply(const frozen_mutation&);
     keyspace::config make_keyspace_config(const keyspace_metadata& ksm);
     const sstring& get_snitch_name() const;
-    future<> clear_snapshot(sstring tag, std::vector<sstring> keyspace_names);
 
     friend std::ostream& operator<<(std::ostream& out, const database& db);
     const std::unordered_map<sstring, keyspace>& get_keyspaces() const {

database_fwd.hh

@@ -35,8 +35,8 @@ class column_definition;
 // keys.hh
 class exploded_clustering_prefix;
 class partition_key;
+class clustering_key;
 class clustering_key_prefix;
-using clustering_key = clustering_key_prefix;
 
 // memtable.hh
 class memtable;

db/batchlog_manager.cc

@@ -56,7 +56,6 @@
 #include "unimplemented.hh"
 #include "db/config.hh"
 #include "gms/failure_detector.hh"
-#include "service/storage_service.hh"
 
 static logging::logger logger("batchlog_manager");
 
@@ -88,8 +87,10 @@ future<> db::batchlog_manager::start() {
                                 );
                             });
                 });
-        auto ring_delay = service::get_local_storage_service().get_ring_delay();
-        _timer.arm(lowres_clock::now() + ring_delay);
+        _timer.arm(
+                lowres_clock::now()
+                        + std::chrono::milliseconds(
+                                service::storage_service::RING_DELAY));
     }
     return make_ready_future<>();
 }
@@ -114,7 +115,7 @@ mutation db::batchlog_manager::get_batch_log_mutation_for(const std::vector<muta
 mutation db::batchlog_manager::get_batch_log_mutation_for(const std::vector<mutation>& mutations, const utils::UUID& id, int32_t version, db_clock::time_point now) {
     auto schema = _qp.db().local().find_schema(system_keyspace::NAME, system_keyspace::BATCHLOG);
     auto key = partition_key::from_singular(*schema, id);
-    auto timestamp = api::new_timestamp();
+    auto timestamp = db_clock::now_in_usecs();
     auto data = [this, &mutations] {
         std::vector<frozen_mutation> fm(mutations.begin(), mutations.end());
         const auto size = std::accumulate(fm.begin(), fm.end(), size_t(0), [](size_t s, auto& m) {

db/commitlog/commitlog.cc

@@ -281,43 +281,6 @@ private:
  * A single commit log file on disk. Manages creation of the file and writing mutations to disk,
  * as well as tracking the last mutation position of any "dirty" CFs covered by the segment file. Segment
  * files are initially allocated to a fixed size and can grow to accomidate a larger value if necessary.
- *
- * The IO flow is somewhat convoluted and goes something like this:
- *
- * Mutation path:
- *  - Adding data to the segment usually writes into the internal buffer
- *  - On EOB or overflow we issue a write to disk ("cycle").
- *      - A cycle call will acquire the segment read lock and send the
- *        buffer to the corresponding position in the file
- *  - If we are periodic and crossed a timing threshold, or running "batch" mode
- *    we might be forced to issue a flush ("sync") after adding data
- *      - A sync call acquires the write lock, thus locking out writes
- *        and waiting for pending writes to finish. It then checks the
- *        high data mark, and issues the actual file flush.
- *        Note that the write lock is released prior to issuing the
- *        actual file flush, thus we are allowed to write data to
- *        after a flush point concurrently with a pending flush.
- *
- * Sync timer:
- *  - In periodic mode, we try to primarily issue sync calls in
- *    a timer task issued every N seconds. The timer does the same
- *    operation as the above described sync, and resets the timeout
- *    so that mutation path will not trigger syncs and delay.
- *
- * Note that we do not care which order segment chunks finish writing
- * to disk, other than all below a flush point must finish before flushing.
- *
- * We currently do not wait for flushes to finish before issueing the next
- * cycle call ("after" flush point in the file). This might not be optimal.
- *
- * To close and finish a segment, we first close the gate object that guards
- * writing data to it, then flush it fully (including waiting for futures create
- * by the timer to run their course), and finally wait for it to
- * become "clean", i.e. get notified that all mutations it holds have been
- * persisted to sstables elsewhere. Once this is done, we can delete the
- * segment. If a segment (object) is deleted without being fully clean, we
- * do not remove the file on disk.
- *
  */
 
 class db::commitlog::segment: public enable_lw_shared_from_this<segment> {
@@ -407,7 +370,6 @@ public:
     void reset_sync_time() {
         _sync_time = clock_type::now();
     }
-    // See class comment for info
     future<sseg_ptr> sync() {
         // Note: this is not a marker for when sync was finished.
         // It is when it was initiated
@@ -424,7 +386,6 @@ public:
     future<> shutdown() {
         return _gate.close();
     }
-    // See class comment for info
     future<sseg_ptr> flush(uint64_t pos = 0) {
         auto me = shared_from_this();
         assert(!me.owned());
@@ -470,7 +431,6 @@ public:
     /**
      * Send any buffer contents to disk and get a new tmp buffer
      */
-    // See class comment for info
     future<sseg_ptr> cycle(size_t s = 0) {
         auto size = clear_buffer_slack();
         auto buf = std::move(_buffer);
@@ -887,7 +847,7 @@ void db::commitlog::segment_manager::flush_segments(bool force) {
 
 future<db::commitlog::segment_manager::sseg_ptr> db::commitlog::segment_manager::allocate_segment(bool active) {
     descriptor d(next_id());
-    return open_file_dma(cfg.commit_log_location + "/" + d.filename(), open_flags::wo | open_flags::create).then([this, d, active](file f) {
+    return engine().open_file_dma(cfg.commit_log_location + "/" + d.filename(), open_flags::wo | open_flags::create).then([this, d, active](file f) {
         // xfs doesn't like files extended betond eof, so enlarge the file
         return f.truncate(max_size).then([this, d, active, f] () mutable {
             auto s = make_lw_shared<segment>(this, d, std::move(f), active);
@@ -1215,7 +1175,7 @@ const db::commitlog::config& db::commitlog::active_config() const {
 
 future<std::unique_ptr<subscription<temporary_buffer<char>, db::replay_position>>>
 db::commitlog::read_log_file(const sstring& filename, commit_load_reader_func next, position_type off) {
-    return open_file_dma(filename, open_flags::ro).then([next = std::move(next), off](file f) {
+    return engine().open_file_dma(filename, open_flags::ro).then([next = std::move(next), off](file f) {
        return std::make_unique<subscription<temporary_buffer<char>, replay_position>>(
            read_log_file(std::move(f), std::move(next), off));
     });
@@ -1350,17 +1310,6 @@ db::commitlog::read_log_file(file f, commit_load_reader_func next, position_type
         }
         future<> read_entry() {
             static constexpr size_t entry_header_size = segment::entry_overhead_size - sizeof(uint32_t);
-
-            /**
-             * #598 - Must check that data left in chunk is enough to even read an entry.
-             * If not, this is small slack space in the chunk end, and we should just go
-             * to the next.
-             */
-            assert(pos <= next);
-            if ((pos + entry_header_size) >= next) {
-                return skip(next - pos);
-            }
-
             return fin.read_exactly(entry_header_size).then([this](temporary_buffer<char> buf) {
                 replay_position rp(id, position_type(pos));
 

db/config.cc

@@ -117,9 +117,8 @@ template<typename K, typename V>
 struct convert<std::unordered_map<K, V>> {
     static Node encode(const std::unordered_map<K, V>& rhs) {
         Node node(NodeType::Map);
-        for (auto& p : rhs) {
-            node.force_insert(p.first, p.second);
-        }
+        for(typename std::map<K, V>::const_iterator it=rhs.begin();it!=rhs.end();++it)
+            node.force_insert(it->first, it->second);
         return node;
     }
     static bool decode(const Node& node, std::unordered_map<K, V>& rhs) {
@@ -410,25 +409,7 @@ future<> db::config::read_from_file(file f) {
 }
 
 future<> db::config::read_from_file(const sstring& filename) {
-    return open_file_dma(filename, open_flags::ro).then([this](file f) {
+    return engine().open_file_dma(filename, open_flags::ro).then([this](file f) {
        return read_from_file(std::move(f));
     });
 }
-
-boost::filesystem::path db::config::get_conf_dir() {
-    using namespace boost::filesystem;
-
-    path confdir;
-    auto* cd = std::getenv("SCYLLA_CONF");
-    if (cd != nullptr) {
-        confdir = path(cd);
-    } else {
-        auto* p = std::getenv("SCYLLA_HOME");
-        if (p != nullptr) {
-            confdir = path(p);
-        }
-        confdir /= "conf";
-    }
-
-    return confdir;
-}

db/config.hh

@@ -121,7 +121,23 @@ public:
      * @return path of the directory where configuration files are located
      *         according the environment variables definitions.
      */
-    static boost::filesystem::path get_conf_dir();
+    static boost::filesystem::path get_conf_dir() {
+        using namespace boost::filesystem;
+
+        path confdir;
+        auto* cd = std::getenv("SCYLLA_CONF");
+        if (cd != nullptr) {
+            confdir = path(cd);
+        } else {
+            auto* p = std::getenv("SCYLLA_HOME");
+            if (p != nullptr) {
+                confdir = path(p);
+            }
+            confdir /= "conf";
+        }
+
+        return confdir;
+    }
 
     typedef std::unordered_map<sstring, sstring> string_map;
     typedef std::vector<sstring> string_list;
@@ -666,7 +682,7 @@ public:
     val(permissions_update_interval_in_ms, uint32_t, 2000, Unused,     \
             "Refresh interval for permissions cache (if enabled). After this interval, cache entries become eligible for refresh. On next access, an async reload is scheduled and the old value is returned until it completes. If permissions_validity_in_ms , then this property must benon-zero."   \
     )   \
-    val(server_encryption_options, string_map, /*none*/, Used,     \
+    val(server_encryption_options, string_map, /*none*/, Unused,     \
             "Enable or disable inter-node encryption. You must also generate keys and provide the appropriate key and trust store locations and passwords. No custom encryption options are currently enabled. The available options are:\n"    \
             "\n"    \
             "internode_encryption : (Default: none ) Enable or disable encryption of inter-node communication using the TLS_RSA_WITH_AES_128_CBC_SHA cipher suite for authentication, key exchange, and encryption of data transfers. The available inter-node options are:\n"  \
@@ -674,20 +690,41 @@ public:
             "\tnone : No encryption.\n" \
             "\tdc : Encrypt the traffic between the data centers (server only).\n"  \
             "\track : Encrypt the traffic between the racks(server only).\n"    \
-            "certificate : (Default: conf/scylla.crt) The location of a PEM-encoded x509 certificate used to identify and encrypt the internode communication.\n"    \
-            "keyfile : (Default: conf/scylla.key) PEM Key file associated with certificate.\n"  \
-            "truststore : (Default: <system truststore> ) Location of the truststore containing the trusted certificate for authenticating remote servers.\n"    \
+            "\tkeystore : (Default: conf/.keystore ) The location of a Java keystore (JKS) suitable for use with Java Secure Socket Extension (JSSE), which is the Java version of the Secure Sockets Layer (SSL), and Transport Layer Security (TLS) protocols. The keystore contains the private key used to encrypt outgoing messages.\n"    \
+            "\tkeystore_password : (Default: cassandra ) Password for the keystore.\n"  \
+            "\ttruststore : (Default: conf/.truststore ) Location of the truststore containing the trusted certificate for authenticating remote servers.\n"    \
+            "\ttruststore_password : (Default: cassandra ) Password for the truststore.\n"  \
+            "\n"    \
+            "The passwords used in these options must match the passwords used when generating the keystore and truststore. For instructions on generating these files, see Creating a Keystore to Use with JSSE.\n"   \
+            "\n"    \
+            "The advanced settings are:\n"  \
+            "\n"    \
+            "\tprotocol : (Default: TLS )\n"    \
+            "\talgorithm : (Default: SunX509 )\n"   \
+            "\tstore_type : (Default: JKS )\n"  \
+            "\tcipher_suites : (Default: TLS_RSA_WITH_AES_128_CBC_SHA , TLS_RSA_WITH_AES_256_CBC_SHA )\n"   \
+            "\trequire_client_auth : (Default: false ) Enables or disables certificate authentication.\n" \
             "Related information: Node-to-node encryption"  \
     )   \
-    val(client_encryption_options, string_map, /*none*/, Used,     \
-            "Enable or disable client-to-node encryption. You must also generate keys and provide the appropriate key and certificate. No custom encryption options are currently enabled. The available options are:\n"    \
+    val(client_encryption_options, string_map, /*none*/, Unused,     \
+            "Enable or disable client-to-node encryption. You must also generate keys and provide the appropriate key and trust store locations and passwords. No custom encryption options are currently enabled. The available options are:\n"    \
             "\n"    \
             "\tenabled : (Default: false ) To enable, set to true.\n"    \
-            "\tcertificate: (Default: conf/scylla.crt) The location of a PEM-encoded x509 certificate used to identify and encrypt the client/server communication.\n"   \
-            "\tkeyfile: (Default: conf/scylla.key) PEM Key file associated with certificate.\n"   \
+            "\tkeystore : (Default: conf/.keystore ) The location of a Java keystore (JKS) suitable for use with Java Secure Socket Extension (JSSE), which is the Java version of the Secure Sockets Layer (SSL), and Transport Layer Security (TLS) protocols. The keystore contains the private key used to encrypt outgoing messages.\n"    \
+            "\tkeystore_password : (Default: cassandra ) Password for the keystore. This must match the password used when generating the keystore and truststore.\n"    \
+            "\trequire_client_auth : (Default: false ) Enables or disables certificate authentication. (Available starting with Cassandra 1.2.3.)\n"    \
+            "\ttruststore : (Default: conf/.truststore ) Set if require_client_auth is true.\n"    \
+            "\ttruststore_password : <truststore_password> Set if require_client_auth is true.\n"    \
+            "\n"    \
+            "The advanced settings are:\n"    \
+            "\n"    \
+            "\tprotocol : (Default: TLS )\n"    \
+            "\talgorithm : (Default: SunX509 )\n"    \
+            "\tstore_type : (Default: JKS )\n"    \
+            "\tcipher_suites : (Default: TLS_RSA_WITH_AES_128_CBC_SHA , TLS_RSA_WITH_AES_256_CBC_SHA )\n"  \
             "Related information: Client-to-node encryption"    \
     )   \
-    val(ssl_storage_port, uint32_t, 7001, Used,     \
+    val(ssl_storage_port, uint32_t, 7001, Unused,     \
             "The SSL port for encrypted communication. Unused unless enabled in encryption_options."  \
     )                                                   \
     val(default_log_level, sstring, "warn", Used, \
@@ -713,9 +750,6 @@ public:
     val(replace_token, sstring, "", Used, "The tokens of the node to replace. Same as -Dcassandra.replace_token in cassandra.") \
     val(replace_address, sstring, "", Used, "The listen_address or broadcast_address of the dead node to replace. Same as -Dcassandra.replace_address.") \
     val(replace_address_first_boot, sstring, "", Used, "Like replace_address option, but if the node has been bootstrapped sucessfully it will be ignored. Same as -Dcassandra.replace_address_first_boot.") \
-    val(override_decommission, bool, false, Used, "Set true to force a decommissioned node to join the cluster") \
-    val(ring_delay_ms, uint32_t, 30 * 1000, Used, "Time a node waits to hear from other nodes before joining the ring in milliseconds. Same as -Dcassandra.ring_delay_ms in cassandra.") \
-    val(developer_mode, bool, false, Used, "Relax environement checks. Setting to true can reduce performance and reliability significantly.") \
     /* done! */
 
 #define _make_value_member(name, type, deflt, status, desc, ...)    \

db/schema_tables.cc

@@ -1269,7 +1269,7 @@ void create_table_from_table_row_and_column_rows(schema_builder& builder, const
     } else {
         // FIXME:
         // is_dense = CFMetaData.calculateIsDense(fullRawComparator, columnDefs);
-        throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+        throw std::runtime_error("not implemented");
     }
 
     bool is_compound = cell_comparator::check_compound(table_row.get_nonnull<sstring>("comparator"));

db/serializer.cc

@@ -187,6 +187,30 @@ void db::serializer<partition_key_view>::skip(input& in) {
     in.skip(len);
 }
 
+template<>
+db::serializer<clustering_key_view>::serializer(const clustering_key_view& key)
+    : _item(key), _size(sizeof(uint16_t) /* size */ + key.representation().size()) {
+}
+
+template<>
+void db::serializer<clustering_key_view>::write(output& out, const clustering_key_view& key) {
+    bytes_view v = key.representation();
+    out.write<uint16_t>(v.size());
+    out.write(v.begin(), v.end());
+}
+
+template<>
+void db::serializer<clustering_key_view>::read(clustering_key_view& b, input& in) {
+    auto len = in.read<uint16_t>();
+    b = clustering_key_view::from_bytes(in.read_view(len));
+}
+
+template<>
+clustering_key_view db::serializer<clustering_key_view>::read(input& in) {
+    auto len = in.read<uint16_t>();
+    return clustering_key_view::from_bytes(in.read_view(len));
+}
+
 template<>
 db::serializer<clustering_key_prefix_view>::serializer(const clustering_key_prefix_view& key)
     : _item(key), _size(sizeof(uint16_t) /* size */ + key.representation().size()) {
@@ -257,6 +281,7 @@ template class db::serializer<atomic_cell_view> ;
 template class db::serializer<collection_mutation_view> ;
 template class db::serializer<utils::UUID> ;
 template class db::serializer<partition_key_view> ;
+template class db::serializer<clustering_key_view> ;
 template class db::serializer<clustering_key_prefix_view> ;
 template class db::serializer<frozen_mutation> ;
 template class db::serializer<db::replay_position> ;

db/serializer.hh

@@ -22,8 +22,6 @@
 #ifndef DB_SERIALIZER_HH_
 #define DB_SERIALIZER_HH_
 
-#include <experimental/optional>
-
 #include "utils/data_input.hh"
 #include "utils/data_output.hh"
 #include "bytes_ostream.hh"
@@ -59,9 +57,9 @@ public:
         return *this;
     }
 
-    static void write(output&, const type&);
-    static void read(type&, input&);
-    static type read(input&);
+    static void write(output&, const T&);
+    static void read(T&, input&);
+    static T read(input&);
     static void skip(input& in);
 
     size_t size() const {
@@ -77,100 +75,11 @@ public:
     void write(data_output& out) const {
         write(out, _item);
     }
-
-    bytes to_bytes() const {
-        bytes b(bytes::initialized_later(), _size);
-        data_output out(b);
-        write(out);
-        return b;
-    }
-
-    static type from_bytes(bytes_view v) {
-        data_input in(v);
-        return read(in);
-    }
 private:
-    const type& _item;
+    const T& _item;
     size_t _size;
 };
 
-template<typename T>
-class serializer<std::experimental::optional<T>> {
-public:
-    typedef std::experimental::optional<T> type;
-    typedef data_output output;
-    typedef data_input input;
-    typedef serializer<T> _MyType;
-
-    serializer(const type& t)
-        : _item(t)
-        , _size(output::serialized_size<bool>() + (t ? serializer<T>(*t).size() : 0))
-    {}
-
-    // apply to memory, must be at least size() large.
-    const _MyType& operator()(output& out) const {
-        write(out, _item);
-        return *this;
-    }
-
-    static void write(output& out, const type& v) {
-        bool en = v;
-        out.write<bool>(en);
-        if (en) {
-            serializer<T>::write(out, *v);
-        }
-    }
-    static void read(type& dst, input& in) {
-        auto en = in.read<bool>();
-        if (en) {
-            dst = serializer<T>::read(in);
-        } else {
-            dst = {};
-        }
-    }
-    static type read(input& in) {
-        type t;
-        read(t, in);
-        return t;
-    }
-    static void skip(input& in) {
-        auto en = in.read<bool>();
-        if (en) {
-            serializer<T>::skip(in);
-        }
-    }
-
-    size_t size() const {
-        return _size;
-    }
-
-    void write(bytes_ostream& out) const {
-        auto buf = out.write_place_holder(_size);
-        data_output data_out((char*)buf, _size);
-        write(data_out, _item);
-    }
-
-    void write(data_output& out) const {
-        write(out, _item);
-    }
-
-    bytes to_bytes() const {
-        bytes b(bytes::initialized_later(), _size);
-        data_output out(b);
-        write(out);
-        return b;
-    }
-
-    static type from_bytes(bytes_view v) {
-        data_input in(v);
-        return read(in);
-    }
-private:
-    const std::experimental::optional<T> _item;
-    size_t _size;
-};
-
-
 template<> serializer<utils::UUID>::serializer(const utils::UUID &);
 template<> void serializer<utils::UUID>::write(output&, const type&);
 template<> void serializer<utils::UUID>::read(utils::UUID&, input&);
@@ -214,6 +123,11 @@ template<> void serializer<partition_key_view>::read(partition_key_view&, input&
 template<> partition_key_view serializer<partition_key_view>::read(input&);
 template<> void serializer<partition_key_view>::skip(input&);
 
+template<> serializer<clustering_key_view>::serializer(const clustering_key_view &);
+template<> void serializer<clustering_key_view>::write(output&, const clustering_key_view&);
+template<> void serializer<clustering_key_view>::read(clustering_key_view&, input&);
+template<> clustering_key_view serializer<clustering_key_view>::read(input&);
+
 template<> serializer<clustering_key_prefix_view>::serializer(const clustering_key_prefix_view &);
 template<> void serializer<clustering_key_prefix_view>::write(output&, const clustering_key_prefix_view&);
 template<> void serializer<clustering_key_prefix_view>::read(clustering_key_prefix_view&, input&);

db/system_keyspace.cc

@@ -464,8 +464,7 @@ static future<> build_bootstrap_info() {
         static auto state_map = std::unordered_map<sstring, bootstrap_state>({
             { "NEEDS_BOOTSTRAP", bootstrap_state::NEEDS_BOOTSTRAP },
             { "COMPLETED", bootstrap_state::COMPLETED },
-            { "IN_PROGRESS", bootstrap_state::IN_PROGRESS },
-            { "DECOMMISSIONED", bootstrap_state::DECOMMISSIONED }
+            { "IN_PROGRESS", bootstrap_state::IN_PROGRESS }
         });
         bootstrap_state state = bootstrap_state::NEEDS_BOOTSTRAP;
 
@@ -486,10 +485,6 @@ future<> init_local_cache() {
     });
 }
 
-future<> deinit_local_cache() {
-    return _local_cache.stop();
-}
-
 void minimal_setup(distributed<database>& db, distributed<cql3::query_processor>& qp) {
     qctx = std::make_unique<query_context>(db, qp);
 }
@@ -801,8 +796,6 @@ future<> remove_endpoint(gms::inet_address ep) {
     }).then([ep] {
         sstring req = "DELETE FROM system.%s WHERE peer = ?";
         return execute_cql(req, PEERS, ep.addr()).discard_result();
-    }).then([] {
-        return force_blocking_flush(PEERS);
     });
 }
 
@@ -881,10 +874,6 @@ bool bootstrap_in_progress() {
     return get_bootstrap_state() == bootstrap_state::IN_PROGRESS;
 }
 
-bool was_decommissioned() {
-    return get_bootstrap_state() == bootstrap_state::DECOMMISSIONED;
-}
-
 bootstrap_state get_bootstrap_state() {
     return _local_cache.local()._state;
 }
@@ -893,8 +882,7 @@ future<> set_bootstrap_state(bootstrap_state state) {
     static std::unordered_map<bootstrap_state, sstring, enum_hash<bootstrap_state>> state_to_name({
         { bootstrap_state::NEEDS_BOOTSTRAP, "NEEDS_BOOTSTRAP" },
         { bootstrap_state::COMPLETED, "COMPLETED" },
-        { bootstrap_state::IN_PROGRESS, "IN_PROGRESS" },
-        { bootstrap_state::DECOMMISSIONED, "DECOMMISSIONED" }
+        { bootstrap_state::IN_PROGRESS, "IN_PROGRESS" }
     });
 
     sstring state_name = state_to_name.at(state);
@@ -1014,55 +1002,5 @@ query(distributed<service::storage_proxy>& proxy, const sstring& cf_name, const
     });
 }
 
-static map_type_impl::native_type prepare_rows_merged(std::unordered_map<int32_t, int64_t>& rows_merged) {
-    map_type_impl::native_type tmp;
-    for (auto& r: rows_merged) {
-        int32_t first = r.first;
-        int64_t second = r.second;
-        auto map_element = std::make_pair<data_value, data_value>(data_value(first), data_value(second));
-        tmp.push_back(std::move(map_element));
-    }
-    return tmp;
-}
-
-future<> update_compaction_history(sstring ksname, sstring cfname, int64_t compacted_at, int64_t bytes_in, int64_t bytes_out,
-                                   std::unordered_map<int32_t, int64_t> rows_merged)
-{
-    // don't write anything when the history table itself is compacted, since that would in turn cause new compactions
-    if (ksname == "system" && cfname == COMPACTION_HISTORY) {
-        return make_ready_future<>();
-    }
-
-    auto map_type = map_type_impl::get_instance(int32_type, long_type, true);
-
-    sstring req = "INSERT INTO system.%s (id, keyspace_name, columnfamily_name, compacted_at, bytes_in, bytes_out, rows_merged) VALUES (?, ?, ?, ?, ?, ?, ?)";
-
-    return execute_cql(req, COMPACTION_HISTORY, utils::UUID_gen::get_time_UUID(), ksname, cfname, compacted_at, bytes_in, bytes_out,
-                       make_map_value(map_type, prepare_rows_merged(rows_merged))).discard_result();
-}
-
-future<std::vector<compaction_history_entry>> get_compaction_history()
-{
-    sstring req = "SELECT * from system.%s";
-    return execute_cql(req, COMPACTION_HISTORY).then([] (::shared_ptr<cql3::untyped_result_set> msg) {
-        std::vector<compaction_history_entry> history;
-
-        for (auto& row : *msg) {
-            compaction_history_entry entry;
-            entry.id = row.get_as<utils::UUID>("id");
-            entry.ks = row.get_as<sstring>("keyspace_name");
-            entry.cf = row.get_as<sstring>("columnfamily_name");
-            entry.compacted_at = row.get_as<int64_t>("compacted_at");
-            entry.bytes_in = row.get_as<int64_t>("bytes_in");
-            entry.bytes_out = row.get_as<int64_t>("bytes_out");
-            if (row.has("rows_merged")) {
-                entry.rows_merged = row.get_map<int32_t, int64_t>("rows_merged");
-            }
-            history.push_back(std::move(entry));
-        }
-        return std::move(history);
-    });
-}
-
 } // namespace system_keyspace
 } // namespace db

db/system_keyspace.hh

@@ -88,7 +88,6 @@ extern schema_ptr built_indexes(); // TODO (from Cassandra): make private
 void minimal_setup(distributed<database>& db, distributed<cql3::query_processor>& qp);
 
 future<> init_local_cache();
-future<> deinit_local_cache();
 future<> setup(distributed<database>& db, distributed<cql3::query_processor>& qp);
 future<> update_schema_version(utils::UUID version);
 future<> update_tokens(std::unordered_set<dht::token> tokens);
@@ -154,8 +153,7 @@ load_dc_rack_info();
 enum class bootstrap_state {
     NEEDS_BOOTSTRAP,
     COMPLETED,
-    IN_PROGRESS,
-    DECOMMISSIONED
+    IN_PROGRESS
 };
 
 #if 0
@@ -260,28 +258,26 @@ enum class bootstrap_state {
         compactionLog.truncateBlocking();
     }
 
+    public static void updateCompactionHistory(String ksname,
+                                               String cfname,
+                                               long compactedAt,
+                                               long bytesIn,
+                                               long bytesOut,
+                                               Map<Integer, Long> rowsMerged)
+    {
+        // don't write anything when the history table itself is compacted, since that would in turn cause new compactions
+        if (ksname.equals("system") && cfname.equals(COMPACTION_HISTORY))
+            return;
+        String req = "INSERT INTO system.%s (id, keyspace_name, columnfamily_name, compacted_at, bytes_in, bytes_out, rows_merged) VALUES (?, ?, ?, ?, ?, ?, ?)";
+        executeInternal(String.format(req, COMPACTION_HISTORY), UUIDGen.getTimeUUID(), ksname, cfname, ByteBufferUtil.bytes(compactedAt), bytesIn, bytesOut, rowsMerged);
+    }
+
     public static TabularData getCompactionHistory() throws OpenDataException
     {
         UntypedResultSet queryResultSet = executeInternal(String.format("SELECT * from system.%s", COMPACTION_HISTORY));
         return CompactionHistoryTabularData.from(queryResultSet);
     }
 #endif
-    struct compaction_history_entry {
-        utils::UUID id;
-        sstring ks;
-        sstring cf;
-        int64_t compacted_at = 0;
-        int64_t bytes_in = 0;
-        int64_t bytes_out = 0;
-        // Key: number of rows merged
-        // Value: counter
-        std::unordered_map<int32_t, int64_t> rows_merged;
-    };
-
-    future<> update_compaction_history(sstring ksname, sstring cfname, int64_t compacted_at, int64_t bytes_in, int64_t bytes_out,
-                                       std::unordered_map<int32_t, int64_t> rows_merged);
-    future<std::vector<compaction_history_entry>> get_compaction_history();
-
     typedef std::vector<db::replay_position> replay_positions;
 
     future<> save_truncation_record(const column_family&, db_clock::time_point truncated_at, db::replay_position);
@@ -523,7 +519,6 @@ enum class bootstrap_state {
 bool bootstrap_complete();
 bool bootstrap_in_progress();
 bootstrap_state get_bootstrap_state();
-bool was_decommissioned();
 future<> set_bootstrap_state(bootstrap_state state);
 
 #if 0

dht/byte_ordered_partitioner.cc

@@ -34,12 +34,12 @@ token byte_ordered_partitioner::get_random_token()
 
 std::map<token, float> byte_ordered_partitioner::describe_ownership(const std::vector<token>& sorted_tokens)
 {
-    throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+    throw std::runtime_error("not implemented");
 }
 
 token byte_ordered_partitioner::midpoint(const token& t1, const token& t2) const
 {
-    throw std::runtime_error(sprint("%s not implemented", __PRETTY_FUNCTION__));
+    throw std::runtime_error("not implemented");
 }
 
 unsigned

dht/i_partitioner.hh

@@ -386,22 +386,12 @@ public:
     friend std::ostream& operator<<(std::ostream&, const ring_position&);
 };
 
-// Trichotomic comparator for ring_position
 struct ring_position_comparator {
     const schema& s;
     ring_position_comparator(const schema& s_) : s(s_) {}
     int operator()(const ring_position& lh, const ring_position& rh) const;
 };
 
-// "less" comparator for ring_position
-struct ring_position_less_comparator {
-    const schema& s;
-    ring_position_less_comparator(const schema& s_) : s(s_) {}
-    bool operator()(const ring_position& lh, const ring_position& rh) const {
-        return lh.less_compare(s, rh);
-    }
-};
-
 struct token_comparator {
     // Return values are those of a trichotomic comparison.
     int operator()(const token& t1, const token& t2) const;

dht/murmur3_partitioner.cc

@@ -88,8 +88,18 @@ inline int64_t long_token(const token& t) {
     return net::ntoh(*lp);
 }
 
+// XXX: Technically, this should be inside long token. However, long_token is
+// used quite a lot in hot paths, so it is better to keep the branches of, if
+// we can. Most our comparators will check for _kind separately,
+// so this should be fine.
 sstring murmur3_partitioner::to_sstring(const token& t) const {
-    return ::to_sstring(long_token(t));
+    int64_t lt;
+    if (t._kind == dht::token::kind::before_all_keys) {
+        lt = std::numeric_limits<long>::min();
+    } else {
+        lt = long_token(t);
+    }
+    return ::to_sstring(lt);
 }
 
 dht::token murmur3_partitioner::from_sstring(const sstring& t) const {
@@ -112,35 +122,17 @@ int murmur3_partitioner::tri_compare(const token& t1, const token& t2) {
     }
 }
 
-// Assuming that x>=y, return the positive difference x-y.
-// The return type is an unsigned type, as the difference may overflow
-// a signed type (e.g., consider very positive x and very negative y).
-template <typename T>
-static std::make_unsigned_t<T> positive_subtract(T x, T y) {
-        return std::make_unsigned_t<T>(x) - std::make_unsigned_t<T>(y);
-}
-
 token murmur3_partitioner::midpoint(const token& t1, const token& t2) const {
     auto l1 = long_token(t1);
     auto l2 = long_token(t2);
-    int64_t mid;
-    if (l1 <= l2) {
-        // To find the midpoint, we cannot use the trivial formula (l1+l2)/2
-        // because the addition can overflow the integer. To avoid this
-        // overflow, we first notice that the above formula is equivalent to
-        // l1 + (l2-l1)/2. Now, "l2-l1" can still overflow a signed integer
-        // (e.g., think of a very positive l2 and very negative l1), but
-        // because l1 <= l2 in this branch, we note that l2-l1 is positive
-        // and fits an *unsigned* int's range. So,
-        mid = l1 + positive_subtract(l2, l1)/2;
-    } else {
-        // When l2 < l1, we need to switch l1 and and l2 in the above
-        // formula, because now l1 - l2 is positive.
-        // Additionally, we consider this case is a "wrap around", so we need
-        // to behave as if l2 + 2^64 was meant instead of l2, i.e., add 2^63
-        // to the average.
-        mid = l2 + positive_subtract(l1, l2)/2 + 0x8000'0000'0000'0000;
+    // long_token is defined as signed, but the arithmetic works out the same
+    // without invoking undefined behavior with a signed type.
+    auto delta = (uint64_t(l2) - uint64_t(l1)) / 2;
+    if (l1 > l2) {
+        // wraparound
+        delta += 0x8000'0000'0000'0000;
     }
+    auto mid = uint64_t(l1) + delta;
     return get_token(mid);
 }
 

dht/range_streamer.cc

@@ -241,11 +241,12 @@ future<streaming::stream_state> range_streamer::fetch_async() {
         for (auto& x : fetch.second) {
             auto& source = x.first;
             auto& ranges = x.second;
+            auto preferred = net::get_local_messaging_service().get_preferred_ip(source);
             /* Send messages to respective folks to stream data over to me */
             if (logger.is_enabled(logging::log_level::debug)) {
                 logger.debug("{}ing from {} ranges {}", _description, source, ranges);
             }
-            _stream_plan.request_ranges(source, keyspace, ranges);
+            _stream_plan.request_ranges(source, preferred, keyspace, ranges);
         }
     }
 

dist/ami/build_ami.sh

@@ -5,16 +5,6 @@ if [ ! -e dist/ami/build_ami.sh ]; then
     exit 1
 fi
 
-TARGET_JSON=scylla.json
-if [ "$1" != "" ]; then
-    TARGET_JSON=$1
-fi
-
-if [ ! -f dist/ami/$TARGET_JSON ]; then
-    echo "dist/ami/$TARGET_JSON does not found"
-    exit 1
-fi
-
 cd dist/ami
 
 if [ ! -f variables.json ]; then
@@ -30,4 +20,4 @@ if [ ! -d packer ]; then
     cd -
 fi
 
-packer/packer build -var-file=variables.json $TARGET_JSON
+packer/packer build -var-file=variables.json scylla.json

dist/ami/build_ami_local.sh

@@ -1,30 +0,0 @@
-#!/bin/sh -e
-
-if [ ! -e dist/ami/build_ami_local.sh ]; then
-    echo "run build_ami_local.sh in top of scylla dir"
-    exit 1
-fi
-
-sudo yum -y install git
-if [ ! -f dist/ami/scylla-server.x86_64.rpm ]; then
-    dist/redhat/build_rpm.sh
-    cp build/rpms/scylla-server-`cat build/SCYLLA-VERSION-FILE`-`cat build/SCYLLA-RELEASE-FILE`.*.x86_64.rpm dist/ami/scylla-server.x86_64.rpm
-fi
-if [ ! -f dist/ami/scylla-jmx.noarch.rpm ]; then
-    cd build
-    git clone --depth 1 https://github.com/scylladb/scylla-jmx.git
-    cd scylla-jmx
-    sh -x -e dist/redhat/build_rpm.sh
-    cd ../..
-    cp build/scylla-jmx/build/rpms/scylla-jmx-`cat build/scylla-jmx/build/SCYLLA-VERSION-FILE`-`cat build/scylla-jmx/build/SCYLLA-RELEASE-FILE`.*.noarch.rpm dist/ami/scylla-jmx.noarch.rpm
-fi
-if [ ! -f dist/ami/scylla-tools.noarch.rpm ]; then
-    cd build
-    git clone --depth 1 https://github.com/scylladb/scylla-tools-java.git
-    cd scylla-tools-java
-    sh -x -e dist/redhat/build_rpm.sh
-    cd ../..
-    cp build/scylla-tools-java/build/rpms/scylla-tools-`cat build/scylla-tools-java/build/SCYLLA-VERSION-FILE`-`cat build/scylla-tools-java/build/SCYLLA-RELEASE-FILE`.*.noarch.rpm dist/ami/scylla-tools.noarch.rpm
-fi
-
-exec dist/ami/build_ami.sh scylla_local.json

dist/ami/files/.bash_profile

@@ -1,45 +0,0 @@
-# .bash_profile
-
-# Get the aliases and functions
-if [ -f ~/.bashrc ]; then
-	. ~/.bashrc
-fi
-
-# User specific environment and startup programs
-
-PATH=$PATH:$HOME/.local/bin:$HOME/bin
-
-export PATH
-
-echo
-echo '   _____            _ _       _____  ____  '
-echo '  / ____|          | | |     |  __ \|  _ \ '
-echo ' | (___   ___ _   _| | | __ _| |  | | |_) |'
-echo '  \___ \ / __| | | | | |/ _` | |  | |  _ < '
-echo '  ____) | (__| |_| | | | (_| | |__| | |_) |'
-echo ' |_____/ \___|\__, |_|_|\__,_|_____/|____/ '
-echo '               __/ |                       '
-echo '              |___/                        '
-echo ''
-echo ''
-echo 'Nodetool:'
-echo '	nodetool --help'
-echo 'CQL Shell:'
-echo '	cqlsh'
-echo 'More documentation available at: '
-echo '	http://www.scylladb.com/doc/'
-echo
-
-if [ "`systemctl is-active scylla-server`" = "active" ]; then
-	tput setaf 4
-	tput bold
-	echo "    ScyllaDB is active."
-	tput sgr0
-else
-	tput setaf 1
-	tput bold
-	echo "    ScyllaDB is not started!"
-	tput sgr0
-	echo "Please wait for startup. To see status of ScyllaDB, run "
-	echo " 'systemctl status scylla-server'"
-fi

dist/ami/files/coredump.conf

@@ -0,0 +1,5 @@
+[Coredump]
+Storage=external
+Compress=yes
+ProcessSizeMax=16G
+ExternalSizeMax=16G

dist/ami/files/scylla-setup.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Scylla Setup
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/lib/scylla/scylla-setup.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target

dist/ami/files/scylla-setup.sh

@@ -0,0 +1,56 @@
+#!/bin/sh -e
+
+RAIDCNT=`grep xvdb /proc/mdstat | wc -l`
+RAIDDEV=`grep xvdb /proc/mdstat | awk '{print $1}'`
+
+if [ $RAIDCNT -ge 1 ]; then
+    echo "RAID already constructed."
+    mount -o noatime /dev/$RAIDDEV /var/lib/scylla
+else
+    echo "RAID does not constructed, going to initialize..."
+
+    dnf update -y
+
+    DISKS=""
+    NR=0
+    for i in xvd{b..z}; do
+        if [ -b /dev/$i ];then
+            echo Found disk /dev/$i
+            DISKS="$DISKS /dev/$i"
+            NR=$((NR+1))
+        fi
+    done
+
+    echo Creating RAID0 for scylla using $NR disk\(s\): $DISKS
+
+    if [ $NR -ge 1 ]; then
+        mdadm --create --verbose --force --run /dev/md0 --level=0 -c256 --raid-devices=$NR $DISKS
+        blockdev --setra 65536 /dev/md0
+        mkfs.xfs /dev/md0 -f
+        echo "DEVICE $DISKS" > /etc/mdadm.conf
+        mdadm --detail --scan >> /etc/mdadm.conf
+        UUID=`blkid /dev/md0 | awk '{print $2}'`
+        mount -o noatime /dev/md0 /var/lib/scylla
+    else
+        echo "WARN: Scylla is not using XFS to store data. Perforamnce will suffer." > /home/fedora/WARN_PLEASE_READ.TXT
+    fi
+
+    mkdir -p /var/lib/scylla/data
+    mkdir -p /var/lib/scylla/commitlog
+    chown scylla:scylla /var/lib/scylla/*
+    chown scylla:scylla /var/lib/scylla/
+
+    CPU_NR=`cat /proc/cpuinfo |grep processor|wc -l`
+    if [ $CPU_NR -ge 8 ]; then
+        NR=$((CPU_NR - 1))
+        grep -v SCYLLA_ARGS /etc/sysconfig/scylla-server | grep -v SET_NIC > /tmp/scylla-server
+        echo SCYLLA_ARGS=\"--cpuset 1-$NR  --smp $NR\" >> /tmp/scylla-server
+        echo SET_NIC=\"yes\" >> /tmp/scylla-server
+        mv /tmp/scylla-server /etc/sysconfig/scylla-server
+    fi
+
+    /usr/lib/scylla/scylla-ami/ds2_configure.py
+fi
+
+systemctl start scylla-server.service
+systemctl start scylla-jmx.service

dist/ami/files/scylla.repo

@@ -0,0 +1,11 @@
+[scylla]
+name=Scylla for Fedora $releasever - $basearch
+baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/fedora/$releasever/$basearch/
+enabled=1
+gpgcheck=0
+
+[scylla-generic]
+name=Scylla for Fedora $releasever 
+baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/fedora/$releasever/noarch/
+enabled=1
+gpgcheck=0

dist/ami/files/setup-ami.sh

@@ -0,0 +1,18 @@
+#!/bin/sh -e
+
+setenforce 0
+sed -e "s/enforcing/disabled/" /etc/sysconfig/selinux > /tmp/selinux
+mv /tmp/selinux /etc/sysconfig/
+dnf update -y
+mv /home/fedora/scylla.repo /etc/yum.repos.d/
+dnf install -y scylla-server scylla-server-debuginfo scylla-jmx scylla-tools
+dnf install -y mdadm xfsprogs
+cp /home/fedora/coredump.conf /etc/systemd/coredump.conf
+mv /home/fedora/scylla-setup.service /usr/lib/systemd/system
+mv /home/fedora/scylla-setup.sh /usr/lib/scylla
+chmod a+rx /usr/lib/scylla/scylla-setup.sh
+mv /home/fedora/scylla-ami /usr/lib/scylla/scylla-ami
+chmod a+rx /usr/lib/scylla/scylla-ami/ds2_configure.py
+systemctl enable scylla-setup.service
+grep -v ' - mounts' /etc/cloud/cloud.cfg > /tmp/cloud.cfg
+mv /tmp/cloud.cfg /etc/cloud/cloud.cfg

dist/ami/scylla.json

@@ -18,23 +18,13 @@
   "provisioners": [
     {
       "type": "file",
-      "source": "files/scylla-ami",
-      "destination": "/home/fedora/scylla-ami"
-    },
-    {
-      "type": "file",
-      "source": "files/.bash_profile",
-      "destination": "/home/fedora/.bash_profile"
-    },
-    {
-      "type": "file",
-      "source": "../../scripts/scylla_install",
-      "destination": "/home/fedora/scylla_install"
+      "source": "files/",
+      "destination": "/home/fedora"
     },
     {
       "type": "shell",
       "inline": [
-          "sudo sh -x -e /home/fedora/scylla_install -a"
+	"sudo sh -x -e /home/fedora/setup-ami.sh"
       ]
     }
   ],

dist/ami/scylla_local.json

@@ -1,67 +0,0 @@
-{
-  "builders": [
-    {
-      "type": "amazon-ebs",
-      "access_key": "{{user `access_key`}}",
-      "secret_key": "{{user `secret_key`}}",
-      "subnet_id": "{{user `subnet_id`}}",
-      "security_group_id": "{{user `security_group_id`}}",
-      "region": "{{user `region`}}",
-      "associate_public_ip_address": "{{user `associate_public_ip_address`}}",
-      "source_ami": "ami-a51564c0",
-      "instance_type": "{{user `instance_type`}}",
-      "ssh_username": "fedora",
-      "ssh_timeout": "5m",
-      "ami_name": "scylla_{{isotime | clean_ami_name}}"
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "file",
-      "source": "files/scylla-ami",
-      "destination": "/home/fedora/scylla-ami"
-    },
-    {
-      "type": "file",
-      "source": "files/.bash_profile",
-      "destination": "/home/fedora/.bash_profile"
-    },
-    {
-      "type": "file",
-      "source": "../../scripts/scylla_install",
-      "destination": "/home/fedora/scylla_install"
-    },
-    {
-      "type": "file",
-      "source": "scylla-server.x86_64.rpm",
-      "destination": "/home/fedora/scylla-server.x86_64.rpm"
-    },
-    {
-      "type": "file",
-      "source": "scylla-jmx.noarch.rpm",
-      "destination": "/home/fedora/scylla-jmx.noarch.rpm"
-    },
-    {
-      "type": "file",
-      "source": "scylla-tools.noarch.rpm",
-      "destination": "/home/fedora/scylla-tools.noarch.rpm"
-    },
-    {
-      "type": "shell",
-      "inline": [
-          "sudo yum install -y /home/fedora/scylla-server.x86_64.rpm /home/fedora/scylla-jmx.noarch.rpm /home/fedora/scylla-tools.noarch.rpm",
-          "sudo mv /home/fedora/scylla-ami /usr/lib/scylla/scylla-ami",
-          "sudo sh -x -e /home/fedora/scylla_install -a -l /home/fedora"
-      ]
-    }
-  ],
-  "variables": {
-    "access_key": "",
-    "secret_key": "",
-    "subnet_id": "",
-    "security_group_id": "",
-    "region": "",
-    "associate_public_ip_address": "",
-    "instance_type": ""
-  }
-}

dist/common/limits.d/scylla.conf

@@ -1,5 +1,4 @@
-scylla  -  core     unlimited
 scylla  -  memlock  unlimited
-scylla  -  nofile   200000
+scylla  -  nofile   100000
 scylla  -  as       unlimited
 scylla  -  nproc    8096

dist/common/scripts/scylla_bootparam_setup

@@ -1,48 +0,0 @@
-#!/bin/sh -e
-#
-#  Copyright (C) 2015 ScyllaDB
-
-print_usage() {
-    echo "scylla_bootparam_setup -a"
-    echo "  -a  AMI instance mode"
-    exit 1
-}
-
-AMI=0
-while getopts a OPT; do
-    case "$OPT" in
-        "a")
-            AMI=1
-            ;;
-        "h")
-            print_usage
-            ;;
-    esac
-done
-
-. /etc/os-release
-
-if [ $AMI -eq 1 ]; then
-    . /etc/sysconfig/scylla-server
-    sed -e "s#append #append clocksource=tsc tsc=reliable hugepagesz=2M hugepages=$NR_HUGEPAGES #" /boot/extlinux/extlinux.conf > /tmp/extlinux.conf
-    mv /tmp/extlinux.conf /boot/extlinux/extlinux.conf
-else
-    . /etc/sysconfig/scylla-server
-    if [ ! -f /etc/default/grub ]; then
-        echo "Unsupported bootloader"
-        exit 1
-    fi
-    if [ "`grep hugepagesz /etc/default/grub`" != "" ] || [ "`grep hugepages /etc/default/grub`" != "" ]; then
-        sed -e "s#hugepagesz=2M ##" /etc/default/grub > /tmp/grub
-        mv /tmp/grub /etc/default/grub
-        sed -e "s#hugepages=[0-9]* ##" /etc/default/grub > /tmp/grub
-        mv /tmp/grub /etc/default/grub
-    fi
-    sed -e "s#^GRUB_CMDLINE_LINUX=\"#GRUB_CMDLINE_LINUX=\"hugepagesz=2M hugepages=$NR_HUGEPAGES #" /etc/default/grub > /tmp/grub
-    mv /tmp/grub /etc/default/grub
-    if [ "$ID" = "ubuntu" ]; then
-        grub2-mkconfig -o /boot/grub/grub.cfg
-    else
-        grub2-mkconfig -o /boot/grub2/grub.cfg
-    fi
-fi

dist/common/scripts/scylla_coredump_setup

@@ -1,16 +0,0 @@
-#!/bin/sh -e
-#
-#  Copyright (C) 2015 ScyllaDB
-
-. /etc/os-release
-
-if [ "$ID" = "ubuntu" ]; then
-    apt-get remove -y apport-noui
-else
-    if [ -f /etc/systemd/coredump.conf ]; then
-        mv /etc/systemd/coredump.conf /etc/systemd/coredump.conf.save
-        systemctl daemon-reload
-    fi
-fi
-
-sysctl -p /etc/sysctl.d/99-scylla.conf

dist/common/scripts/scylla_ntp_setup

@@ -1,38 +0,0 @@
-#!/bin/sh -e
-#
-#  Copyright (C) 2015 ScyllaDB
-
-print_usage() {
-    echo "scylla_ntp_setup -a"
-    echo "  -a  AMI instance mode"
-    exit 1
-}
-
-AMI=0
-while getopts a OPT; do
-    case "$OPT" in
-        "a")
-            AMI=1
-            ;;
-        "h")
-            print_usage
-            ;;
-    esac
-done
-
-. /etc/os-release
-if [ "$NAME" = "Ubuntu" ]; then
-    apt-get install -y ntp ntpdate
-    service ntp stop
-    ntpdate `cat /etc/ntp.conf |grep "^server"|head -n1|awk '{print $2}'`
-    service ntp start
-else
-    yum install -y ntp ntpdate || true
-    if [ $AMI -eq 1 ]; then
-        sed -e s#fedora.pool.ntp.org#amazon.pool.ntp.org# /etc/ntp.conf > /tmp/ntp.conf
-        mv /tmp/ntp.conf /etc/ntp.conf
-    fi
-    systemctl enable ntpd.service
-    ntpdate `cat /etc/ntp.conf |grep "^server"|head -n1|awk '{print $2}'`
-    systemctl start ntpd.service
-fi

dist/common/scripts/scylla_prepare

@@ -1,43 +1,5 @@
 #!/bin/sh -e
 
-if [ "$AMI" = "yes" ]; then
-    RAIDCNT=`grep xvdb /proc/mdstat | wc -l`
-    RAIDDEV=`grep xvdb /proc/mdstat | awk '{print $1}'`
-
-    if [ $RAIDCNT -ge 1 ]; then
-        echo "RAID already constructed."
-        if [ "`mount|grep /var/lib/scylla`" = "" ]; then
-            mount -o noatime /dev/$RAIDDEV /var/lib/scylla
-        fi
-    else
-        echo "RAID does not constructed, going to initialize..."
-
-        if [ "$AMI_KEEP_VERSION" != "yes" ]; then
-            yum update -y
-        fi
-
-        DISKS=""
-        for i in /dev/xvd{b..z}; do
-            if [ -b $i ];then
-                echo "Found disk $i"
-                if [ "$DISKS" = "" ]; then
-                    DISKS=$i
-                else
-                    DISKS="$DISKS,$i"
-                fi
-            fi
-        done
-
-        if [ "$DISKS" != "" ]; then
-            /usr/lib/scylla/scylla_raid_setup -d $DISKS
-        else
-            echo "ERROR: Scylla is not using XFS to store data. The scylla service will refuse to start." > /home/fedora/SCYLLA_SETUP_ERROR.LOG
-        fi
-
-        /usr/lib/scylla/scylla-ami/ds2_configure.py
-    fi
-fi
-
 if [ "$NETWORK_MODE" = "virtio" ]; then
     ip tuntap del mode tap dev $TAP
     ip tuntap add mode tap dev $TAP user $USER one_queue vnet_hdr
@@ -53,10 +15,10 @@ elif [ "$NETWORK_MODE" = "dpdk" ]; then
     done
 else # NETWORK_MODE = posix
     if [ "$SET_NIC" = "yes" ]; then
-        sudo sh /usr/lib/scylla/posix_net_conf.sh $IFNAME >/dev/null 2>&1 || true
+        sudo sh /usr/lib/scylla/posix_net_conf.sh >/dev/null 2>&1 || true
     fi
 fi
 . /etc/os-release
-if [ "$ID" = "ubuntu" ]; then
+if [ "$NAME" = "Ubuntu" ]; then
     hugeadm --create-mounts
 fi

dist/common/scripts/scylla_raid_setup

@@ -1,61 +0,0 @@
-#!/bin/sh -e
-#
-#  Copyright (C) 2015 ScyllaDB
-
-print_usage() {
-    echo "scylla-raid-setup -d /dev/hda,/dev/hdb... -r /dev/md0 -u"
-    echo "  -d  specify disks for RAID"
-    echo "  -r  MD device name for RAID"
-    echo "  -u  update /etc/fstab for RAID"
-    exit 1
-}
-
-RAID=/dev/md0
-FSTAB=0
-while getopts d:r:uh OPT; do
-    case "$OPT" in
-        "d")
-            DISKS=`echo $OPTARG|tr -s ',' ' '`
-            NR_DISK=$((`echo $OPTARG|grep , -o|wc -w` + 1))
-            ;;
-        "r")
-            RAID=$OPTARG
-            ;;
-        "u")
-            FSTAB=1
-            ;;
-        "h")
-            print_usage
-            ;;
-    esac
-done
-
-if [ "$DISKS" = "" ]; then
-    print_usage
-fi
-
-echo Creating RAID0 for scylla using $NR_DISK disk\(s\): $DISKS
-if [ -e $RAID ]; then
-    echo "$RAID is already using"
-    exit 1
-fi
-if [ "`mount|grep /var/lib/scylla`" != "" ]; then
-    echo "/var/lib/scylla is already mounted"
-    exit 1
-fi
-mdadm --create --verbose --force --run $RAID --level=0 -c256 --raid-devices=$NR_DISK $DISKS
-blockdev --setra 65536 $RAID
-mkfs.xfs $RAID -f
-echo "DEVICE $DISKS" > /etc/mdadm.conf
-mdadm --detail --scan >> /etc/mdadm.conf
-if [ $FSTAB -ne 0 ]; then
-    UUID=`blkid $RAID | awk '{print $2}'`
-    echo "$UUID /var/lib/scylla xfs noatime 0 0" >> /etc/fstab
-fi
-mount -t xfs -o noatime $RAID /var/lib/scylla
-
-mkdir -p /var/lib/scylla/data
-mkdir -p /var/lib/scylla/commitlog
-mkdir -p /var/lib/scylla/coredump
-chown scylla:scylla /var/lib/scylla/*
-chown scylla:scylla /var/lib/scylla/

dist/common/scripts/scylla_save_coredump

@@ -1,10 +0,0 @@
-#!/bin/sh -e
-#
-#  Copyright (C) 2015 ScyllaDB
-
-FILE=$1
-TIME=`date --date @$2 +%F-%T`
-PID=$3
-
-mkdir -p /var/lib/scylla/coredump
-/usr/bin/gzip -c > /var/lib/scylla/coredump/core.$FILE-$TIME-$PID.gz

dist/common/scripts/scylla_sysconfig_setup

@@ -1,100 +0,0 @@
-#!/bin/sh -e
-#
-#  Copyright (C) 2015 ScyllaDB
-
-print_usage() {
-    echo "scylla-sysconfig-setup -n eth0 -m posix -p 64 -u scylla -g scylla -d /var/lib/scylla -c /etc/scylla -N -a -k"
-    echo "  -n  specify NIC"
-    echo "  -m  network mode (posix, dpdk)"
-    echo "  -p  number of hugepages"
-    echo "  -u  user (dpdk requires root)"
-    echo "  -g  group (dpdk requires root)"
-    echo "  -d  scylla home directory"
-    echo "  -c  scylla config directory"
-    echo "  -N  setup NIC's interrupts, RPS, XPS"
-    echo "  -a  AMI instance mode"
-    echo "  -k  keep package version on AMI"
-    exit 1
-}
-
-. /etc/os-release
-if [ "$ID" = "ubuntu" ]; then
-    SYSCONFIG=/etc/default
-else
-    SYSCONFIG=/etc/sysconfig
-fi
-
-NIC=eth0
-NETWORK_MODE=posix
-NR_HUGEPAGES=64
-USER=scylla
-GROUP=scylla
-SCYLLA_HOME=/var/lib/scylla
-SCYLLA_CONF=/etc/scylla
-SETUP_NIC=0
-SET_NIC="no"
-AMI=no
-AMI_KEEP_VERSION=no
-SCYLLA_ARGS=
-while getopts n:m:p:u:g:d:c:Nakh OPT; do
-    case "$OPT" in
-        "n")
-            NIC=$OPTARG
-            ;;
-        "m")
-            NETWORK_MODE=$OPTARG
-            ;;
-        "p")
-            NR_HUGEPAGES=$OPTARG
-            ;;
-        "u")
-            USER=$OPTARG
-            ;;
-        "g")
-            GROUP=$OPTARG
-            ;;
-        "d")
-            SCYLLA_HOME=$OPTARG
-            ;;
-        "c")
-            SCYLLA_CONF=$OPTARG
-            ;;
-        "N")
-            SETUP_NIC=1
-            ;;
-        "a")
-            AMI=yes
-            ;;
-        "k")
-            AMI_KEEP_VERSION=yes
-            ;;
-        "h")
-            print_usage
-            ;;
-    esac
-done
-
-echo Setting parameters on $SYSCONFIG/scylla-server
-
-ETHDRV=`/usr/lib/scylla/dpdk_nic_bind.py --status | grep if=$NIC | sed -e "s/^.*drv=//" -e "s/ .*$//"`
-ETHPCIID=`/usr/lib/scylla/dpdk_nic_bind.py --status | grep if=$NIC | awk '{print $1}'`
-NR_CPU=`cat /proc/cpuinfo |grep processor|wc -l`
-if [ $NR_CPU -ge 8 ]; then
-    NR=$((NR_CPU - 1))
-    SET_NIC="yes"
-    SCYLLA_ARGS="--cpuset 1-$NR  --smp $NR"
-fi
-sed -e s#^NETWORK_MODE=.*#NETWORK_MODE=$NETWORK_MODE# \
-    -e s#^ETHDRV=.*#ETHDRV=$ETHDRV# \
-    -e s#^ETHPCIID=.*#ETHPCIID=$ETHPCIID# \
-    -e s#^NR_HUGEPAGES=.*#NR_HUGEPAGES=$NR_HUGEPAGES# \
-    -e s#^USER=.*#USER=$USER# \
-    -e s#^GROUP=.*#GROUP=$GROUP# \
-    -e s#^SCYLLA_HOME=.*#SCYLLA_HOME=$SCYLLA_HOME# \
-    -e s#^SCYLLA_CONF=.*#SCYLLA_CONF=$SCYLLA_CONF# \
-    -e s#^SET_NIC=.*#SET_NIC=$SET_NIC# \
-    -e s#^SCYLLA_ARGS=.*#SCYLLA_ARGS="$SCYLLA_ARGS"# \
-    -e s#^AMI=.*#AMI="$AMI"# \
-    -e s#^AMI_KEEP_VERSION=.*#AMI_KEEP_VERSION="$AMI_KEEP_VERSION"# \
-    $SYSCONFIG/scylla-server > /tmp/scylla-server
-mv /tmp/scylla-server $SYSCONFIG/scylla-server

dist/common/sysconfig/scylla-server

@@ -7,12 +7,6 @@ TAP=tap0
 # bridge device name (virtio)
 BRIDGE=virbr0
 
-# ethernet device name
-IFNAME=eth0
-
-# setup NIC's interrupts, RPS, XPS (posix)
-SET_NIC=no
-
 # ethernet device driver (dpdk)
 ETHDRV=
 
@@ -36,9 +30,3 @@ SCYLLA_CONF=/etc/scylla
 
 # additional arguments
 SCYLLA_ARGS=""
-
-# setup as AMI instance
-AMI=no
-
-# do not upgrade Scylla packages on AMI startup
-AMI_KEEP_VERSION=no

dist/common/sysctl.d/99-scylla.conf

@@ -1 +0,0 @@
-kernel.core_pattern=|/usr/lib/scylla/scylla_save_coredump %e %t %p

dist/docker/Dockerfile

@@ -1,13 +1,11 @@
-FROM centos:7
+FROM fedora:22
 
 MAINTAINER Avi Kivity <avi@cloudius-systems.com>
 
-RUN yum -y install epel-release
 ADD scylla.repo /etc/yum.repos.d/
-RUN yum -y update
-RUN yum -y remove boost-thread boost-system
-RUN yum -y install scylla-server hostname
-RUN yum clean all
+RUN dnf -y update
+RUN dnf -y install scylla-server hostname
+RUN dnf clean all
 ADD start-scylla /start-scylla
 RUN chown scylla /start-scylla
 

dist/docker/scylla.repo

@@ -1,23 +1,11 @@
 [scylla]
-name=Scylla for Centos $releasever - $basearch
-baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/centos/$releasever/$basearch/
+name=Scylla for Fedora $releasever - $basearch
+baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/fedora/$releasever/$basearch/
 enabled=1
 gpgcheck=0
 
 [scylla-generic]
-name=Scylla for centos $releasever
-baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/centos/$releasever/noarch/
-enabled=1
-gpgcheck=0
-
-[scylla-3rdparty]
-name=Scylla 3rdParty for Centos $releasever - $basearch
-baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/3rdparty/centos/$releasever/$basearch/
-enabled=1
-gpgcheck=0
-
-[scylla-3rdparty-generic]
-name=Scylla 3rdParty for Centos $releasever
-baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/3rdparty/centos/$releasever/noarch/
+name=Scylla for Fedora $releasever
+baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/fedora/$releasever/noarch/
 enabled=1
 gpgcheck=0

dist/docker/start-scylla

@@ -4,9 +4,9 @@ IP=$(hostname -i)
 sed -e "s/seeds:.*/seeds: $IP/g" /var/lib/scylla/conf/scylla.yaml > $HOME/scylla.yaml
 /usr/bin/scylla --log-to-syslog 1 \
                 --log-to-stdout 0 \
-                --developer-mode true \
                 --default-log-level info \
                 --options-file $HOME/scylla.yaml \
                 --listen-address $IP \
                 --rpc-address $IP \
-                --network-stack posix
+                --network-stack posix \
+                --smp 1

dist/redhat/build_rpm.sh

@@ -7,21 +7,21 @@ if [ ! -e dist/redhat/build_rpm.sh ]; then
     exit 1
 fi
 
-. /etc/os-release
-if [ "$ID" != "fedora" ] && [ "$ID" != "centos" ]; then
+OS=`awk '{print $1}' /etc/redhat-release`
+if [ "$OS" != "Fedora" ] && [ "$OS" != "CentOS" ]; then
     echo "Unsupported distribution"
     exit 1
 fi
-if [ "$ID" = "fedora" ] && [ ! -f /usr/bin/mock ]; then
+if [ "$OS" = "Fedora" ] && [ ! -f /usr/bin/mock ]; then
     sudo yum -y install mock
-elif [ "$ID" = "centos" ] && [ ! -f /usr/bin/yum-builddep ]; then
+elif [ "$OS" = "CentOS" ] && [ ! -f /usr/bin/yum-builddep ]; then
     sudo yum -y install yum-utils
 fi
 if [ ! -f /usr/bin/git ]; then
     sudo yum -y install git
 fi
 mkdir -p $RPMBUILD/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}
-if [ "$ID" = "centos" ]; then
+if [ "$OS" = "CentOS" ]; then
     ./dist/redhat/centos_dep/build_dependency.sh
 fi
 VERSION=$(./SCYLLA-VERSION-GEN)
@@ -33,7 +33,7 @@ rm -f version
 cp dist/redhat/scylla-server.spec.in $RPMBUILD/SPECS/scylla-server.spec
 sed -i -e "s/@@VERSION@@/$SCYLLA_VERSION/g" $RPMBUILD/SPECS/scylla-server.spec
 sed -i -e "s/@@RELEASE@@/$SCYLLA_RELEASE/g" $RPMBUILD/SPECS/scylla-server.spec
-if [ "$ID" = "fedora" ]; then
+if [ "$OS" = "Fedora" ]; then
     rpmbuild -bs --define "_topdir $RPMBUILD" $RPMBUILD/SPECS/scylla-server.spec
     mock rebuild --resultdir=`pwd`/build/rpms $RPMBUILD/SRPMS/scylla-server-$VERSION*.src.rpm
 else

dist/redhat/scylla-server.spec.in

@@ -8,18 +8,13 @@ License:        AGPLv3
 URL:            http://www.scylladb.com/
 Source0:        %{name}-@@VERSION@@-@@RELEASE@@.tar
 
-BuildRequires:  libaio-devel boost-devel libstdc++-devel cryptopp-devel hwloc-devel numactl-devel libpciaccess-devel libxml2-devel zlib-devel thrift-devel yaml-cpp-devel lz4-devel snappy-devel jsoncpp-devel systemd-devel xz-devel openssl-devel libcap-devel libselinux-devel libgcrypt-devel libgpg-error-devel elfutils-devel krb5-devel libcom_err-devel libattr-devel pcre-devel elfutils-libelf-devel bzip2-devel keyutils-libs-devel xfsprogs-devel make gnutls-devel
+BuildRequires:  libaio-devel boost-devel libstdc++-devel cryptopp-devel hwloc-devel numactl-devel libpciaccess-devel libxml2-devel zlib-devel thrift-devel yaml-cpp-devel lz4-devel snappy-devel jsoncpp-devel systemd-devel xz-devel openssl-devel libcap-devel libselinux-devel libgcrypt-devel libgpg-error-devel elfutils-devel krb5-devel libcom_err-devel libattr-devel pcre-devel elfutils-libelf-devel bzip2-devel keyutils-libs-devel xfsprogs-devel make
 %{?fedora:BuildRequires: ninja-build ragel antlr3-tool antlr3-C++-devel python3 gcc-c++ libasan libubsan}
 %{?rhel:BuildRequires: scylla-ninja-build scylla-ragel scylla-antlr3-tool scylla-antlr3-C++-devel python34 scylla-gcc-c++ >= 5.1.1}
-Requires:       systemd-libs xfsprogs mdadm hwloc
+Requires:       systemd-libs xfsprogs
 
 %description
 
-%define __debug_install_post   \
-    %{_rpmconfigdir}/find-debuginfo.sh %{?_missing_build_ids_terminate_build:--strict-build-id} %{?_find_debuginfo_opts} "%{_builddir}/%{?buildsubdir}";\
-    cp scylla-gdb.py ${RPM_BUILD_ROOT}/usr/src/debug/%{name}-%{version}/;\
-    %{nil}
-
 %prep
 %setup -q
 
@@ -35,7 +30,6 @@ ninja-build -j2
 %install
 rm -rf $RPM_BUILD_ROOT
 mkdir -p $RPM_BUILD_ROOT%{_bindir}
-mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysctl.d/
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/security/limits.d/
 mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/scylla/
@@ -43,7 +37,6 @@ mkdir -p $RPM_BUILD_ROOT%{_docdir}/scylla/
 mkdir -p $RPM_BUILD_ROOT%{_unitdir}
 mkdir -p $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 
-install -m644 dist/common/sysctl.d/99-scylla.conf $RPM_BUILD_ROOT%{_sysconfdir}/sysctl.d/
 install -m644 dist/common/sysconfig/scylla-server $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/
 install -m644 dist/common/limits.d/scylla.conf $RPM_BUILD_ROOT%{_sysconfdir}/security/limits.d/
 install -d -m755 $RPM_BUILD_ROOT%{_sysconfdir}/scylla
@@ -51,7 +44,6 @@ install -m644 conf/scylla.yaml $RPM_BUILD_ROOT%{_sysconfdir}/scylla/
 install -m644 conf/cassandra-rackdc.properties $RPM_BUILD_ROOT%{_sysconfdir}/scylla/
 install -m644 dist/redhat/systemd/scylla-server.service $RPM_BUILD_ROOT%{_unitdir}/
 install -m755 dist/common/scripts/* $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
-install -m755 dist/redhat/scripts/* $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 install -m755 seastar/scripts/posix_net_conf.sh  $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 install -m755 seastar/dpdk/tools/dpdk_nic_bind.py $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 install -m755 build/release/scylla $RPM_BUILD_ROOT%{_bindir}
@@ -65,7 +57,6 @@ install -m644 licenses/* $RPM_BUILD_ROOT%{_docdir}/scylla/licenses/
 install -d -m755 $RPM_BUILD_ROOT%{_sharedstatedir}/scylla/
 install -d -m755 $RPM_BUILD_ROOT%{_sharedstatedir}/scylla/data
 install -d -m755 $RPM_BUILD_ROOT%{_sharedstatedir}/scylla/commitlog
-install -d -m755 $RPM_BUILD_ROOT%{_sharedstatedir}/scylla/coredump
 install -d -m755 $RPM_BUILD_ROOT%{_prefix}/lib/scylla/swagger-ui
 cp -r swagger-ui/dist $RPM_BUILD_ROOT%{_prefix}/lib/scylla/swagger-ui
 install -d -m755 $RPM_BUILD_ROOT%{_prefix}/lib/scylla/api
@@ -84,27 +75,13 @@ TMP=""
 if [ -d /var/lib/scylla/conf ] && [ ! -L /var/lib/scylla/conf ]; then
     cp -a /var/lib/scylla/conf /tmp/%{name}-%{version}-%{release}
 fi
-# Adding IFNAME for previous version of sysconfig
-if [ -f /etc/sysconfig/scylla-server ] && [ `grep IFNAME -r /etc/sysconfig/scylla-server|wc -l` -eq 0 ]; then
-    echo "# ethernet device name" >> /etc/sysconfig/scylla-server
-    echo "IFNAME=eth0" >> /etc/sysconfig/scylla-server
-fi
-if [ -d /usr/lib/scylla/scylla-ami ]; then
-    echo "# setup as AMI instance" >> /etc/sysconfig/scylla-server
-    echo "AMI=no" >> /etc/sysconfig/scylla-server
-    echo "# do not upgrade Scylla packages on AMI startup" >> /etc/sysconfig/scylla-server
-    echo "AMI_KEEP_VERSION=no" >> /etc/sysconfig/scylla-server
-fi
 
 %post
 grep -v api_ui_dir /etc/scylla/scylla.yaml | grep -v api_doc_dir > /tmp/scylla.yaml
 echo "api_ui_dir: /usr/lib/scylla/swagger-ui/dist/" >> /tmp/scylla.yaml
 echo "api_doc_dir: /usr/lib/scylla/api/api-doc/" >> /tmp/scylla.yaml
 mv /tmp/scylla.yaml /etc/scylla/scylla.yaml
-# Upgrade coredump settings
-if [ -f /etc/systemd/coredump.conf ];then
-    /usr/lib/scylla/scylla_coredump_setup
-fi
+
 %systemd_post scylla-server.service
 
 %preun
@@ -128,7 +105,6 @@ rm -rf $RPM_BUILD_ROOT
 
 %config(noreplace) %{_sysconfdir}/sysconfig/scylla-server
 %{_sysconfdir}/security/limits.d/scylla.conf
-%{_sysconfdir}/sysctl.d/99-scylla.conf
 %attr(0755,root,root) %dir %{_sysconfdir}/scylla
 %config(noreplace) %{_sysconfdir}/scylla/scylla.yaml
 %config(noreplace) %{_sysconfdir}/scylla/cassandra-rackdc.properties
@@ -142,12 +118,6 @@ rm -rf $RPM_BUILD_ROOT
 %{_prefix}/lib/scylla/scylla_prepare
 %{_prefix}/lib/scylla/scylla_run
 %{_prefix}/lib/scylla/scylla_stop
-%{_prefix}/lib/scylla/scylla_save_coredump
-%{_prefix}/lib/scylla/scylla_coredump_setup
-%{_prefix}/lib/scylla/scylla_raid_setup
-%{_prefix}/lib/scylla/scylla_sysconfig_setup
-%{_prefix}/lib/scylla/scylla_bootparam_setup
-%{_prefix}/lib/scylla/scylla_ntp_setup
 %{_prefix}/lib/scylla/posix_net_conf.sh
 %{_prefix}/lib/scylla/dpdk_nic_bind.py
 %{_prefix}/lib/scylla/dpdk_nic_bind.pyc
@@ -157,7 +127,6 @@ rm -rf $RPM_BUILD_ROOT
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/data
 %attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/commitlog
-%attr(0755,scylla,scylla) %dir %{_sharedstatedir}/scylla/coredump
 
 %changelog
 * Tue Jul 21 2015 Takuya ASADA <syuu@cloudius-systems.com>

dist/redhat/systemd/scylla-server.service

@@ -5,14 +5,13 @@ After=network.target libvirtd.service
 [Service]
 Type=simple
 LimitMEMLOCK=infinity
-LimitNOFILE=200000
+LimitNOFILE=100000
 LimitAS=infinity
 LimitNPROC=8096
 EnvironmentFile=/etc/sysconfig/scylla-server
 ExecStartPre=/usr/lib/scylla/scylla_prepare
 ExecStart=/usr/lib/scylla/scylla_run
 ExecStopPost=/usr/lib/scylla/scylla_stop
-TimeoutStartSec=900
 KillMode=process
 Restart=no
 

dist/ubuntu/build_deb.sh

@@ -38,7 +38,7 @@ sudo apt-get -y update
 
 ./dist/ubuntu/dep/build_dependency.sh
 
-DEP="libyaml-cpp-dev liblz4-dev libsnappy-dev libcrypto++-dev libjsoncpp-dev libaio-dev ragel ninja-build git liblz4-1 libaio1 hugepages software-properties-common libgnutls28-dev libhwloc-dev libnuma-dev libpciaccess-dev"
+DEP="libyaml-cpp-dev liblz4-dev libsnappy-dev libcrypto++-dev libjsoncpp-dev libaio-dev ragel ninja-build git liblz4-1 libaio1 hugepages software-properties-common"
 
 if [ "$RELEASE" = "14.04" ]; then
     DEP="$DEP libboost1.55-dev libboost-program-options1.55.0 libboost-program-options1.55-dev libboost-system1.55.0 libboost-system1.55-dev libboost-thread1.55.0 libboost-thread1.55-dev libboost-test1.55.0 libboost-test1.55-dev libboost-filesystem1.55-dev libboost-filesystem1.55.0 libsnappy1"
@@ -55,6 +55,6 @@ if [ "$RELEASE" != "15.10" ]; then
     sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
     sudo apt-get -y update
 fi
-sudo apt-get -y install g++-4.9
+sudo apt-get -y install g++-5
 
 debuild -r fakeroot -us -uc

dist/ubuntu/debian/control

@@ -4,11 +4,11 @@ Homepage: http://scylladb.com
 Section: database
 Priority: optional
 Standards-Version: 3.9.5
-Build-Depends: debhelper (>= 9), libyaml-cpp-dev, liblz4-dev, libsnappy-dev, libcrypto++-dev, libjsoncpp-dev, libaio-dev, libthrift-dev, thrift-compiler, antlr3, antlr3-c++-dev, ragel, g++-4.9, ninja-build, git, libboost-program-options1.55-dev | libboost-program-options-dev, libboost-filesystem1.55-dev | libboost-filesystem-dev, libboost-system1.55-dev | libboost-system-dev, libboost-thread1.55-dev | libboost-thread-dev, libboost-test1.55-dev | libboost-test-dev, libgnutls28-dev, libhwloc-dev, libnuma-dev, libpciaccess-dev
+Build-Depends: debhelper (>= 9), libyaml-cpp-dev, liblz4-dev, libsnappy-dev, libcrypto++-dev, libjsoncpp-dev, libaio-dev, libthrift-dev, thrift-compiler, antlr3, antlr3-c++-dev, ragel, g++-5, ninja-build, git, libboost-program-options1.55-dev | libboost-program-options-dev, libboost-filesystem1.55-dev | libboost-filesystem-dev, libboost-system1.55-dev | libboost-system-dev, libboost-thread1.55-dev | libboost-thread-dev, libboost-test1.55-dev | libboost-test-dev
 
 Package: scylla-server
 Architecture: amd64
-Depends: ${shlibs:Depends}, ${misc:Depends}, hugepages, adduser, mdadm, xfsprogs, hwloc-nox
+Depends: ${shlibs:Depends}, ${misc:Depends}, hugepages, adduser
 Description: Scylla database server binaries 
  Scylla is a highly scalable, eventually consistent, distributed,
  partitioned row DB.

dist/ubuntu/debian/rules

@@ -4,13 +4,12 @@ DOC = $(CURDIR)/debian/scylla-server/usr/share/doc/scylla-server
 SCRIPTS = $(CURDIR)/debian/scylla-server/usr/lib/scylla
 SWAGGER = $(SCRIPTS)/swagger-ui
 API = $(SCRIPTS)/api
-SYSCTL = $(CURDIR)/debian/scylla-server/etc/sysctl.d
 LIMITS= $(CURDIR)/debian/scylla-server/etc/security/limits.d
 LIBS = $(CURDIR)/debian/scylla-server/usr/lib
 CONF = $(CURDIR)/debian/scylla-server/etc/scylla
 
 override_dh_auto_build:
-	./configure.py --disable-xen --enable-dpdk --mode=release --static-stdc++ --compiler=g++-4.9
+	./configure.py --disable-xen --enable-dpdk --mode=release --static-stdc++ --compiler=g++-5
 	ninja
 
 override_dh_auto_clean:
@@ -22,9 +21,6 @@ override_dh_auto_install:
 	mkdir -p $(LIMITS) && \
 	cp $(CURDIR)/dist/common/limits.d/scylla.conf  $(LIMITS)
 
-	mkdir -p $(SYSCTL) && \
-	cp $(CURDIR)/dist/common/sysctl.d/99-scylla.conf $(SYSCTL)
-
 	mkdir -p $(CONF) && \
 	cp $(CURDIR)/conf/scylla.yaml $(CONF)
 	cp $(CURDIR)/conf/cassandra-rackdc.properties $(CONF)
@@ -38,7 +34,6 @@ override_dh_auto_install:
 	mkdir -p $(SCRIPTS) && \
 	cp $(CURDIR)/seastar/dpdk/tools/dpdk_nic_bind.py $(SCRIPTS)
 	cp $(CURDIR)/dist/common/scripts/* $(SCRIPTS)
-	cp $(CURDIR)/dist/ubuntu/scripts/* $(SCRIPTS)
 
 	mkdir -p $(SWAGGER) && \
 	cp -r $(CURDIR)/swagger-ui/dist $(SWAGGER)
@@ -52,7 +47,6 @@ override_dh_auto_install:
 
 	mkdir -p $(CURDIR)/debian/scylla-server/var/lib/scylla/data
 	mkdir -p $(CURDIR)/debian/scylla-server/var/lib/scylla/commitlog
-	mkdir -p $(CURDIR)/debian/scylla-server/var/lib/scylla/coredump
 
 override_dh_strip:
 	dh_strip --dbg-package=scylla-server-dbg

dist/ubuntu/scripts/scylla_run

@@ -1,19 +0,0 @@
-#!/bin/bash -e
-
-args="--log-to-syslog 1 --log-to-stdout 0 --default-log-level info $SCYLLA_ARGS"
-
-if [ "$NETWORK_MODE" = "posix" ]; then
-    args="$args --network-stack posix"
-elif [ "$NETWORK_MODE" = "virtio" ]; then
-    args="$args --network-stack native"
-elif [ "$NETWORK_MODE" = "dpdk" ]; then
-    args="$args --network-stack native --dpdk-pmd"
-fi
-
-export HOME=/var/lib/scylla
-ulimit -c unlimited
-ulimit -l unlimited
-ulimit -n 200000
-ulimit -m unlimited
-ulimit -u 8096
-exec sudo -E -u $USER /usr/bin/scylla $args

frozen_mutation.cc

@@ -27,7 +27,6 @@
 #include "mutation_partition_serializer.hh"
 #include "utils/UUID.hh"
 #include "utils/data_input.hh"
-#include "query-result-set.hh"
 
 //
 // Representation layout:
@@ -50,11 +49,6 @@ frozen_mutation::key(const schema& s) const {
     return partition_key_view_serializer::read(in);
 }
 
-dht::decorated_key
-frozen_mutation::decorated_key(const schema& s) const {
-    return dht::global_partitioner().decorate_key(s, key(s));
-}
-
 frozen_mutation::frozen_mutation(bytes&& b)
     : _bytes(std::move(b))
 { }
@@ -94,11 +88,3 @@ mutation_partition_view frozen_mutation::partition() const {
     partition_key_view_serializer::skip(in);
     return mutation_partition_view::from_bytes(in.read_view(in.avail()));
 }
-
-std::ostream& operator<<(std::ostream& out, const frozen_mutation::printer& pr) {
-    return out << pr.self.unfreeze(pr.schema);
-}
-
-frozen_mutation::printer frozen_mutation::pretty_printer(schema_ptr s) const {
-    return { *this, std::move(s) };
-}

frozen_mutation.hh

@@ -21,7 +21,6 @@
 
 #pragma once
 
-#include "dht/i_partitioner.hh"
 #include "atomic_cell.hh"
 #include "keys.hh"
 #include "mutation_partition_view.hh"
@@ -52,17 +51,8 @@ public:
     bytes_view representation() const { return _bytes; }
     utils::UUID column_family_id() const;
     partition_key_view key(const schema& s) const;
-    dht::decorated_key decorated_key(const schema& s) const;
     mutation_partition_view partition() const;
     mutation unfreeze(schema_ptr s) const;
-
-    struct printer {
-        const frozen_mutation& self;
-        schema_ptr schema;
-        friend std::ostream& operator<<(std::ostream&, const printer&);
-    };
-
-    printer pretty_printer(schema_ptr) const;
 };
 
 frozen_mutation freeze(const mutation& m);

gms/endpoint_state.hh

@@ -54,7 +54,7 @@ namespace gms {
  */
 class endpoint_state {
 public:
-    using clk = std::chrono::steady_clock;
+    using clk = std::chrono::high_resolution_clock;
 private:
     heart_beat_state _heart_beat_state;
     std::map<application_state, versioned_value> _application_state;
@@ -109,11 +109,7 @@ public:
     }
 
     void add_application_state(application_state key, versioned_value value) {
-        if (_application_state.count(key)) {
-            _application_state.at(key) = value;
-        } else {
-            _application_state.emplace(key, value);
-        }
+        _application_state[key] = value;
     }
 
     /* getters and setters */

gms/gossiper.cc

@@ -91,8 +91,11 @@ void gossiper::set_seeds(std::set<inet_address> _seeds) {
 }
 
 std::chrono::milliseconds gossiper::quarantine_delay() {
-    auto& ss = service::get_local_storage_service();
-    return ss.get_ring_delay() * 2;
+    return std::chrono::milliseconds(service::storage_service::RING_DELAY * 2);
+}
+
+static auto storage_service_ring_delay() {
+    return std::chrono::milliseconds(service::storage_service::RING_DELAY);
 }
 
 auto& storage_service_value_factory() {
@@ -582,9 +585,9 @@ void gossiper::run() {
             logger.trace("=== Gossip round FAIL");
         }
 
-        if (logger.is_enabled(logging::log_level::trace)) {
+        if (logger.is_enabled(logging::log_level::debug)) {
             for (auto& x : endpoint_state_map) {
-                logger.trace("ep={}, eps={}", x.first, x.second);
+                logger.debug("ep={}, eps={}", x.first, x.second);
             }
         }
         _scheduled_gossip_task.arm(INTERVAL);
@@ -759,9 +762,8 @@ future<> gossiper::advertise_removing(inet_address endpoint, utils::UUID host_id
         // remember this node's generation
         int generation = state.get_heart_beat_state().get_generation();
         logger.info("Removing host: {}", host_id);
-        auto ring_delay = service::get_local_storage_service().get_ring_delay();
-        logger.info("Sleeping for {}ms to ensure {} does not change", ring_delay.count(), endpoint);
-        sleep(ring_delay).get();
+        logger.info("Sleeping for {}ms to ensure {} does not change", service::storage_service::RING_DELAY, endpoint);
+        sleep(storage_service_ring_delay()).get();
         // make sure it did not change
         auto& eps = endpoint_state_map.at(endpoint);
         if (eps.get_heart_beat_state().get_generation() != generation) {
@@ -821,9 +823,9 @@ future<> gossiper::assassinate_endpoint(sstring address) {
 
                 int generation = ep_state.get_heart_beat_state().get_generation();
                 int heartbeat = ep_state.get_heart_beat_state().get_heart_beat_version();
-                logger.info("Sleeping for {} ms to ensure {} does not change", ss.get_ring_delay().count(), endpoint);
+                logger.info("Sleeping for {} ms to ensure {} does not change", service::storage_service::RING_DELAY, endpoint);
                 // make sure it did not change
-                sleep(ss.get_ring_delay()).get();
+                sleep(storage_service_ring_delay()).get();
 
                 auto it = endpoint_state_map.find(endpoint);
                 if (it == endpoint_state_map.end()) {
@@ -1127,7 +1129,7 @@ void gossiper::handle_major_state_change(inet_address ep, const endpoint_state&
     if (!is_dead_state(ep_state)) {
         mark_alive(ep, ep_state);
     } else {
-        logger.debug("Not marking {} alive due to dead state {}", ep, get_gossip_status(eps));
+        logger.debug("Not marking {} alive due to dead state", ep);
         mark_dead(ep, ep_state);
     }
     _subscribers.for_each([ep, ep_state] (auto& subscriber) {
@@ -1140,7 +1142,14 @@ void gossiper::handle_major_state_change(inet_address ep, const endpoint_state&
 }
 
 bool gossiper::is_dead_state(const endpoint_state& eps) const {
-    sstring state = get_gossip_status(eps);
+    if (!eps.get_application_state(application_state::STATUS)) {
+        return false;
+    }
+    auto value = eps.get_application_state(application_state::STATUS)->value;
+    std::vector<sstring> pieces;
+    boost::split(pieces, value, boost::is_any_of(","));
+    assert(pieces.size() > 0);
+    sstring state = pieces[0];
     for (auto& deadstate : DEAD_STATES) {
         if (state == deadstate) {
             return true;
@@ -1150,11 +1159,38 @@ bool gossiper::is_dead_state(const endpoint_state& eps) const {
 }
 
 bool gossiper::is_shutdown(const inet_address& endpoint) const {
-    return get_gossip_status(endpoint) == sstring(versioned_value::SHUTDOWN);
+    auto ep_state = get_endpoint_state_for_endpoint(endpoint);
+    if (!ep_state) {
+        return false;
+    }
+
+    auto app_state = ep_state->get_application_state(application_state::STATUS);
+    if (!app_state) {
+        return false;
+    }
+
+    auto value = app_state->value;
+    std::vector<sstring> pieces;
+    boost::split(pieces, value, boost::is_any_of(","));
+    assert(pieces.size() > 0);
+    sstring state = pieces[0];
+
+    return state == sstring(versioned_value::SHUTDOWN);
 }
 
+
 bool gossiper::is_silent_shutdown_state(const endpoint_state& ep_state) const{
-    sstring state = get_gossip_status(ep_state);
+    auto app_state = ep_state.get_application_state(application_state::STATUS);
+    if (!app_state) {
+        return false;
+    }
+
+    auto value = app_state->value;
+    std::vector<sstring> pieces;
+    boost::split(pieces, value, boost::is_any_of(","));
+    assert(pieces.size() > 0);
+    sstring state = pieces[0];
+
     for (auto& deadstate : SILENT_SHUTDOWN_STATES) {
         if (state == deadstate) {
             return true;
@@ -1333,8 +1369,7 @@ future<> gossiper::do_shadow_round() {
                     return make_ready_future<>();
                 }).get();
             }
-            auto& ss = service::get_local_storage_service();
-            if (clk::now() > t + ss.get_ring_delay() * 60) {
+            if (clk::now() > t + storage_service_ring_delay() * 60) {
                 throw std::runtime_error(sprint("Unable to gossip with any seeds (ShadowRound)"));
             }
             if (this->_in_shadow_round) {
@@ -1536,24 +1571,5 @@ void gossiper::force_newer_generation() {
     }
 }
 
-sstring gossiper::get_gossip_status(const endpoint_state& ep_state) const {
-    auto app_state = ep_state.get_application_state(application_state::STATUS);
-    if (!app_state) {
-        return "";
-    }
-    auto value = app_state->value;
-    std::vector<sstring> pieces;
-    boost::split(pieces, value, boost::is_any_of(","));
-    assert(pieces.size() > 0);
-    return pieces[0];
-}
-
-sstring gossiper::get_gossip_status(const inet_address& endpoint) const {
-    auto ep_state = get_endpoint_state_for_endpoint(endpoint);
-    if (!ep_state) {
-        return "";
-    }
-    return get_gossip_status(*ep_state);
-}
 
 } // namespace gms

gms/gossiper.hh

@@ -78,7 +78,7 @@ class i_failure_detector;
  */
 class gossiper : public i_failure_detection_event_listener, public seastar::async_sharded_service<gossiper> {
 public:
-    using clk = std::chrono::steady_clock;
+    using clk = std::chrono::high_resolution_clock;
 private:
     using messaging_verb = net::messaging_verb;
     using messaging_service = net::messaging_service;
@@ -497,9 +497,6 @@ public:
     bool is_silent_shutdown_state(const endpoint_state& ep_state) const;
     void mark_as_shutdown(const inet_address& endpoint);
     void force_newer_generation();
-private:
-    sstring get_gossip_status(const endpoint_state& ep_state) const;
-    sstring get_gossip_status(const inet_address& endpoint) const;
 };
 
 extern distributed<gossiper> _the_gossiper;

gms/i_endpoint_state_change_subscriber.hh

@@ -65,9 +65,9 @@ public:
      */
     virtual void on_join(inet_address endpoint, endpoint_state ep_state) = 0;
 
-    virtual void before_change(inet_address endpoint, endpoint_state current_state, application_state new_statekey, const versioned_value& newvalue) = 0;
+    virtual void before_change(inet_address endpoint, endpoint_state current_state, application_state new_statekey, versioned_value newvalue) = 0;
 
-    virtual void on_change(inet_address endpoint, application_state state, const versioned_value& value) = 0;
+    virtual void on_change(inet_address endpoint, application_state state, versioned_value value) = 0;
 
     virtual void on_alive(inet_address endpoint, endpoint_state state) = 0;
 

gms/versioned_value.hh

@@ -96,6 +96,11 @@ public:
                value   == other.value;
     }
 
+    versioned_value()
+        : version(version_generator::get_next_version())
+        , value("") {
+    }
+
 private:
     versioned_value(const sstring& value, int version = version_generator::get_next_version())
         : version(version), value(value) {

init.cc

@@ -45,49 +45,10 @@ future<> init_storage_service(distributed<database>& db) {
     });
 }
 
-future<> init_ms_fd_gossiper(sstring listen_address
-                , uint16_t storage_port
-                , uint16_t ssl_storage_port
-                , sstring ms_encrypt_what
-                , sstring ms_trust_store
-                , sstring ms_cert
-                , sstring ms_key
-                , db::seed_provider_type seed_provider
-                , sstring cluster_name
-                , double phi)
-{
+future<> init_ms_fd_gossiper(sstring listen_address, uint16_t port, db::seed_provider_type seed_provider, sstring cluster_name, double phi) {
     const gms::inet_address listen(listen_address);
-
-    using encrypt_what = net::messaging_service::encrypt_what;
-    using namespace seastar::tls;
-
-    encrypt_what ew = encrypt_what::none;
-    if (ms_encrypt_what == "all") {
-        ew = encrypt_what::all;
-    } else if (ms_encrypt_what == "dc") {
-        ew = encrypt_what::dc;
-    } else if (ms_encrypt_what == "rack") {
-        ew = encrypt_what::rack;
-    }
-
-    future<> f = make_ready_future<>();
-    ::shared_ptr<server_credentials> creds;
-
-    if (ew != encrypt_what::none) {
-        // note: credentials are immutable after this, and ok to share across shards
-        creds = ::make_shared<server_credentials>(::make_shared<dh_params>(dh_params::level::MEDIUM));
-        f = creds->set_x509_key_file(ms_cert, ms_key, x509_crt_format::PEM).then([creds, ms_trust_store] {
-            return ms_trust_store.empty()
-                            ? creds->set_system_trust()
-                                            : creds->set_x509_trust_file(ms_trust_store, x509_crt_format::PEM)
-                                              ;
-
-        });
-    }
-
     // Init messaging_service
-    return f.then([listen, storage_port, creds, ssl_storage_port, ew] {
-        return net::get_messaging_service().start(listen, storage_port, ew, ssl_storage_port, creds);
+    return net::get_messaging_service().start(listen, std::move(port)).then([] {
         // #293 - do not stop anything
         //engine().at_exit([] { return net::get_messaging_service().stop(); });
     }).then([phi] {