diff --git a/IDL.md b/IDL.md new file mode 100644 index 0000000000..3a3d085cc6 --- /dev/null +++ b/IDL.md @@ -0,0 +1,103 @@ +#IDL definition +The schema we use similar to c++ schema. +Use class or struct similar to the object you need the serializer for. +Use namespace when applicable. + +##keywords +* class/struct - a class or a struct like C++ + class/struct can have final or stub marker +* namespace - has the same C++ meaning +* enum class - has the same C++ meaning +* final modifier for class - when a class mark as final it will not contain a size parameter. Note that final class cannot be extended by future version, so use with care +* stub class - when a class is mark as stub, it means that no code will be generated for this class and it is only there as a documentation. +* version attributes - mark with [[version id ]] mark that a field is available from a specific version +* template - A template class definition like C++ +##Syntax + +###Namespace +``` +namespace ns_name { namespace-body } +``` +* ns_name: either a previously unused identifier, in which case this is original-namespace-definition or the name of a namespace, in which case this is extension-namespace-definition +* namespace-body: possibly empty sequence of declarations of any kind (including class and struct definitions as well as nested namespaces) + +###class/struct +` +class-key class-name final(optional) stub(optional) { member-specification } ;(optional) +` +* class-key: one of class or struct. +* class-name: the name of the class that's being defined. optionally followed by keyword final, optionally followed by keyword stub +* final: when a class mark as final, it means it can not be extended and there is no need to serialize its size, use with care. +* stub: when a class is mark as stub, it means no code will generate for it and it is added for documentation only. +* member-specification: list of access specifiers, and public member accessor see class member below. +* to be compatible with C++ a class definition can be followed by a semicolon. +###enum +`enum-key identifier enum-base { enumerator-list(optional) }` +* enum-key: only enum class is supported +* identifier: the name of the enumeration that's being declared. +* enum-base: colon (:), followed by a type-specifier-seq that names an integral type (see the C++ standard for the full list of all possible integral types). +* enumerator-list: comma-separated list of enumerator definitions, each of which is either simply an identifier, which becomes the name of the enumerator, or an identifier with an initializer: identifier = integral value. +Note that though C++ allows constexpr as an initialize value, it makes the documentation less readable, hence is not permitted. + +###class member +`type member-access attributes(optional) default-value(optional);` +* type: Any valid C++ type, following the C++ notation. note that there should be a serializer for the type, but deceleration order is not mandatory +* member-access: is the way the member can be access. If the member is public it can be the name itself. if not it could be a getter function that should be followed by braces. Note that getter can (and probably should) be const methods. +* attributes: Attributes define by square brackets. Currently are use to mark a version in which a specific member was added [ [ version version-number] ] would mark that the specific member was added in the given version number. + +###template +`template < parameter-list > class-declaration` +* parameter-list - a non-empty comma-separated list of the template parameters. +* class-decleration - (See class section) The class name declared become a template name. + +##IDL example +Forward slashes comments are ignored until the end of the line. +``` +namespace utils { +// An example of a stub class +class UUID stub { + int64_t most_sig_bits; + int64_t least_sig_bits; +} +} + +namespace gms { +//an enum example +enum class application_state:int {STATUS = 0, + LOAD, + SCHEMA, + DC}; + +// example of final class +class versioned_value final { +// getter and setter as public member + int version; + sstring value; +} + +class heart_beat_state { +//getter as function + int32_t get_generation(); +//default value example + int32_t get_heart_beat_version() = 1; +} + +class endpoint_state { + heart_beat_state get_heart_beat_state(); + std::map get_application_state_map(); +} + +class gossip_digest { + inet_address get_endpoint(); + int32_t get_generation(); +//mark that a field was added on a specific version + int32_t get_max_version() [ [version 0.14.2] ]; +} + +class gossip_digest_ack { + std::vector digests(); + std::map get_endpoint_state_map(); +} +} +``` + diff --git a/README.md b/README.md index c9a0215b68..8d8df373ea 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ git submodule update --recursive * Installing required packages: ``` -sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel numactl-devel hwloc-devel libpciaccess-devel libxml2-devel +sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel numactl-devel hwloc-devel libpciaccess-devel libxml2-devel python3-pyparsing ``` * Build Scylla diff --git a/configure.py b/configure.py index 12e15bb97f..545f0764bd 100755 --- a/configure.py +++ b/configure.py @@ -465,8 +465,12 @@ api = ['api/api.cc', 'api/api-doc/system.json', 'api/system.cc' ] +idls = ['idl/gossip_digest.idl.hh', + ] -scylla_tests_dependencies = scylla_core + [ +serialize = idls + ['serializer.inc.hh'] + +scylla_tests_dependencies = scylla_core + api + serialize + [ 'tests/cql_test_env.cc', 'tests/cql_assertions.cc', 'tests/result_set_assertions.cc', @@ -479,7 +483,7 @@ scylla_tests_seastar_deps = [ ] deps = { - 'scylla': ['main.cc'] + scylla_core + api, + 'scylla': serialize + ['main.cc'] + scylla_core + api, } tests_not_using_seastar_test_framework = set([ @@ -656,6 +660,12 @@ with open(buildfile, 'w') as f: rule swagger command = seastar/json/json2code.py -f $in -o $out description = SWAGGER $out + rule serializer + command = ./idl-compiler.py --ns ser -f $in -o $out + description = IDL compiler $out + rule serializer_inc + command = ./idl-compiler.py $in -o $out + description = Combine IDLs $out rule ninja command = {ninja} -C $subdir $target restat = 1 @@ -692,6 +702,8 @@ with open(buildfile, 'w') as f: compiles = {} ragels = {} swaggers = {} + serializers = {} + serializer_inc = {} thrifts = set() antlr3_grammars = set() for binary in build_artifacts: @@ -745,6 +757,12 @@ with open(buildfile, 'w') as f: elif src.endswith('.rl'): hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh') ragels[hh] = src + elif src.endswith('.idl.hh'): + hh = '$builddir/' + mode + '/gen/' + src.replace('.idl.hh', '.dist.hh') + serializers[hh] = src + elif src.endswith('.inc.hh'): + hh = '$builddir/' + mode + '/gen/' + src + serializer_inc[hh] = src elif src.endswith('.json'): hh = '$builddir/' + mode + '/gen/' + src + '.hh' swaggers[hh] = src @@ -763,6 +781,8 @@ with open(buildfile, 'w') as f: for g in antlr3_grammars: gen_headers += g.headers('$builddir/{}/gen'.format(mode)) gen_headers += list(swaggers.keys()) + gen_headers += list(serializers.keys()) + gen_headers += list(serializer_inc.keys()) f.write('build {}: cxx.{} {} || {} \n'.format(obj, mode, src, ' '.join(gen_headers))) if src in extra_cxxflags: f.write(' cxxflags = {seastar_cflags} $cxxflags $cxxflags_{mode} {extra_cxxflags}\n'.format(mode = mode, extra_cxxflags = extra_cxxflags[src], **modeval)) @@ -772,6 +792,11 @@ with open(buildfile, 'w') as f: for hh in swaggers: src = swaggers[hh] f.write('build {}: swagger {}\n'.format(hh,src)) + for hh in serializers: + src = serializers[hh] + f.write('build {}: serializer {}\n'.format(hh,src)) + for hh in serializer_inc: + f.write('build {}: serializer_inc {}\n'.format(hh, " ".join(serializers.keys()))) for thrift in thrifts: outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode))) f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source)) diff --git a/gms/endpoint_state.hh b/gms/endpoint_state.hh index 508e2dae31..62b223149a 100644 --- a/gms/endpoint_state.hh +++ b/gms/endpoint_state.hh @@ -81,6 +81,14 @@ public: , _is_alive(true) { } + endpoint_state(heart_beat_state&& initial_hb_state, + const std::map& application_state) + : _heart_beat_state(std::move(initial_hb_state)) + ,_application_state(application_state) + , _update_timestamp(clk::now()) + , _is_alive(true) { + } + heart_beat_state& get_heart_beat_state() { return _heart_beat_state; } diff --git a/gms/inet_address.hh b/gms/inet_address.hh index 809b3665b3..50baac3785 100644 --- a/gms/inet_address.hh +++ b/gms/inet_address.hh @@ -37,6 +37,9 @@ public: inet_address(int32_t ip) : _addr(uint32_t(ip)) { } + explicit inet_address(uint32_t ip) + : _addr(ip) { + } inet_address(net::ipv4_address&& addr) : _addr(std::move(addr)) {} const net::ipv4_address& addr() const { diff --git a/gms/versioned_value.cc b/gms/versioned_value.cc index f105830fb6..0c9d8d75a2 100644 --- a/gms/versioned_value.cc +++ b/gms/versioned_value.cc @@ -36,6 +36,7 @@ * along with Scylla. If not, see . */ #include "gms/versioned_value.hh" +#include "message/messaging_service.hh" namespace gms { @@ -67,4 +68,8 @@ size_t versioned_value::serialized_size() const { return serialize_string_size(value) + serialize_int32_size; } +versioned_value versioned_value::factory::network_version() { + return versioned_value(sprint("%s",net::messaging_service::current_version)); +} + } diff --git a/gms/versioned_value.hh b/gms/versioned_value.hh index e90db8b820..cc64ae169b 100644 --- a/gms/versioned_value.hh +++ b/gms/versioned_value.hh @@ -46,7 +46,6 @@ #include "gms/inet_address.hh" #include "dht/i_partitioner.hh" #include "to_string.hh" -#include "message/messaging_service.hh" #include "version.hh" #include #include @@ -96,7 +95,7 @@ public: value == other.value; } -private: +public: versioned_value(const sstring& value, int version = version_generator::get_next_version()) : version(version), value(value) { #if 0 @@ -112,8 +111,10 @@ private: : version(version), value(std::move(value)) { } + versioned_value() + : version(-1) { + } -public: int compare_to(const versioned_value &value) { return version - value.version; } @@ -228,9 +229,7 @@ public: return versioned_value(version::release()); } - versioned_value network_version() { - return versioned_value(sprint("%s",net::messaging_service::current_version)); - } + versioned_value network_version(); versioned_value internal_ip(const sstring &private_ip) { return versioned_value(private_ip); diff --git a/idl-compiler.py b/idl-compiler.py new file mode 100755 index 0000000000..cddb30b8aa --- /dev/null +++ b/idl-compiler.py @@ -0,0 +1,329 @@ +#!/usr/bin/python3 +# +# Copyright 2016 ScyllaDB +# + +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . + +import json +import sys +import re +import glob +import argparse +import os +from string import Template +import pyparsing as pp + + +EXTENSION = '.idl.hh' +READ_BUFF = 'input_buffer' +WRITE_BUFF = 'output_buffer' +SERIALIZER = 'serialize' +DESERIALIZER = 'deserialize' +SETSIZE = 'set_size' +SIZETYPE = 'size_type' + +parser = argparse.ArgumentParser(description="""Generate serializer helper function""") + +parser.add_argument('-o', help='Output file', default='') +parser.add_argument('-f', help='input file', default='') +parser.add_argument('--ns', help="""namespace, when set function will be created +under the given namespace""", default='') +parser.add_argument('file', nargs='*', help="combine one or more file names for the genral include files") + +config = parser.parse_args() + + + +def fprint(f, *args): + for arg in args: + f.write(arg) + +def fprintln(f, *args): + for arg in args: + f.write(arg) + f.write('\n') + +def print_cw(f): + fprintln(f, """ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + + + /* + * This is an auto-generated code, do not modify directly. + */ +#pragma once + """) + +def parse_file(file_name): + first = pp.Word(pp.alphas + "_", exact=1) + rest = pp.Word(pp.alphanums + "_") + + number = pp.Word(pp.nums) + identifier = pp.Combine(first + pp.Optional(rest)) + + lbrace = pp.Literal('{').suppress() + rbrace = pp.Literal('}').suppress() + cls = pp.Literal('class') + colon = pp.Literal(":") + semi = pp.Literal(";").suppress() + langle = pp.Literal("<") + rangle = pp.Literal(">") + equals = pp.Literal("=") + comma = pp.Literal(",") + lparen = pp.Literal("(") + rparen = pp.Literal(")") + lbrack = pp.Literal("[") + rbrack = pp.Literal("]") + mins = pp.Literal("-") + struct = pp.Literal('struct') + template = pp.Literal('template') + final = pp.Literal('final').setResultsName("final") + stub = pp.Literal('stub').setResultsName("stub") + with_colon = pp.Word(pp.alphanums + "_" + ":") + btype = with_colon + type = pp.Forward() + nestedParens = pp.nestedExpr('<', '>') + + tmpl = pp.Group(btype + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress()) + type << (tmpl | btype) + enum_lit = pp.Literal('enum') + enum_class = pp.Group(enum_lit + cls) + ns = pp.Literal("namespace") + + enum_init = equals.suppress() + pp.Optional(mins) + number + enum_value = pp.Group(identifier + pp.Optional(enum_init)) + enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace) + content = pp.Forward() + + member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen))) + attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress()) + namespace = pp.Group(ns.setResultsName("type") + identifier.setResultsName("name") + lbrace + pp.Group(pp.OneOrMore(content)).setResultsName("content") + rbrace) + enum = pp.Group(enum_class.setResultsName("type") + identifier.setResultsName("name") + colon.suppress() + identifier.setResultsName("underline_type") + enum_values.setResultsName("enum_values") + pp.Optional(semi).suppress()) + default_value = equals.suppress() + pp.SkipTo(';') + class_member = pp.Group(type.setResultsName("type") + member_name.setResultsName("name") + pp.Optional(attrib).setResultsName("attribute") + pp.Optional(default_value).setResultsName("default") + semi.suppress()).setResultsName("member") + template_param = pp.Group(identifier.setResultsName("type") + identifier.setResultsName("name")) + template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param)).setResultsName("params") + rangle) + class_content = pp.Forward() + class_def = pp.Group(pp.Optional(template_def).setResultsName("template") + (cls | struct).setResultsName("type") + with_colon.setResultsName("name") + pp.Optional(final) + pp.Optional(stub) + lbrace + pp.Group(pp.OneOrMore(class_content)).setResultsName("members") + rbrace + pp.Optional(semi)) + content << (enum | class_def | namespace) + class_content << (enum | class_def | class_member) + rt = pp.OneOrMore(content) + singleLineComment = "//" + pp.restOfLine + rt.ignore(singleLineComment) + rt.ignore(pp.cStyleComment) + return rt.parseFile(file_name, parseAll=True) + + +def combine_ns(namespaces): + return "::".join(namespaces) + +def open_namespaces(namespaces): + return "".join(map(lambda a: "namespace " + a + " { ", namespaces)) + +def close_namespaces(namespaces): + return "".join(map(lambda a: "}", namespaces)) + +def set_namespace(namespaces): + ns = combine_ns(namespaces) + ns_open = open_namespaces(namespaces) + ns_close = close_namespaces(namespaces) + return [ns, ns_open, ns_close] + +def declare_class(hout, name, ns_open, ns_close): + clas_def = ns_open + name + ";" + ns_close + fprintln(hout, "\n", clas_def) + +def declear_methods(hout, name, template_param = ""): + if config.ns != '': + fprintln(hout, "namespace ", config.ns, " {") + fprintln(hout, Template(""" +template +void $ser_func(Output& buf, const $name& v); + +template +$name $deser_func(Input& buf, rpc::type<$name>);""").substitute({'ser_func': SERIALIZER, 'deser_func' : DESERIALIZER, 'name' : name, 'sizetype' : SIZETYPE, 'tmp_param' : template_param })) + if config.ns != '': + fprintln(hout, "}") + +def handle_enum(enum, hout, cout, namespaces , parent_template_param = []): + [ns, ns_open, ns_close] = set_namespace(namespaces) + temp_def = ',' + ", ".join(map(lambda a: a[0] + " " + a[1], parent_template_param)) if parent_template_param else "" + name = enum["name"] if ns == "" else ns + "::" + enum["name"] + declear_methods(hout, name, temp_def) + fprintln(cout, Template(""" +template +void $ser_func(Output& buf, const $name& v) { + serialize(buf, static_cast<$type>(v)); +} + +template +$name $deser_func(Input& buf, rpc::type<$name>) { + return static_cast<$name>(deserialize(buf, rpc::type<$type>())); +}""").substitute({'ser_func': SERIALIZER, 'deser_func' : DESERIALIZER, 'name' : name, 'size_type' : SIZETYPE, 'type': enum['underline_type'], 'temp_def' : temp_def})) + + +def join_template(lst): + return "<" + ", ".join([param_type(l) for l in lst]) + ">" + +def param_type(lst): + if isinstance(lst, str): + return lst + if len(lst) == 1: + return lst[0] + return lst[0] + join_template(lst[1]) + +def is_class(obj): + return obj["type"] == "class" or obj["type"] == "struct" + +def is_enum(obj): + try: + return not isinstance(obj["type"], str) and "".join(obj["type"]) == 'enumclass' + except: + return False + +def handle_class(cls, hout, cout, namespaces=[], parent_template_param = []): + if "stub" in cls: + return + [ns, ns_open, ns_close] = set_namespace(namespaces) + tpl = "template" in cls + template_param_list = (cls["template"][0]["params"].asList() if tpl else []) + template_param = ", ".join(map(lambda a: a[0] + " " + a[1], template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else "" + template = "template <"+ template_param +">\n" if tpl else "" + template_class_param = "<" + ",".join(map(lambda a: a[1], template_param_list)) + ">" if tpl else "" + temp_def = ',' + template_param if template_param != "" else "" + + if ns == "": + name = cls["name"] + else: + name = ns + "::" + cls["name"] + full_name = name + template_class_param + for param in cls["members"]: + if is_class(param): + handle_class(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list) + elif is_enum(param): + handle_enum(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list) + declear_methods(hout, name + template_class_param, temp_def) + modifier = "final" in cls + + fprintln(cout, Template(""" +template +void $func(Output& buf, const $name& obj) {""").substitute({'func' : SERIALIZER, 'name' : full_name, 'temp_def': temp_def})) + if not modifier: + fprintln(cout, Template(""" $set_size(buf, obj);""").substitute({'func' : SERIALIZER, 'set_size' : SETSIZE, 'name' : name, 'sizetype' : SIZETYPE})) + for param in cls["members"]: + if is_class(param) or is_enum(param): + continue + fprintln(cout, Template(""" $func(buf, obj.$var);""").substitute({'func' : SERIALIZER, 'var' : param["name"]})) + fprintln(cout, "}") + + fprintln(cout, Template(""" +template +$name$temp_param $func(Input& buf, rpc::type<$name$temp_param>) {""").substitute({'func' : DESERIALIZER, 'name' : name, 'temp_def': temp_def, 'temp_param' : template_class_param})) + if not modifier: + fprintln(cout, Template(""" $size_type size = $func(buf, rpc::type<$size_type>()); + Input in = buf.read_substream(size - sizeof($size_type));""").substitute({'func' : DESERIALIZER, 'size_type' : SIZETYPE})) + else: + fprintln(cout, """ Input& in = buf;""") + params = [] + for index, param in enumerate(cls["members"]): + if is_class(param) or is_enum(param): + continue + local_param = "__local_" + str(index) + if "attribute" in param: + deflt = param["default"] if param["default"] else param["type"] + "()" + fprintln(cout, Template(""" $typ $local = (in.size()>0) ? + $func(in, rpc::type<$typ>()) : $default;""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param, 'default': deflt})) + else: + fprintln(cout, Template(""" $typ $local = $func(in, rpc::type<$typ>());""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param})) + params.append("std::move(" + local_param + ")") + fprintln(cout, Template(""" + $name$temp_param res {$params}; + return res; +}""").substitute({'name' : name, 'params': ", ".join(params), 'temp_param' : template_class_param})) + + +def handle_objects(tree, hout, cout, namespaces=[]): + for obj in tree: + if is_class(obj): + handle_class(obj, hout, cout, namespaces) + elif is_enum(obj): + handle_enum(obj, hout, cout, namespaces) + elif obj["type"] == "namespace": + handle_objects(obj["content"], hout, cout, namespaces + [obj["name"]]) + else: + print("unknown type ", obj, obj["type"]) + +def load_file(name): + if config.o: + cout = open(config.o.replace('.hh', '.impl.hh'), "w+") + hout = open(config.o, "w+") + else: + cout = open(name.replace(EXTENSION, '.dist.impl.hh'), "w+") + hout = open(name.replace(EXTENSION, '.dist.hh'), "w+") + print_cw(hout) + fprintln(hout, """ + /* + * The generate code should be included in a header file after + * The object definition + */ + """) + print_cw(cout) + if config.ns != '': + fprintln(cout, "namespace ", config.ns, " {") + data = parse_file(name) + if data: + handle_objects(data, hout, cout) + if config.ns != '': + fprintln(cout, "}") + cout.close() + hout.close() + +def general_include(files): + name = config.o if config.o else "serializer.dist.hh" + cout = open(name.replace('.hh', '.impl.hh'), "w+") + hout = open(name, "w+") + print_cw(cout) + print_cw(hout) + for n in files: + fprintln(hout, '#include "' + n +'"') + fprintln(cout, '#include "' + n.replace(".dist.hh", '.dist.impl.hh') +'"') + cout.close() + hout.close() +if config.file: + general_include(config.file) +elif config.f != '': + load_file(config.f) diff --git a/idl/gossip_digest.idl.hh b/idl/gossip_digest.idl.hh new file mode 100644 index 0000000000..de0d052d98 --- /dev/null +++ b/idl/gossip_digest.idl.hh @@ -0,0 +1,83 @@ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ +namespace utils { +class UUID final { + int64_t most_sig_bits; + int64_t least_sig_bits; +}; + +} + +namespace gms { +enum class application_state:int {STATUS = 0, + LOAD, + SCHEMA, + DC, + RACK, + RELEASE_VERSION, + REMOVAL_COORDINATOR, + INTERNAL_IP, + RPC_ADDRESS, + X_11_PADDING, + SEVERITY, + NET_VERSION, + HOST_ID, + TOKENS, + X1, + X2, + X3, + X4, + X5, + X6, + X7, + X8, + X9, + X10}; +class inet_address final { + uint32_t raw_addr(); +}; + +class versioned_value { + sstring value; + int version; +}; + +class heart_beat_state { + int32_t get_generation(); + int32_t get_heart_beat_version(); +}; + +class endpoint_state { + gms::heart_beat_state get_heart_beat_state(); + std::map get_application_state_map(); +} + +class gossip_digest { + gms::inet_address get_endpoint(); + int32_t get_generation(); + int32_t get_max_version(); +} + +class gossip_digest_ack { + std::vector get_gossip_digest_list(); + std::map get_endpoint_state_map(); +} +} diff --git a/message/messaging_service.cc b/message/messaging_service.cc index 125ec2c875..7ea5146832 100644 --- a/message/messaging_service.cc +++ b/message/messaging_service.cc @@ -36,9 +36,21 @@ #include "dht/i_partitioner.hh" #include "range.hh" #include "frozen_schema.hh" +#include "serializer_impl.hh" namespace net { +template +void write(serializer, Output& out, const gms::gossip_digest_ack& data) { + ser::serialize(out, data); +} + +template +gms::gossip_digest_ack +read(serializer, Input& in, rpc::type type) { + return ser::deserialize(in, type); +} + static logging::logger logger("messaging_service"); using inet_address = gms::inet_address; diff --git a/serializer.hh b/serializer.hh new file mode 100644 index 0000000000..d6f15be733 --- /dev/null +++ b/serializer.hh @@ -0,0 +1,155 @@ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ +#pragma once + +#include +#include "core/sstring.hh" +#include +namespace rpc { + class simple_output_stream; + class measuring_output_stream; +} +namespace ser { +using size_type = uint32_t; + +template +inline T deserialize_integral(Input& input) { + static_assert(std::is_integral::value, "T should be integral"); + T data; + input.read(reinterpret_cast(&data), sizeof(T)); + return le_to_cpu(data); +} + +template +inline void serialize_integral(Output& output, T data) { + static_assert(std::is_integral::value, "T should be integral"); + data = cpu_to_le(data); + output.write(reinterpret_cast(&data), sizeof(T)); +} + +// For integer type +template +bool deserialize(Input& input, rpc::type) { + return deserialize(input, rpc::type()); +} +template +int8_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint8_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +int16_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint16_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +int32_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint32_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +int64_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint64_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} + +template +void serialize(Output& output, bool data) { + serialize(output, uint8_t(data)); +} +template +void serialize(Output& output, int8_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint8_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, int16_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint16_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, int32_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint32_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, int64_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint64_t data) { + serialize_integral(output, data); +} +template +void safe_serialize_as_uint32(Output& output, uint64_t data); + +// For vectors + +template +inline void serialize(Output& out, const std::vector& v); +template +inline std::vector deserialize(Input& in, rpc::type>); + +template +inline void serialize(Output& out, const std::map& v); +template +inline std::map deserialize(Input& in, rpc::type>); +template +size_type get_sizeof(const T& obj); +// For sstring +template +void serialize(Output& out, const sstring& v); +template +sstring deserialize(Input& in, rpc::type); + +template +void set_size(rpc::simple_output_stream& os, const T& obj); + +template +void set_size(rpc::measuring_output_stream& os, const T& obj); +} + +/* + * Import the auto generated forward decleration code + */ + +#include "serializer.inc.hh" diff --git a/serializer_impl.hh b/serializer_impl.hh new file mode 100644 index 0000000000..46b9b9ab98 --- /dev/null +++ b/serializer_impl.hh @@ -0,0 +1,126 @@ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include "serializer.hh" +#include "serializer.inc.impl.hh" + +namespace ser { + +template +void set_size(rpc::simple_output_stream& os, const T& obj) { + serialize(os, get_sizeof(obj)); +} + +template +void set_size(rpc::measuring_output_stream& os, const T& obj) { + serialize(os, uint32_t(0)); +} + + +template +void safe_serialize_as_uint32(Output& out, uint64_t data) { + if (data > std::numeric_limits::max()) { + throw std::runtime_error("Size is too big for serialization"); + } + serialize(out, uint32_t(data)); +} +template +inline void serialize(Output& out, const std::vector& v) { + safe_serialize_as_uint32(out, v.size()); + for (auto&& e : v) { + serialize(out, e); + } +} +template +inline std::vector deserialize(Input& in, rpc::type>) { + auto sz = deserialize(in, rpc::type()); + std::vector v; + v.reserve(sz); + while (sz--) { + v.push_back(deserialize(in, rpc::type())); + } + return v; +} + +template +inline void serialize(Output& out, const std::map& v) { + safe_serialize_as_uint32(out, v.size()); + for (auto&& e : v) { + serialize(out, e.first); + serialize(out, e.second); + } +} +template +inline std::map deserialize(Input& in, rpc::type>) { + auto sz = deserialize(in, rpc::type()); + std::map m; + while (sz--) { + K k = deserialize(in, rpc::type()); + V v = deserialize(in, rpc::type()); + m[k] = v; + } + return m; +} + +template +inline void serialize(Output& out, const std::experimental::optional& v) { + serialize(out, bool(v)); + if (v) { + serialize(out, v.value()); + } +} +template +inline std::experimental::optional deserialize(Input& in, rpc::type>) { + std::experimental::optional v; + auto b = deserialize(in, rpc::type()); + if (b) { + v = deserialize(in, rpc::type()); + } + return v; +} + +template +void serialize(Output& out, const sstring& v) { + safe_serialize_as_uint32(out, uint32_t(v.size())); + out.write(v.begin(), v.size()); +} +template +sstring deserialize(Input& in, rpc::type) { + auto sz = deserialize(in, rpc::type()); + sstring v(sstring::initialized_later(), sz); + in.read(v.begin(), sz); + return v; +} + +template +size_type get_sizeof(const T& obj) { + rpc::measuring_output_stream ms; + serialize(ms, obj); + auto size = ms.size(); + if (size > std::numeric_limits::max()) { + throw std::runtime_error("Object is too big for get_sizeof"); + } + return size; +} + +}