From 4a34ed82a3090b5c35e84a1af336c67824c8b010 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 7 Jan 2016 21:45:24 +0200 Subject: [PATCH 01/10] Add code generation for serializer and deserializer The code generation takes a schema file and create two files from it, one with a dist.hh extension containing the forward declarations and a second with dist.impl.hh with the actual implementation. Because the rpc uses templating for the input and output streams. The generated functions are templates. For each class, struct or enum, two functions are created: serialize - that gets the output buffer as template parameter and serialize the object to it. There must be a public way to get to each of the parameters in the class (either a getter or the parameter should be public) deserialize - that gets an input buffer, and return the deserialize object (and by reference the number of char it read). To create the return object, the class must have a public constructor with all of its parameters. The solution description can be found here: https://github.com/scylladb/scylla/wiki/Serializer-Deserializer-Code-generation Signed-off-by: Amnon Heiman --- idl-compiler.py | 329 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100755 idl-compiler.py diff --git a/idl-compiler.py b/idl-compiler.py new file mode 100755 index 0000000000..cddb30b8aa --- /dev/null +++ b/idl-compiler.py @@ -0,0 +1,329 @@ +#!/usr/bin/python3 +# +# Copyright 2016 ScyllaDB +# + +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . + +import json +import sys +import re +import glob +import argparse +import os +from string import Template +import pyparsing as pp + + +EXTENSION = '.idl.hh' +READ_BUFF = 'input_buffer' +WRITE_BUFF = 'output_buffer' +SERIALIZER = 'serialize' +DESERIALIZER = 'deserialize' +SETSIZE = 'set_size' +SIZETYPE = 'size_type' + +parser = argparse.ArgumentParser(description="""Generate serializer helper function""") + +parser.add_argument('-o', help='Output file', default='') +parser.add_argument('-f', help='input file', default='') +parser.add_argument('--ns', help="""namespace, when set function will be created +under the given namespace""", default='') +parser.add_argument('file', nargs='*', help="combine one or more file names for the genral include files") + +config = parser.parse_args() + + + +def fprint(f, *args): + for arg in args: + f.write(arg) + +def fprintln(f, *args): + for arg in args: + f.write(arg) + f.write('\n') + +def print_cw(f): + fprintln(f, """ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + + + /* + * This is an auto-generated code, do not modify directly. + */ +#pragma once + """) + +def parse_file(file_name): + first = pp.Word(pp.alphas + "_", exact=1) + rest = pp.Word(pp.alphanums + "_") + + number = pp.Word(pp.nums) + identifier = pp.Combine(first + pp.Optional(rest)) + + lbrace = pp.Literal('{').suppress() + rbrace = pp.Literal('}').suppress() + cls = pp.Literal('class') + colon = pp.Literal(":") + semi = pp.Literal(";").suppress() + langle = pp.Literal("<") + rangle = pp.Literal(">") + equals = pp.Literal("=") + comma = pp.Literal(",") + lparen = pp.Literal("(") + rparen = pp.Literal(")") + lbrack = pp.Literal("[") + rbrack = pp.Literal("]") + mins = pp.Literal("-") + struct = pp.Literal('struct') + template = pp.Literal('template') + final = pp.Literal('final').setResultsName("final") + stub = pp.Literal('stub').setResultsName("stub") + with_colon = pp.Word(pp.alphanums + "_" + ":") + btype = with_colon + type = pp.Forward() + nestedParens = pp.nestedExpr('<', '>') + + tmpl = pp.Group(btype + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress()) + type << (tmpl | btype) + enum_lit = pp.Literal('enum') + enum_class = pp.Group(enum_lit + cls) + ns = pp.Literal("namespace") + + enum_init = equals.suppress() + pp.Optional(mins) + number + enum_value = pp.Group(identifier + pp.Optional(enum_init)) + enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace) + content = pp.Forward() + + member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen))) + attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress()) + namespace = pp.Group(ns.setResultsName("type") + identifier.setResultsName("name") + lbrace + pp.Group(pp.OneOrMore(content)).setResultsName("content") + rbrace) + enum = pp.Group(enum_class.setResultsName("type") + identifier.setResultsName("name") + colon.suppress() + identifier.setResultsName("underline_type") + enum_values.setResultsName("enum_values") + pp.Optional(semi).suppress()) + default_value = equals.suppress() + pp.SkipTo(';') + class_member = pp.Group(type.setResultsName("type") + member_name.setResultsName("name") + pp.Optional(attrib).setResultsName("attribute") + pp.Optional(default_value).setResultsName("default") + semi.suppress()).setResultsName("member") + template_param = pp.Group(identifier.setResultsName("type") + identifier.setResultsName("name")) + template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param)).setResultsName("params") + rangle) + class_content = pp.Forward() + class_def = pp.Group(pp.Optional(template_def).setResultsName("template") + (cls | struct).setResultsName("type") + with_colon.setResultsName("name") + pp.Optional(final) + pp.Optional(stub) + lbrace + pp.Group(pp.OneOrMore(class_content)).setResultsName("members") + rbrace + pp.Optional(semi)) + content << (enum | class_def | namespace) + class_content << (enum | class_def | class_member) + rt = pp.OneOrMore(content) + singleLineComment = "//" + pp.restOfLine + rt.ignore(singleLineComment) + rt.ignore(pp.cStyleComment) + return rt.parseFile(file_name, parseAll=True) + + +def combine_ns(namespaces): + return "::".join(namespaces) + +def open_namespaces(namespaces): + return "".join(map(lambda a: "namespace " + a + " { ", namespaces)) + +def close_namespaces(namespaces): + return "".join(map(lambda a: "}", namespaces)) + +def set_namespace(namespaces): + ns = combine_ns(namespaces) + ns_open = open_namespaces(namespaces) + ns_close = close_namespaces(namespaces) + return [ns, ns_open, ns_close] + +def declare_class(hout, name, ns_open, ns_close): + clas_def = ns_open + name + ";" + ns_close + fprintln(hout, "\n", clas_def) + +def declear_methods(hout, name, template_param = ""): + if config.ns != '': + fprintln(hout, "namespace ", config.ns, " {") + fprintln(hout, Template(""" +template +void $ser_func(Output& buf, const $name& v); + +template +$name $deser_func(Input& buf, rpc::type<$name>);""").substitute({'ser_func': SERIALIZER, 'deser_func' : DESERIALIZER, 'name' : name, 'sizetype' : SIZETYPE, 'tmp_param' : template_param })) + if config.ns != '': + fprintln(hout, "}") + +def handle_enum(enum, hout, cout, namespaces , parent_template_param = []): + [ns, ns_open, ns_close] = set_namespace(namespaces) + temp_def = ',' + ", ".join(map(lambda a: a[0] + " " + a[1], parent_template_param)) if parent_template_param else "" + name = enum["name"] if ns == "" else ns + "::" + enum["name"] + declear_methods(hout, name, temp_def) + fprintln(cout, Template(""" +template +void $ser_func(Output& buf, const $name& v) { + serialize(buf, static_cast<$type>(v)); +} + +template +$name $deser_func(Input& buf, rpc::type<$name>) { + return static_cast<$name>(deserialize(buf, rpc::type<$type>())); +}""").substitute({'ser_func': SERIALIZER, 'deser_func' : DESERIALIZER, 'name' : name, 'size_type' : SIZETYPE, 'type': enum['underline_type'], 'temp_def' : temp_def})) + + +def join_template(lst): + return "<" + ", ".join([param_type(l) for l in lst]) + ">" + +def param_type(lst): + if isinstance(lst, str): + return lst + if len(lst) == 1: + return lst[0] + return lst[0] + join_template(lst[1]) + +def is_class(obj): + return obj["type"] == "class" or obj["type"] == "struct" + +def is_enum(obj): + try: + return not isinstance(obj["type"], str) and "".join(obj["type"]) == 'enumclass' + except: + return False + +def handle_class(cls, hout, cout, namespaces=[], parent_template_param = []): + if "stub" in cls: + return + [ns, ns_open, ns_close] = set_namespace(namespaces) + tpl = "template" in cls + template_param_list = (cls["template"][0]["params"].asList() if tpl else []) + template_param = ", ".join(map(lambda a: a[0] + " " + a[1], template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else "" + template = "template <"+ template_param +">\n" if tpl else "" + template_class_param = "<" + ",".join(map(lambda a: a[1], template_param_list)) + ">" if tpl else "" + temp_def = ',' + template_param if template_param != "" else "" + + if ns == "": + name = cls["name"] + else: + name = ns + "::" + cls["name"] + full_name = name + template_class_param + for param in cls["members"]: + if is_class(param): + handle_class(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list) + elif is_enum(param): + handle_enum(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list) + declear_methods(hout, name + template_class_param, temp_def) + modifier = "final" in cls + + fprintln(cout, Template(""" +template +void $func(Output& buf, const $name& obj) {""").substitute({'func' : SERIALIZER, 'name' : full_name, 'temp_def': temp_def})) + if not modifier: + fprintln(cout, Template(""" $set_size(buf, obj);""").substitute({'func' : SERIALIZER, 'set_size' : SETSIZE, 'name' : name, 'sizetype' : SIZETYPE})) + for param in cls["members"]: + if is_class(param) or is_enum(param): + continue + fprintln(cout, Template(""" $func(buf, obj.$var);""").substitute({'func' : SERIALIZER, 'var' : param["name"]})) + fprintln(cout, "}") + + fprintln(cout, Template(""" +template +$name$temp_param $func(Input& buf, rpc::type<$name$temp_param>) {""").substitute({'func' : DESERIALIZER, 'name' : name, 'temp_def': temp_def, 'temp_param' : template_class_param})) + if not modifier: + fprintln(cout, Template(""" $size_type size = $func(buf, rpc::type<$size_type>()); + Input in = buf.read_substream(size - sizeof($size_type));""").substitute({'func' : DESERIALIZER, 'size_type' : SIZETYPE})) + else: + fprintln(cout, """ Input& in = buf;""") + params = [] + for index, param in enumerate(cls["members"]): + if is_class(param) or is_enum(param): + continue + local_param = "__local_" + str(index) + if "attribute" in param: + deflt = param["default"] if param["default"] else param["type"] + "()" + fprintln(cout, Template(""" $typ $local = (in.size()>0) ? + $func(in, rpc::type<$typ>()) : $default;""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param, 'default': deflt})) + else: + fprintln(cout, Template(""" $typ $local = $func(in, rpc::type<$typ>());""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param})) + params.append("std::move(" + local_param + ")") + fprintln(cout, Template(""" + $name$temp_param res {$params}; + return res; +}""").substitute({'name' : name, 'params': ", ".join(params), 'temp_param' : template_class_param})) + + +def handle_objects(tree, hout, cout, namespaces=[]): + for obj in tree: + if is_class(obj): + handle_class(obj, hout, cout, namespaces) + elif is_enum(obj): + handle_enum(obj, hout, cout, namespaces) + elif obj["type"] == "namespace": + handle_objects(obj["content"], hout, cout, namespaces + [obj["name"]]) + else: + print("unknown type ", obj, obj["type"]) + +def load_file(name): + if config.o: + cout = open(config.o.replace('.hh', '.impl.hh'), "w+") + hout = open(config.o, "w+") + else: + cout = open(name.replace(EXTENSION, '.dist.impl.hh'), "w+") + hout = open(name.replace(EXTENSION, '.dist.hh'), "w+") + print_cw(hout) + fprintln(hout, """ + /* + * The generate code should be included in a header file after + * The object definition + */ + """) + print_cw(cout) + if config.ns != '': + fprintln(cout, "namespace ", config.ns, " {") + data = parse_file(name) + if data: + handle_objects(data, hout, cout) + if config.ns != '': + fprintln(cout, "}") + cout.close() + hout.close() + +def general_include(files): + name = config.o if config.o else "serializer.dist.hh" + cout = open(name.replace('.hh', '.impl.hh'), "w+") + hout = open(name, "w+") + print_cw(cout) + print_cw(hout) + for n in files: + fprintln(hout, '#include "' + n +'"') + fprintln(cout, '#include "' + n.replace(".dist.hh", '.dist.impl.hh') +'"') + cout.close() + hout.close() +if config.file: + general_include(config.file) +elif config.f != '': + load_file(config.f) From ddc3fe1328f647b5a612e1908b0f105613d29f70 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 8 Jan 2016 11:35:17 +0200 Subject: [PATCH 02/10] endpoint_state adds a constructor for all serialized parameters An external deserialize function needs a constructor with all the serialized parameters. Signed-off-by: Amnon Heiman --- gms/endpoint_state.hh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gms/endpoint_state.hh b/gms/endpoint_state.hh index 508e2dae31..62b223149a 100644 --- a/gms/endpoint_state.hh +++ b/gms/endpoint_state.hh @@ -81,6 +81,14 @@ public: , _is_alive(true) { } + endpoint_state(heart_beat_state&& initial_hb_state, + const std::map& application_state) + : _heart_beat_state(std::move(initial_hb_state)) + ,_application_state(application_state) + , _update_timestamp(clk::now()) + , _is_alive(true) { + } + heart_beat_state& get_heart_beat_state() { return _heart_beat_state; } From 8a4d211a990727cbcc6ade08592dfbb258015207 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 8 Jan 2016 11:45:34 +0200 Subject: [PATCH 03/10] Changes the versioned_value to make serializeble This patch contains two changes, it make the constructor with parameters public. And it removes the dependency in messaging_service.hh from the header file by moving some of the code to the .cc file. Signed-off-by: Amnon Heiman --- gms/versioned_value.cc | 5 +++++ gms/versioned_value.hh | 11 +++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/gms/versioned_value.cc b/gms/versioned_value.cc index f105830fb6..0c9d8d75a2 100644 --- a/gms/versioned_value.cc +++ b/gms/versioned_value.cc @@ -36,6 +36,7 @@ * along with Scylla. If not, see . */ #include "gms/versioned_value.hh" +#include "message/messaging_service.hh" namespace gms { @@ -67,4 +68,8 @@ size_t versioned_value::serialized_size() const { return serialize_string_size(value) + serialize_int32_size; } +versioned_value versioned_value::factory::network_version() { + return versioned_value(sprint("%s",net::messaging_service::current_version)); +} + } diff --git a/gms/versioned_value.hh b/gms/versioned_value.hh index e90db8b820..cc64ae169b 100644 --- a/gms/versioned_value.hh +++ b/gms/versioned_value.hh @@ -46,7 +46,6 @@ #include "gms/inet_address.hh" #include "dht/i_partitioner.hh" #include "to_string.hh" -#include "message/messaging_service.hh" #include "version.hh" #include #include @@ -96,7 +95,7 @@ public: value == other.value; } -private: +public: versioned_value(const sstring& value, int version = version_generator::get_next_version()) : version(version), value(value) { #if 0 @@ -112,8 +111,10 @@ private: : version(version), value(std::move(value)) { } + versioned_value() + : version(-1) { + } -public: int compare_to(const versioned_value &value) { return version - value.version; } @@ -228,9 +229,7 @@ public: return versioned_value(version::release()); } - versioned_value network_version() { - return versioned_value(sprint("%s",net::messaging_service::current_version)); - } + versioned_value network_version(); versioned_value internal_ip(const sstring &private_ip) { return versioned_value(private_ip); From d27734b9be9996ee688ff0b868dc349d4b8b800a Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 8 Jan 2016 11:48:51 +0200 Subject: [PATCH 04/10] Add a constructor to inet_address from uint32_t inet_address uses uint32_t to store the ip address, but its constructor is int32_t. So this patch adds such a constructor. Signed-off-by: Amnon Heiman --- gms/inet_address.hh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gms/inet_address.hh b/gms/inet_address.hh index 809b3665b3..50baac3785 100644 --- a/gms/inet_address.hh +++ b/gms/inet_address.hh @@ -37,6 +37,9 @@ public: inet_address(int32_t ip) : _addr(uint32_t(ip)) { } + explicit inet_address(uint32_t ip) + : _addr(ip) { + } inet_address(net::ipv4_address&& addr) : _addr(std::move(addr)) {} const net::ipv4_address& addr() const { From 0715dcd6bab570751bbfd578b0ef5c76be91918c Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 8 Jan 2016 13:20:38 +0200 Subject: [PATCH 05/10] A schema definition for gossip_digest_ack This is a definition example for gossip_digest_ack with all its sub classes. It can be used by the code generator to create the serializer and deserializer functions. Signed-off-by: Amnon Heiman --- idl/gossip_digest.idl.hh | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 idl/gossip_digest.idl.hh diff --git a/idl/gossip_digest.idl.hh b/idl/gossip_digest.idl.hh new file mode 100644 index 0000000000..de0d052d98 --- /dev/null +++ b/idl/gossip_digest.idl.hh @@ -0,0 +1,83 @@ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ +namespace utils { +class UUID final { + int64_t most_sig_bits; + int64_t least_sig_bits; +}; + +} + +namespace gms { +enum class application_state:int {STATUS = 0, + LOAD, + SCHEMA, + DC, + RACK, + RELEASE_VERSION, + REMOVAL_COORDINATOR, + INTERNAL_IP, + RPC_ADDRESS, + X_11_PADDING, + SEVERITY, + NET_VERSION, + HOST_ID, + TOKENS, + X1, + X2, + X3, + X4, + X5, + X6, + X7, + X8, + X9, + X10}; +class inet_address final { + uint32_t raw_addr(); +}; + +class versioned_value { + sstring value; + int version; +}; + +class heart_beat_state { + int32_t get_generation(); + int32_t get_heart_beat_version(); +}; + +class endpoint_state { + gms::heart_beat_state get_heart_beat_state(); + std::map get_application_state_map(); +} + +class gossip_digest { + gms::inet_address get_endpoint(); + int32_t get_generation(); + int32_t get_max_version(); +} + +class gossip_digest_ack { + std::vector get_gossip_digest_list(); + std::map get_endpoint_state_map(); +} +} From 451cf2692c0f4e9d5cfa4f055a061a422dcd0932 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Tue, 19 Jan 2016 11:38:44 +0200 Subject: [PATCH 06/10] configure.py Add serializer code generation from schema This patch adds rules and the idl schema to configure, which will call the code generation to create the serialization and deserialization functions. There is also a rule to create the header file that include the auto generated header files. Signed-off-by: Amnon Heiman --- configure.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/configure.py b/configure.py index 38adf4af6e..cca848712c 100755 --- a/configure.py +++ b/configure.py @@ -465,8 +465,12 @@ api = ['api/api.cc', 'api/api-doc/system.json', 'api/system.cc' ] +idls = ['idl/gossip_digest.idl.hh', + ] -scylla_tests_dependencies = scylla_core + [ +serialize = idls + ['serializer.inc.hh'] + +scylla_tests_dependencies = scylla_core + api + serialize + [ 'tests/cql_test_env.cc', 'tests/cql_assertions.cc', 'tests/result_set_assertions.cc', @@ -479,7 +483,7 @@ scylla_tests_seastar_deps = [ ] deps = { - 'scylla': ['main.cc'] + scylla_core + api, + 'scylla': serialize + ['main.cc'] + scylla_core + api, } tests_not_using_seastar_test_framework = set([ @@ -656,6 +660,12 @@ with open(buildfile, 'w') as f: rule swagger command = seastar/json/json2code.py -f $in -o $out description = SWAGGER $out + rule serializer + command = ./idl-compiler.py --ns ser -f $in -o $out + description = IDL compiler $out + rule serializer_inc + command = ./idl-compiler.py $in -o $out + description = Combine IDLs $out rule ninja command = {ninja} -C $subdir $target restat = 1 @@ -692,6 +702,8 @@ with open(buildfile, 'w') as f: compiles = {} ragels = {} swaggers = {} + serializers = {} + serializer_inc = {} thrifts = set() antlr3_grammars = set() for binary in build_artifacts: @@ -745,6 +757,12 @@ with open(buildfile, 'w') as f: elif src.endswith('.rl'): hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh') ragels[hh] = src + elif src.endswith('.idl.hh'): + hh = '$builddir/' + mode + '/gen/' + src.replace('.idl.hh', '.dist.hh') + serializers[hh] = src + elif src.endswith('.inc.hh'): + hh = '$builddir/' + mode + '/gen/' + src + serializer_inc[hh] = src elif src.endswith('.json'): hh = '$builddir/' + mode + '/gen/' + src + '.hh' swaggers[hh] = src @@ -763,6 +781,8 @@ with open(buildfile, 'w') as f: for g in antlr3_grammars: gen_headers += g.headers('$builddir/{}/gen'.format(mode)) gen_headers += list(swaggers.keys()) + gen_headers += list(serializers.keys()) + gen_headers += list(serializer_inc.keys()) f.write('build {}: cxx.{} {} || {} \n'.format(obj, mode, src, ' '.join(gen_headers))) if src in extra_cxxflags: f.write(' cxxflags = {seastar_cflags} $cxxflags $cxxflags_{mode} {extra_cxxflags}\n'.format(mode = mode, extra_cxxflags = extra_cxxflags[src], **modeval)) @@ -772,6 +792,11 @@ with open(buildfile, 'w') as f: for hh in swaggers: src = swaggers[hh] f.write('build {}: swagger {}\n'.format(hh,src)) + for hh in serializers: + src = serializers[hh] + f.write('build {}: serializer {}\n'.format(hh,src)) + for hh in serializer_inc: + f.write('build {}: serializer_inc {}\n'.format(hh, " ".join(serializers.keys()))) for thrift in thrifts: outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode))) f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source)) From b6253630722fb92c90c922b7de160fadc0303158 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 8 Jan 2016 13:26:42 +0200 Subject: [PATCH 07/10] Adding the serializer decleration and implementation files. This patch adds the serializer and serializer_imple files. They holds the functions that are not auto generated: primitives and templates (map and vector) It also holds the include to the auto-generated code. Signed-off-by: Amnon Heiman --- serializer.hh | 155 +++++++++++++++++++++++++++++++++++++++++++++ serializer_impl.hh | 126 ++++++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 serializer.hh create mode 100644 serializer_impl.hh diff --git a/serializer.hh b/serializer.hh new file mode 100644 index 0000000000..d6f15be733 --- /dev/null +++ b/serializer.hh @@ -0,0 +1,155 @@ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ +#pragma once + +#include +#include "core/sstring.hh" +#include +namespace rpc { + class simple_output_stream; + class measuring_output_stream; +} +namespace ser { +using size_type = uint32_t; + +template +inline T deserialize_integral(Input& input) { + static_assert(std::is_integral::value, "T should be integral"); + T data; + input.read(reinterpret_cast(&data), sizeof(T)); + return le_to_cpu(data); +} + +template +inline void serialize_integral(Output& output, T data) { + static_assert(std::is_integral::value, "T should be integral"); + data = cpu_to_le(data); + output.write(reinterpret_cast(&data), sizeof(T)); +} + +// For integer type +template +bool deserialize(Input& input, rpc::type) { + return deserialize(input, rpc::type()); +} +template +int8_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint8_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +int16_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint16_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +int32_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint32_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +int64_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} +template +uint64_t deserialize(Input& input, rpc::type) { + return deserialize_integral(input); +} + +template +void serialize(Output& output, bool data) { + serialize(output, uint8_t(data)); +} +template +void serialize(Output& output, int8_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint8_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, int16_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint16_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, int32_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint32_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, int64_t data) { + serialize_integral(output, data); +} +template +void serialize(Output& output, uint64_t data) { + serialize_integral(output, data); +} +template +void safe_serialize_as_uint32(Output& output, uint64_t data); + +// For vectors + +template +inline void serialize(Output& out, const std::vector& v); +template +inline std::vector deserialize(Input& in, rpc::type>); + +template +inline void serialize(Output& out, const std::map& v); +template +inline std::map deserialize(Input& in, rpc::type>); +template +size_type get_sizeof(const T& obj); +// For sstring +template +void serialize(Output& out, const sstring& v); +template +sstring deserialize(Input& in, rpc::type); + +template +void set_size(rpc::simple_output_stream& os, const T& obj); + +template +void set_size(rpc::measuring_output_stream& os, const T& obj); +} + +/* + * Import the auto generated forward decleration code + */ + +#include "serializer.inc.hh" diff --git a/serializer_impl.hh b/serializer_impl.hh new file mode 100644 index 0000000000..46b9b9ab98 --- /dev/null +++ b/serializer_impl.hh @@ -0,0 +1,126 @@ +/* + * Copyright 2016 ScyllaDB + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include "serializer.hh" +#include "serializer.inc.impl.hh" + +namespace ser { + +template +void set_size(rpc::simple_output_stream& os, const T& obj) { + serialize(os, get_sizeof(obj)); +} + +template +void set_size(rpc::measuring_output_stream& os, const T& obj) { + serialize(os, uint32_t(0)); +} + + +template +void safe_serialize_as_uint32(Output& out, uint64_t data) { + if (data > std::numeric_limits::max()) { + throw std::runtime_error("Size is too big for serialization"); + } + serialize(out, uint32_t(data)); +} +template +inline void serialize(Output& out, const std::vector& v) { + safe_serialize_as_uint32(out, v.size()); + for (auto&& e : v) { + serialize(out, e); + } +} +template +inline std::vector deserialize(Input& in, rpc::type>) { + auto sz = deserialize(in, rpc::type()); + std::vector v; + v.reserve(sz); + while (sz--) { + v.push_back(deserialize(in, rpc::type())); + } + return v; +} + +template +inline void serialize(Output& out, const std::map& v) { + safe_serialize_as_uint32(out, v.size()); + for (auto&& e : v) { + serialize(out, e.first); + serialize(out, e.second); + } +} +template +inline std::map deserialize(Input& in, rpc::type>) { + auto sz = deserialize(in, rpc::type()); + std::map m; + while (sz--) { + K k = deserialize(in, rpc::type()); + V v = deserialize(in, rpc::type()); + m[k] = v; + } + return m; +} + +template +inline void serialize(Output& out, const std::experimental::optional& v) { + serialize(out, bool(v)); + if (v) { + serialize(out, v.value()); + } +} +template +inline std::experimental::optional deserialize(Input& in, rpc::type>) { + std::experimental::optional v; + auto b = deserialize(in, rpc::type()); + if (b) { + v = deserialize(in, rpc::type()); + } + return v; +} + +template +void serialize(Output& out, const sstring& v) { + safe_serialize_as_uint32(out, uint32_t(v.size())); + out.write(v.begin(), v.size()); +} +template +sstring deserialize(Input& in, rpc::type) { + auto sz = deserialize(in, rpc::type()); + sstring v(sstring::initialized_later(), sz); + in.read(v.begin(), sz); + return v; +} + +template +size_type get_sizeof(const T& obj) { + rpc::measuring_output_stream ms; + serialize(ms, obj); + auto size = ms.size(); + if (size > std::numeric_limits::max()) { + throw std::runtime_error("Object is too big for get_sizeof"); + } + return size; +} + +} From 577ce0d231a4cb23e580d749fcdf4bf5e44ded1e Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 8 Jan 2016 14:26:51 +0200 Subject: [PATCH 08/10] Adding a sepcific template initialization in messaging_service to use the serializer This patch adds a specific template initialization so that the rpc would use the serializer and deserializer that are auto-generated. Signed-off-by: Amnon Heiman --- message/messaging_service.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/message/messaging_service.cc b/message/messaging_service.cc index 125ec2c875..7ea5146832 100644 --- a/message/messaging_service.cc +++ b/message/messaging_service.cc @@ -36,9 +36,21 @@ #include "dht/i_partitioner.hh" #include "range.hh" #include "frozen_schema.hh" +#include "serializer_impl.hh" namespace net { +template +void write(serializer, Output& out, const gms::gossip_digest_ack& data) { + ser::serialize(out, data); +} + +template +gms::gossip_digest_ack +read(serializer, Input& in, rpc::type type) { + return ser::deserialize(in, type); +} + static logging::logger logger("messaging_service"); using inet_address = gms::inet_address; From f266c2ed4249ffce954a90cf43244174b67a5dd4 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Fri, 22 Jan 2016 20:45:18 +0200 Subject: [PATCH 09/10] README.md: Add dependency for pyparsing python3 python3 needs to install pyparsing excplicitely. This adds the installation of python3-pyparsing to the require dependencies in the README.md Signed-off-by: Amnon Heiman --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c9a0215b68..8d8df373ea 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ git submodule update --recursive * Installing required packages: ``` -sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel numactl-devel hwloc-devel libpciaccess-devel libxml2-devel +sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel numactl-devel hwloc-devel libpciaccess-devel libxml2-devel python3-pyparsing ``` * Build Scylla From 0006f236a6217edf56593f116c90f06a0ab36637 Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Sun, 24 Jan 2016 09:36:02 +0200 Subject: [PATCH 10/10] Add an IDL definition file This adds the IDL definition file. It is also cover in the wiki page: https://github.com/scylladb/scylla/wiki/Serializer-Deserializer-Code-generation Signed-off-by: Amnon Heiman --- IDL.md | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 IDL.md diff --git a/IDL.md b/IDL.md new file mode 100644 index 0000000000..3a3d085cc6 --- /dev/null +++ b/IDL.md @@ -0,0 +1,103 @@ +#IDL definition +The schema we use similar to c++ schema. +Use class or struct similar to the object you need the serializer for. +Use namespace when applicable. + +##keywords +* class/struct - a class or a struct like C++ + class/struct can have final or stub marker +* namespace - has the same C++ meaning +* enum class - has the same C++ meaning +* final modifier for class - when a class mark as final it will not contain a size parameter. Note that final class cannot be extended by future version, so use with care +* stub class - when a class is mark as stub, it means that no code will be generated for this class and it is only there as a documentation. +* version attributes - mark with [[version id ]] mark that a field is available from a specific version +* template - A template class definition like C++ +##Syntax + +###Namespace +``` +namespace ns_name { namespace-body } +``` +* ns_name: either a previously unused identifier, in which case this is original-namespace-definition or the name of a namespace, in which case this is extension-namespace-definition +* namespace-body: possibly empty sequence of declarations of any kind (including class and struct definitions as well as nested namespaces) + +###class/struct +` +class-key class-name final(optional) stub(optional) { member-specification } ;(optional) +` +* class-key: one of class or struct. +* class-name: the name of the class that's being defined. optionally followed by keyword final, optionally followed by keyword stub +* final: when a class mark as final, it means it can not be extended and there is no need to serialize its size, use with care. +* stub: when a class is mark as stub, it means no code will generate for it and it is added for documentation only. +* member-specification: list of access specifiers, and public member accessor see class member below. +* to be compatible with C++ a class definition can be followed by a semicolon. +###enum +`enum-key identifier enum-base { enumerator-list(optional) }` +* enum-key: only enum class is supported +* identifier: the name of the enumeration that's being declared. +* enum-base: colon (:), followed by a type-specifier-seq that names an integral type (see the C++ standard for the full list of all possible integral types). +* enumerator-list: comma-separated list of enumerator definitions, each of which is either simply an identifier, which becomes the name of the enumerator, or an identifier with an initializer: identifier = integral value. +Note that though C++ allows constexpr as an initialize value, it makes the documentation less readable, hence is not permitted. + +###class member +`type member-access attributes(optional) default-value(optional);` +* type: Any valid C++ type, following the C++ notation. note that there should be a serializer for the type, but deceleration order is not mandatory +* member-access: is the way the member can be access. If the member is public it can be the name itself. if not it could be a getter function that should be followed by braces. Note that getter can (and probably should) be const methods. +* attributes: Attributes define by square brackets. Currently are use to mark a version in which a specific member was added [ [ version version-number] ] would mark that the specific member was added in the given version number. + +###template +`template < parameter-list > class-declaration` +* parameter-list - a non-empty comma-separated list of the template parameters. +* class-decleration - (See class section) The class name declared become a template name. + +##IDL example +Forward slashes comments are ignored until the end of the line. +``` +namespace utils { +// An example of a stub class +class UUID stub { + int64_t most_sig_bits; + int64_t least_sig_bits; +} +} + +namespace gms { +//an enum example +enum class application_state:int {STATUS = 0, + LOAD, + SCHEMA, + DC}; + +// example of final class +class versioned_value final { +// getter and setter as public member + int version; + sstring value; +} + +class heart_beat_state { +//getter as function + int32_t get_generation(); +//default value example + int32_t get_heart_beat_version() = 1; +} + +class endpoint_state { + heart_beat_state get_heart_beat_state(); + std::map get_application_state_map(); +} + +class gossip_digest { + inet_address get_endpoint(); + int32_t get_generation(); +//mark that a field was added on a specific version + int32_t get_max_version() [ [version 0.14.2] ]; +} + +class gossip_digest_ack { + std::vector digests(); + std::map get_endpoint_state_map(); +} +} +``` +