mirror of
https://github.com/scylladb/scylladb.git
synced 2026-05-31 12:06:44 +00:00
Merge "Serializer Deserializer code generation" from Amnon
"The series do the following: It adds the code generation Perform the needed changes in the current classes so each would have getter for each of its serializable value and a constructor from the serialized values. It adds a schema definition that cover gossip_diget_ack It changes the messaging_service to use the generated code. An overall explanation of the solution with a description of the schema IDL can be found on the wiki page: https://github.com/scylladb/scylla/wiki/Serializer-Deserializer-Code-generation "
This commit is contained in:
Avi Kivity
103
IDL.md
Normal file
103
IDL.md
Normal file
@@ -0,0 +1,103 @@
|
||||
#IDL definition
|
||||
The schema we use similar to c++ schema.
|
||||
Use class or struct similar to the object you need the serializer for.
|
||||
Use namespace when applicable.
|
||||
|
||||
##keywords
|
||||
* class/struct - a class or a struct like C++
|
||||
class/struct can have final or stub marker
|
||||
* namespace - has the same C++ meaning
|
||||
* enum class - has the same C++ meaning
|
||||
* final modifier for class - when a class mark as final it will not contain a size parameter. Note that final class cannot be extended by future version, so use with care
|
||||
* stub class - when a class is mark as stub, it means that no code will be generated for this class and it is only there as a documentation.
|
||||
* version attributes - mark with [[version id ]] mark that a field is available from a specific version
|
||||
* template - A template class definition like C++
|
||||
##Syntax
|
||||
|
||||
###Namespace
|
||||
```
|
||||
namespace ns_name { namespace-body }
|
||||
```
|
||||
* ns_name: either a previously unused identifier, in which case this is original-namespace-definition or the name of a namespace, in which case this is extension-namespace-definition
|
||||
* namespace-body: possibly empty sequence of declarations of any kind (including class and struct definitions as well as nested namespaces)
|
||||
|
||||
###class/struct
|
||||
`
|
||||
class-key class-name final(optional) stub(optional) { member-specification } ;(optional)
|
||||
`
|
||||
* class-key: one of class or struct.
|
||||
* class-name: the name of the class that's being defined. optionally followed by keyword final, optionally followed by keyword stub
|
||||
* final: when a class mark as final, it means it can not be extended and there is no need to serialize its size, use with care.
|
||||
* stub: when a class is mark as stub, it means no code will generate for it and it is added for documentation only.
|
||||
* member-specification: list of access specifiers, and public member accessor see class member below.
|
||||
* to be compatible with C++ a class definition can be followed by a semicolon.
|
||||
###enum
|
||||
`enum-key identifier enum-base { enumerator-list(optional) }`
|
||||
* enum-key: only enum class is supported
|
||||
* identifier: the name of the enumeration that's being declared.
|
||||
* enum-base: colon (:), followed by a type-specifier-seq that names an integral type (see the C++ standard for the full list of all possible integral types).
|
||||
* enumerator-list: comma-separated list of enumerator definitions, each of which is either simply an identifier, which becomes the name of the enumerator, or an identifier with an initializer: identifier = integral value.
|
||||
Note that though C++ allows constexpr as an initialize value, it makes the documentation less readable, hence is not permitted.
|
||||
|
||||
###class member
|
||||
`type member-access attributes(optional) default-value(optional);`
|
||||
* type: Any valid C++ type, following the C++ notation. note that there should be a serializer for the type, but deceleration order is not mandatory
|
||||
* member-access: is the way the member can be access. If the member is public it can be the name itself. if not it could be a getter function that should be followed by braces. Note that getter can (and probably should) be const methods.
|
||||
* attributes: Attributes define by square brackets. Currently are use to mark a version in which a specific member was added [ [ version version-number] ] would mark that the specific member was added in the given version number.
|
||||
|
||||
###template
|
||||
`template < parameter-list > class-declaration`
|
||||
* parameter-list - a non-empty comma-separated list of the template parameters.
|
||||
* class-decleration - (See class section) The class name declared become a template name.
|
||||
|
||||
##IDL example
|
||||
Forward slashes comments are ignored until the end of the line.
|
||||
```
|
||||
namespace utils {
|
||||
// An example of a stub class
|
||||
class UUID stub {
|
||||
int64_t most_sig_bits;
|
||||
int64_t least_sig_bits;
|
||||
}
|
||||
}
|
||||
|
||||
namespace gms {
|
||||
//an enum example
|
||||
enum class application_state:int {STATUS = 0,
|
||||
LOAD,
|
||||
SCHEMA,
|
||||
DC};
|
||||
|
||||
// example of final class
|
||||
class versioned_value final {
|
||||
// getter and setter as public member
|
||||
int version;
|
||||
sstring value;
|
||||
}
|
||||
|
||||
class heart_beat_state {
|
||||
//getter as function
|
||||
int32_t get_generation();
|
||||
//default value example
|
||||
int32_t get_heart_beat_version() = 1;
|
||||
}
|
||||
|
||||
class endpoint_state {
|
||||
heart_beat_state get_heart_beat_state();
|
||||
std::map<application_state, versioned_value> get_application_state_map();
|
||||
}
|
||||
|
||||
class gossip_digest {
|
||||
inet_address get_endpoint();
|
||||
int32_t get_generation();
|
||||
//mark that a field was added on a specific version
|
||||
int32_t get_max_version() [ [version 0.14.2] ];
|
||||
}
|
||||
|
||||
class gossip_digest_ack {
|
||||
std::vector<gossip_digest> digests();
|
||||
std::map<inet_address, gms::endpoint_state> get_endpoint_state_map();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -15,7 +15,7 @@ git submodule update --recursive
|
||||
* Installing required packages:
|
||||
|
||||
```
|
||||
sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel numactl-devel hwloc-devel libpciaccess-devel libxml2-devel
|
||||
sudo yum install yaml-cpp-devel lz4-devel zlib-devel snappy-devel jsoncpp-devel thrift-devel antlr3-tool antlr3-C++-devel libasan libubsan gcc-c++ gnutls-devel ninja-build ragel libaio-devel cryptopp-devel xfsprogs-devel numactl-devel hwloc-devel libpciaccess-devel libxml2-devel python3-pyparsing
|
||||
```
|
||||
|
||||
* Build Scylla
|
||||
|
||||
29
configure.py
29
configure.py
@@ -465,8 +465,12 @@ api = ['api/api.cc',
|
||||
'api/api-doc/system.json',
|
||||
'api/system.cc'
|
||||
]
|
||||
idls = ['idl/gossip_digest.idl.hh',
|
||||
]
|
||||
|
||||
scylla_tests_dependencies = scylla_core + [
|
||||
serialize = idls + ['serializer.inc.hh']
|
||||
|
||||
scylla_tests_dependencies = scylla_core + api + serialize + [
|
||||
'tests/cql_test_env.cc',
|
||||
'tests/cql_assertions.cc',
|
||||
'tests/result_set_assertions.cc',
|
||||
@@ -479,7 +483,7 @@ scylla_tests_seastar_deps = [
|
||||
]
|
||||
|
||||
deps = {
|
||||
'scylla': ['main.cc'] + scylla_core + api,
|
||||
'scylla': serialize + ['main.cc'] + scylla_core + api,
|
||||
}
|
||||
|
||||
tests_not_using_seastar_test_framework = set([
|
||||
@@ -656,6 +660,12 @@ with open(buildfile, 'w') as f:
|
||||
rule swagger
|
||||
command = seastar/json/json2code.py -f $in -o $out
|
||||
description = SWAGGER $out
|
||||
rule serializer
|
||||
command = ./idl-compiler.py --ns ser -f $in -o $out
|
||||
description = IDL compiler $out
|
||||
rule serializer_inc
|
||||
command = ./idl-compiler.py $in -o $out
|
||||
description = Combine IDLs $out
|
||||
rule ninja
|
||||
command = {ninja} -C $subdir $target
|
||||
restat = 1
|
||||
@@ -692,6 +702,8 @@ with open(buildfile, 'w') as f:
|
||||
compiles = {}
|
||||
ragels = {}
|
||||
swaggers = {}
|
||||
serializers = {}
|
||||
serializer_inc = {}
|
||||
thrifts = set()
|
||||
antlr3_grammars = set()
|
||||
for binary in build_artifacts:
|
||||
@@ -745,6 +757,12 @@ with open(buildfile, 'w') as f:
|
||||
elif src.endswith('.rl'):
|
||||
hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh')
|
||||
ragels[hh] = src
|
||||
elif src.endswith('.idl.hh'):
|
||||
hh = '$builddir/' + mode + '/gen/' + src.replace('.idl.hh', '.dist.hh')
|
||||
serializers[hh] = src
|
||||
elif src.endswith('.inc.hh'):
|
||||
hh = '$builddir/' + mode + '/gen/' + src
|
||||
serializer_inc[hh] = src
|
||||
elif src.endswith('.json'):
|
||||
hh = '$builddir/' + mode + '/gen/' + src + '.hh'
|
||||
swaggers[hh] = src
|
||||
@@ -763,6 +781,8 @@ with open(buildfile, 'w') as f:
|
||||
for g in antlr3_grammars:
|
||||
gen_headers += g.headers('$builddir/{}/gen'.format(mode))
|
||||
gen_headers += list(swaggers.keys())
|
||||
gen_headers += list(serializers.keys())
|
||||
gen_headers += list(serializer_inc.keys())
|
||||
f.write('build {}: cxx.{} {} || {} \n'.format(obj, mode, src, ' '.join(gen_headers)))
|
||||
if src in extra_cxxflags:
|
||||
f.write(' cxxflags = {seastar_cflags} $cxxflags $cxxflags_{mode} {extra_cxxflags}\n'.format(mode = mode, extra_cxxflags = extra_cxxflags[src], **modeval))
|
||||
@@ -772,6 +792,11 @@ with open(buildfile, 'w') as f:
|
||||
for hh in swaggers:
|
||||
src = swaggers[hh]
|
||||
f.write('build {}: swagger {}\n'.format(hh,src))
|
||||
for hh in serializers:
|
||||
src = serializers[hh]
|
||||
f.write('build {}: serializer {}\n'.format(hh,src))
|
||||
for hh in serializer_inc:
|
||||
f.write('build {}: serializer_inc {}\n'.format(hh, " ".join(serializers.keys())))
|
||||
for thrift in thrifts:
|
||||
outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode)))
|
||||
f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source))
|
||||
|
||||
@@ -81,6 +81,14 @@ public:
|
||||
, _is_alive(true) {
|
||||
}
|
||||
|
||||
endpoint_state(heart_beat_state&& initial_hb_state,
|
||||
const std::map<application_state, versioned_value>& application_state)
|
||||
: _heart_beat_state(std::move(initial_hb_state))
|
||||
,_application_state(application_state)
|
||||
, _update_timestamp(clk::now())
|
||||
, _is_alive(true) {
|
||||
}
|
||||
|
||||
heart_beat_state& get_heart_beat_state() {
|
||||
return _heart_beat_state;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,9 @@ public:
|
||||
inet_address(int32_t ip)
|
||||
: _addr(uint32_t(ip)) {
|
||||
}
|
||||
explicit inet_address(uint32_t ip)
|
||||
: _addr(ip) {
|
||||
}
|
||||
inet_address(net::ipv4_address&& addr) : _addr(std::move(addr)) {}
|
||||
|
||||
const net::ipv4_address& addr() const {
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include "gms/versioned_value.hh"
|
||||
#include "message/messaging_service.hh"
|
||||
|
||||
namespace gms {
|
||||
|
||||
@@ -67,4 +68,8 @@ size_t versioned_value::serialized_size() const {
|
||||
return serialize_string_size(value) + serialize_int32_size;
|
||||
}
|
||||
|
||||
versioned_value versioned_value::factory::network_version() {
|
||||
return versioned_value(sprint("%s",net::messaging_service::current_version));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -46,7 +46,6 @@
|
||||
#include "gms/inet_address.hh"
|
||||
#include "dht/i_partitioner.hh"
|
||||
#include "to_string.hh"
|
||||
#include "message/messaging_service.hh"
|
||||
#include "version.hh"
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
@@ -96,7 +95,7 @@ public:
|
||||
value == other.value;
|
||||
}
|
||||
|
||||
private:
|
||||
public:
|
||||
versioned_value(const sstring& value, int version = version_generator::get_next_version())
|
||||
: version(version), value(value) {
|
||||
#if 0
|
||||
@@ -112,8 +111,10 @@ private:
|
||||
: version(version), value(std::move(value)) {
|
||||
}
|
||||
|
||||
versioned_value()
|
||||
: version(-1) {
|
||||
}
|
||||
|
||||
public:
|
||||
int compare_to(const versioned_value &value) {
|
||||
return version - value.version;
|
||||
}
|
||||
@@ -228,9 +229,7 @@ public:
|
||||
return versioned_value(version::release());
|
||||
}
|
||||
|
||||
versioned_value network_version() {
|
||||
return versioned_value(sprint("%s",net::messaging_service::current_version));
|
||||
}
|
||||
versioned_value network_version();
|
||||
|
||||
versioned_value internal_ip(const sstring &private_ip) {
|
||||
return versioned_value(private_ip);
|
||||
|
||||
329
idl-compiler.py
Executable file
329
idl-compiler.py
Executable file
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/python3
|
||||
#
|
||||
# Copyright 2016 ScyllaDB
|
||||
#
|
||||
|
||||
#
|
||||
# This file is part of Scylla.
|
||||
#
|
||||
# Scylla is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Scylla is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import json
|
||||
import sys
|
||||
import re
|
||||
import glob
|
||||
import argparse
|
||||
import os
|
||||
from string import Template
|
||||
import pyparsing as pp
|
||||
|
||||
|
||||
EXTENSION = '.idl.hh'
|
||||
READ_BUFF = 'input_buffer'
|
||||
WRITE_BUFF = 'output_buffer'
|
||||
SERIALIZER = 'serialize'
|
||||
DESERIALIZER = 'deserialize'
|
||||
SETSIZE = 'set_size'
|
||||
SIZETYPE = 'size_type'
|
||||
|
||||
parser = argparse.ArgumentParser(description="""Generate serializer helper function""")
|
||||
|
||||
parser.add_argument('-o', help='Output file', default='')
|
||||
parser.add_argument('-f', help='input file', default='')
|
||||
parser.add_argument('--ns', help="""namespace, when set function will be created
|
||||
under the given namespace""", default='')
|
||||
parser.add_argument('file', nargs='*', help="combine one or more file names for the genral include files")
|
||||
|
||||
config = parser.parse_args()
|
||||
|
||||
|
||||
|
||||
def fprint(f, *args):
|
||||
for arg in args:
|
||||
f.write(arg)
|
||||
|
||||
def fprintln(f, *args):
|
||||
for arg in args:
|
||||
f.write(arg)
|
||||
f.write('\n')
|
||||
|
||||
def print_cw(f):
|
||||
fprintln(f, """
|
||||
/*
|
||||
* Copyright 2016 ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is part of Scylla.
|
||||
*
|
||||
* Scylla is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Scylla is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
|
||||
/*
|
||||
* This is an auto-generated code, do not modify directly.
|
||||
*/
|
||||
#pragma once
|
||||
""")
|
||||
|
||||
def parse_file(file_name):
|
||||
first = pp.Word(pp.alphas + "_", exact=1)
|
||||
rest = pp.Word(pp.alphanums + "_")
|
||||
|
||||
number = pp.Word(pp.nums)
|
||||
identifier = pp.Combine(first + pp.Optional(rest))
|
||||
|
||||
lbrace = pp.Literal('{').suppress()
|
||||
rbrace = pp.Literal('}').suppress()
|
||||
cls = pp.Literal('class')
|
||||
colon = pp.Literal(":")
|
||||
semi = pp.Literal(";").suppress()
|
||||
langle = pp.Literal("<")
|
||||
rangle = pp.Literal(">")
|
||||
equals = pp.Literal("=")
|
||||
comma = pp.Literal(",")
|
||||
lparen = pp.Literal("(")
|
||||
rparen = pp.Literal(")")
|
||||
lbrack = pp.Literal("[")
|
||||
rbrack = pp.Literal("]")
|
||||
mins = pp.Literal("-")
|
||||
struct = pp.Literal('struct')
|
||||
template = pp.Literal('template')
|
||||
final = pp.Literal('final').setResultsName("final")
|
||||
stub = pp.Literal('stub').setResultsName("stub")
|
||||
with_colon = pp.Word(pp.alphanums + "_" + ":")
|
||||
btype = with_colon
|
||||
type = pp.Forward()
|
||||
nestedParens = pp.nestedExpr('<', '>')
|
||||
|
||||
tmpl = pp.Group(btype + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress())
|
||||
type << (tmpl | btype)
|
||||
enum_lit = pp.Literal('enum')
|
||||
enum_class = pp.Group(enum_lit + cls)
|
||||
ns = pp.Literal("namespace")
|
||||
|
||||
enum_init = equals.suppress() + pp.Optional(mins) + number
|
||||
enum_value = pp.Group(identifier + pp.Optional(enum_init))
|
||||
enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace)
|
||||
content = pp.Forward()
|
||||
|
||||
member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen)))
|
||||
attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress())
|
||||
namespace = pp.Group(ns.setResultsName("type") + identifier.setResultsName("name") + lbrace + pp.Group(pp.OneOrMore(content)).setResultsName("content") + rbrace)
|
||||
enum = pp.Group(enum_class.setResultsName("type") + identifier.setResultsName("name") + colon.suppress() + identifier.setResultsName("underline_type") + enum_values.setResultsName("enum_values") + pp.Optional(semi).suppress())
|
||||
default_value = equals.suppress() + pp.SkipTo(';')
|
||||
class_member = pp.Group(type.setResultsName("type") + member_name.setResultsName("name") + pp.Optional(attrib).setResultsName("attribute") + pp.Optional(default_value).setResultsName("default") + semi.suppress()).setResultsName("member")
|
||||
template_param = pp.Group(identifier.setResultsName("type") + identifier.setResultsName("name"))
|
||||
template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param)).setResultsName("params") + rangle)
|
||||
class_content = pp.Forward()
|
||||
class_def = pp.Group(pp.Optional(template_def).setResultsName("template") + (cls | struct).setResultsName("type") + with_colon.setResultsName("name") + pp.Optional(final) + pp.Optional(stub) + lbrace + pp.Group(pp.OneOrMore(class_content)).setResultsName("members") + rbrace + pp.Optional(semi))
|
||||
content << (enum | class_def | namespace)
|
||||
class_content << (enum | class_def | class_member)
|
||||
rt = pp.OneOrMore(content)
|
||||
singleLineComment = "//" + pp.restOfLine
|
||||
rt.ignore(singleLineComment)
|
||||
rt.ignore(pp.cStyleComment)
|
||||
return rt.parseFile(file_name, parseAll=True)
|
||||
|
||||
|
||||
def combine_ns(namespaces):
|
||||
return "::".join(namespaces)
|
||||
|
||||
def open_namespaces(namespaces):
|
||||
return "".join(map(lambda a: "namespace " + a + " { ", namespaces))
|
||||
|
||||
def close_namespaces(namespaces):
|
||||
return "".join(map(lambda a: "}", namespaces))
|
||||
|
||||
def set_namespace(namespaces):
|
||||
ns = combine_ns(namespaces)
|
||||
ns_open = open_namespaces(namespaces)
|
||||
ns_close = close_namespaces(namespaces)
|
||||
return [ns, ns_open, ns_close]
|
||||
|
||||
def declare_class(hout, name, ns_open, ns_close):
|
||||
clas_def = ns_open + name + ";" + ns_close
|
||||
fprintln(hout, "\n", clas_def)
|
||||
|
||||
def declear_methods(hout, name, template_param = ""):
|
||||
if config.ns != '':
|
||||
fprintln(hout, "namespace ", config.ns, " {")
|
||||
fprintln(hout, Template("""
|
||||
template <typename Output$tmp_param>
|
||||
void $ser_func(Output& buf, const $name& v);
|
||||
|
||||
template <typename Input$tmp_param>
|
||||
$name $deser_func(Input& buf, rpc::type<$name>);""").substitute({'ser_func': SERIALIZER, 'deser_func' : DESERIALIZER, 'name' : name, 'sizetype' : SIZETYPE, 'tmp_param' : template_param }))
|
||||
if config.ns != '':
|
||||
fprintln(hout, "}")
|
||||
|
||||
def handle_enum(enum, hout, cout, namespaces , parent_template_param = []):
|
||||
[ns, ns_open, ns_close] = set_namespace(namespaces)
|
||||
temp_def = ',' + ", ".join(map(lambda a: a[0] + " " + a[1], parent_template_param)) if parent_template_param else ""
|
||||
name = enum["name"] if ns == "" else ns + "::" + enum["name"]
|
||||
declear_methods(hout, name, temp_def)
|
||||
fprintln(cout, Template("""
|
||||
template<typename Output$temp_def>
|
||||
void $ser_func(Output& buf, const $name& v) {
|
||||
serialize(buf, static_cast<$type>(v));
|
||||
}
|
||||
|
||||
template<typename Input$temp_def>
|
||||
$name $deser_func(Input& buf, rpc::type<$name>) {
|
||||
return static_cast<$name>(deserialize(buf, rpc::type<$type>()));
|
||||
}""").substitute({'ser_func': SERIALIZER, 'deser_func' : DESERIALIZER, 'name' : name, 'size_type' : SIZETYPE, 'type': enum['underline_type'], 'temp_def' : temp_def}))
|
||||
|
||||
|
||||
def join_template(lst):
|
||||
return "<" + ", ".join([param_type(l) for l in lst]) + ">"
|
||||
|
||||
def param_type(lst):
|
||||
if isinstance(lst, str):
|
||||
return lst
|
||||
if len(lst) == 1:
|
||||
return lst[0]
|
||||
return lst[0] + join_template(lst[1])
|
||||
|
||||
def is_class(obj):
|
||||
return obj["type"] == "class" or obj["type"] == "struct"
|
||||
|
||||
def is_enum(obj):
|
||||
try:
|
||||
return not isinstance(obj["type"], str) and "".join(obj["type"]) == 'enumclass'
|
||||
except:
|
||||
return False
|
||||
|
||||
def handle_class(cls, hout, cout, namespaces=[], parent_template_param = []):
|
||||
if "stub" in cls:
|
||||
return
|
||||
[ns, ns_open, ns_close] = set_namespace(namespaces)
|
||||
tpl = "template" in cls
|
||||
template_param_list = (cls["template"][0]["params"].asList() if tpl else [])
|
||||
template_param = ", ".join(map(lambda a: a[0] + " " + a[1], template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else ""
|
||||
template = "template <"+ template_param +">\n" if tpl else ""
|
||||
template_class_param = "<" + ",".join(map(lambda a: a[1], template_param_list)) + ">" if tpl else ""
|
||||
temp_def = ',' + template_param if template_param != "" else ""
|
||||
|
||||
if ns == "":
|
||||
name = cls["name"]
|
||||
else:
|
||||
name = ns + "::" + cls["name"]
|
||||
full_name = name + template_class_param
|
||||
for param in cls["members"]:
|
||||
if is_class(param):
|
||||
handle_class(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list)
|
||||
elif is_enum(param):
|
||||
handle_enum(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list)
|
||||
declear_methods(hout, name + template_class_param, temp_def)
|
||||
modifier = "final" in cls
|
||||
|
||||
fprintln(cout, Template("""
|
||||
template<typename Output$temp_def>
|
||||
void $func(Output& buf, const $name& obj) {""").substitute({'func' : SERIALIZER, 'name' : full_name, 'temp_def': temp_def}))
|
||||
if not modifier:
|
||||
fprintln(cout, Template(""" $set_size(buf, obj);""").substitute({'func' : SERIALIZER, 'set_size' : SETSIZE, 'name' : name, 'sizetype' : SIZETYPE}))
|
||||
for param in cls["members"]:
|
||||
if is_class(param) or is_enum(param):
|
||||
continue
|
||||
fprintln(cout, Template(""" $func(buf, obj.$var);""").substitute({'func' : SERIALIZER, 'var' : param["name"]}))
|
||||
fprintln(cout, "}")
|
||||
|
||||
fprintln(cout, Template("""
|
||||
template<typename Input$temp_def>
|
||||
$name$temp_param $func(Input& buf, rpc::type<$name$temp_param>) {""").substitute({'func' : DESERIALIZER, 'name' : name, 'temp_def': temp_def, 'temp_param' : template_class_param}))
|
||||
if not modifier:
|
||||
fprintln(cout, Template(""" $size_type size = $func(buf, rpc::type<$size_type>());
|
||||
Input in = buf.read_substream(size - sizeof($size_type));""").substitute({'func' : DESERIALIZER, 'size_type' : SIZETYPE}))
|
||||
else:
|
||||
fprintln(cout, """ Input& in = buf;""")
|
||||
params = []
|
||||
for index, param in enumerate(cls["members"]):
|
||||
if is_class(param) or is_enum(param):
|
||||
continue
|
||||
local_param = "__local_" + str(index)
|
||||
if "attribute" in param:
|
||||
deflt = param["default"] if param["default"] else param["type"] + "()"
|
||||
fprintln(cout, Template(""" $typ $local = (in.size()>0) ?
|
||||
$func(in, rpc::type<$typ>()) : $default;""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param, 'default': deflt}))
|
||||
else:
|
||||
fprintln(cout, Template(""" $typ $local = $func(in, rpc::type<$typ>());""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param}))
|
||||
params.append("std::move(" + local_param + ")")
|
||||
fprintln(cout, Template("""
|
||||
$name$temp_param res {$params};
|
||||
return res;
|
||||
}""").substitute({'name' : name, 'params': ", ".join(params), 'temp_param' : template_class_param}))
|
||||
|
||||
|
||||
def handle_objects(tree, hout, cout, namespaces=[]):
|
||||
for obj in tree:
|
||||
if is_class(obj):
|
||||
handle_class(obj, hout, cout, namespaces)
|
||||
elif is_enum(obj):
|
||||
handle_enum(obj, hout, cout, namespaces)
|
||||
elif obj["type"] == "namespace":
|
||||
handle_objects(obj["content"], hout, cout, namespaces + [obj["name"]])
|
||||
else:
|
||||
print("unknown type ", obj, obj["type"])
|
||||
|
||||
def load_file(name):
|
||||
if config.o:
|
||||
cout = open(config.o.replace('.hh', '.impl.hh'), "w+")
|
||||
hout = open(config.o, "w+")
|
||||
else:
|
||||
cout = open(name.replace(EXTENSION, '.dist.impl.hh'), "w+")
|
||||
hout = open(name.replace(EXTENSION, '.dist.hh'), "w+")
|
||||
print_cw(hout)
|
||||
fprintln(hout, """
|
||||
/*
|
||||
* The generate code should be included in a header file after
|
||||
* The object definition
|
||||
*/
|
||||
""")
|
||||
print_cw(cout)
|
||||
if config.ns != '':
|
||||
fprintln(cout, "namespace ", config.ns, " {")
|
||||
data = parse_file(name)
|
||||
if data:
|
||||
handle_objects(data, hout, cout)
|
||||
if config.ns != '':
|
||||
fprintln(cout, "}")
|
||||
cout.close()
|
||||
hout.close()
|
||||
|
||||
def general_include(files):
|
||||
name = config.o if config.o else "serializer.dist.hh"
|
||||
cout = open(name.replace('.hh', '.impl.hh'), "w+")
|
||||
hout = open(name, "w+")
|
||||
print_cw(cout)
|
||||
print_cw(hout)
|
||||
for n in files:
|
||||
fprintln(hout, '#include "' + n +'"')
|
||||
fprintln(cout, '#include "' + n.replace(".dist.hh", '.dist.impl.hh') +'"')
|
||||
cout.close()
|
||||
hout.close()
|
||||
if config.file:
|
||||
general_include(config.file)
|
||||
elif config.f != '':
|
||||
load_file(config.f)
|
||||
83
idl/gossip_digest.idl.hh
Normal file
83
idl/gossip_digest.idl.hh
Normal file
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2016 ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is part of Scylla.
|
||||
*
|
||||
* Scylla is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Scylla is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
namespace utils {
|
||||
class UUID final {
|
||||
int64_t most_sig_bits;
|
||||
int64_t least_sig_bits;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
namespace gms {
|
||||
enum class application_state:int {STATUS = 0,
|
||||
LOAD,
|
||||
SCHEMA,
|
||||
DC,
|
||||
RACK,
|
||||
RELEASE_VERSION,
|
||||
REMOVAL_COORDINATOR,
|
||||
INTERNAL_IP,
|
||||
RPC_ADDRESS,
|
||||
X_11_PADDING,
|
||||
SEVERITY,
|
||||
NET_VERSION,
|
||||
HOST_ID,
|
||||
TOKENS,
|
||||
X1,
|
||||
X2,
|
||||
X3,
|
||||
X4,
|
||||
X5,
|
||||
X6,
|
||||
X7,
|
||||
X8,
|
||||
X9,
|
||||
X10};
|
||||
class inet_address final {
|
||||
uint32_t raw_addr();
|
||||
};
|
||||
|
||||
class versioned_value {
|
||||
sstring value;
|
||||
int version;
|
||||
};
|
||||
|
||||
class heart_beat_state {
|
||||
int32_t get_generation();
|
||||
int32_t get_heart_beat_version();
|
||||
};
|
||||
|
||||
class endpoint_state {
|
||||
gms::heart_beat_state get_heart_beat_state();
|
||||
std::map<gms::application_state, gms::versioned_value> get_application_state_map();
|
||||
}
|
||||
|
||||
class gossip_digest {
|
||||
gms::inet_address get_endpoint();
|
||||
int32_t get_generation();
|
||||
int32_t get_max_version();
|
||||
}
|
||||
|
||||
class gossip_digest_ack {
|
||||
std::vector<gms::gossip_digest> get_gossip_digest_list();
|
||||
std::map<gms::inet_address, gms::endpoint_state> get_endpoint_state_map();
|
||||
}
|
||||
}
|
||||
@@ -36,9 +36,21 @@
|
||||
#include "dht/i_partitioner.hh"
|
||||
#include "range.hh"
|
||||
#include "frozen_schema.hh"
|
||||
#include "serializer_impl.hh"
|
||||
|
||||
namespace net {
|
||||
|
||||
template<typename Output>
|
||||
void write(serializer, Output& out, const gms::gossip_digest_ack& data) {
|
||||
ser::serialize(out, data);
|
||||
}
|
||||
|
||||
template <typename Input>
|
||||
gms::gossip_digest_ack
|
||||
read(serializer, Input& in, rpc::type<gms::gossip_digest_ack> type) {
|
||||
return ser::deserialize(in, type);
|
||||
}
|
||||
|
||||
static logging::logger logger("messaging_service");
|
||||
|
||||
using inet_address = gms::inet_address;
|
||||
|
||||
155
serializer.hh
Normal file
155
serializer.hh
Normal file
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* Copyright 2016 ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is part of Scylla.
|
||||
*
|
||||
* Scylla is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Scylla is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include "core/sstring.hh"
|
||||
#include <unordered_map>
|
||||
namespace rpc {
|
||||
class simple_output_stream;
|
||||
class measuring_output_stream;
|
||||
}
|
||||
namespace ser {
|
||||
using size_type = uint32_t;
|
||||
|
||||
template<typename T, typename Input>
|
||||
inline T deserialize_integral(Input& input) {
|
||||
static_assert(std::is_integral<T>::value, "T should be integral");
|
||||
T data;
|
||||
input.read(reinterpret_cast<char*>(&data), sizeof(T));
|
||||
return le_to_cpu(data);
|
||||
}
|
||||
|
||||
template<typename T, typename Output>
|
||||
inline void serialize_integral(Output& output, T data) {
|
||||
static_assert(std::is_integral<T>::value, "T should be integral");
|
||||
data = cpu_to_le(data);
|
||||
output.write(reinterpret_cast<const char*>(&data), sizeof(T));
|
||||
}
|
||||
|
||||
// For integer type
|
||||
template<typename Input>
|
||||
bool deserialize(Input& input, rpc::type<bool>) {
|
||||
return deserialize(input, rpc::type<uint8_t>());
|
||||
}
|
||||
template<typename Input>
|
||||
int8_t deserialize(Input& input, rpc::type<int8_t>) {
|
||||
return deserialize_integral<int8_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
uint8_t deserialize(Input& input, rpc::type<uint8_t>) {
|
||||
return deserialize_integral<uint8_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
int16_t deserialize(Input& input, rpc::type<int16_t>) {
|
||||
return deserialize_integral<int16_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
uint16_t deserialize(Input& input, rpc::type<uint16_t>) {
|
||||
return deserialize_integral<uint16_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
int32_t deserialize(Input& input, rpc::type<int32_t>) {
|
||||
return deserialize_integral<int32_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
uint32_t deserialize(Input& input, rpc::type<uint32_t>) {
|
||||
return deserialize_integral<uint32_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
int64_t deserialize(Input& input, rpc::type<int64_t>) {
|
||||
return deserialize_integral<int64_t>(input);
|
||||
}
|
||||
template<typename Input>
|
||||
uint64_t deserialize(Input& input, rpc::type<uint64_t>) {
|
||||
return deserialize_integral<uint64_t>(input);
|
||||
}
|
||||
|
||||
template<typename Output>
|
||||
void serialize(Output& output, bool data) {
|
||||
serialize(output, uint8_t(data));
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, int8_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, uint8_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, int16_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, uint16_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, int32_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, uint32_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, int64_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void serialize(Output& output, uint64_t data) {
|
||||
serialize_integral(output, data);
|
||||
}
|
||||
template<typename Output>
|
||||
void safe_serialize_as_uint32(Output& output, uint64_t data);
|
||||
|
||||
// For vectors
|
||||
|
||||
template<typename T, typename Output>
|
||||
inline void serialize(Output& out, const std::vector<T>& v);
|
||||
template<typename T, typename Input>
|
||||
inline std::vector<T> deserialize(Input& in, rpc::type<std::vector<T>>);
|
||||
|
||||
template<typename K, typename V, typename Output>
|
||||
inline void serialize(Output& out, const std::map<K, V>& v);
|
||||
template<typename K, typename V, typename Input>
|
||||
inline std::map<K, V> deserialize(Input& in, rpc::type<std::map<K, V>>);
|
||||
template<typename T>
|
||||
size_type get_sizeof(const T& obj);
|
||||
// For sstring
|
||||
template<typename Output>
|
||||
void serialize(Output& out, const sstring& v);
|
||||
template<typename Input>
|
||||
sstring deserialize(Input& in, rpc::type<sstring>);
|
||||
|
||||
template<typename T>
|
||||
void set_size(rpc::simple_output_stream& os, const T& obj);
|
||||
|
||||
template<typename T>
|
||||
void set_size(rpc::measuring_output_stream& os, const T& obj);
|
||||
}
|
||||
|
||||
/*
|
||||
* Import the auto generated forward decleration code
|
||||
*/
|
||||
|
||||
#include "serializer.inc.hh"
|
||||
126
serializer_impl.hh
Normal file
126
serializer_impl.hh
Normal file
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Copyright 2016 ScyllaDB
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file is part of Scylla.
|
||||
*
|
||||
* Scylla is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* Scylla is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "serializer.hh"
|
||||
#include "serializer.inc.impl.hh"
|
||||
|
||||
namespace ser {
|
||||
|
||||
template<typename T>
|
||||
void set_size(rpc::simple_output_stream& os, const T& obj) {
|
||||
serialize(os, get_sizeof(obj));
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void set_size(rpc::measuring_output_stream& os, const T& obj) {
|
||||
serialize(os, uint32_t(0));
|
||||
}
|
||||
|
||||
|
||||
template<typename Output>
|
||||
void safe_serialize_as_uint32(Output& out, uint64_t data) {
|
||||
if (data > std::numeric_limits<uint32_t>::max()) {
|
||||
throw std::runtime_error("Size is too big for serialization");
|
||||
}
|
||||
serialize(out, uint32_t(data));
|
||||
}
|
||||
template<typename T, typename Output>
|
||||
inline void serialize(Output& out, const std::vector<T>& v) {
|
||||
safe_serialize_as_uint32(out, v.size());
|
||||
for (auto&& e : v) {
|
||||
serialize(out, e);
|
||||
}
|
||||
}
|
||||
template<typename T, typename Input>
|
||||
inline std::vector<T> deserialize(Input& in, rpc::type<std::vector<T>>) {
|
||||
auto sz = deserialize(in, rpc::type<uint32_t>());
|
||||
std::vector<T> v;
|
||||
v.reserve(sz);
|
||||
while (sz--) {
|
||||
v.push_back(deserialize(in, rpc::type<T>()));
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
template<typename K, typename V, typename Output>
|
||||
inline void serialize(Output& out, const std::map<K, V>& v) {
|
||||
safe_serialize_as_uint32(out, v.size());
|
||||
for (auto&& e : v) {
|
||||
serialize(out, e.first);
|
||||
serialize(out, e.second);
|
||||
}
|
||||
}
|
||||
template<typename K, typename V, typename Input>
|
||||
inline std::map<K, V> deserialize(Input& in, rpc::type<std::map<K, V>>) {
|
||||
auto sz = deserialize(in, rpc::type<uint32_t>());
|
||||
std::map<K, V> m;
|
||||
while (sz--) {
|
||||
K k = deserialize(in, rpc::type<K>());
|
||||
V v = deserialize(in, rpc::type<V>());
|
||||
m[k] = v;
|
||||
}
|
||||
return m;
|
||||
}
|
||||
|
||||
template<typename T, typename Output>
|
||||
inline void serialize(Output& out, const std::experimental::optional<T>& v) {
|
||||
serialize(out, bool(v));
|
||||
if (v) {
|
||||
serialize(out, v.value());
|
||||
}
|
||||
}
|
||||
template<typename T, typename Input>
|
||||
inline std::experimental::optional<T> deserialize(Input& in, rpc::type<std::experimental::optional<T>>) {
|
||||
std::experimental::optional<T> v;
|
||||
auto b = deserialize(in, rpc::type<bool>());
|
||||
if (b) {
|
||||
v = deserialize(in, rpc::type<T>());
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
template<typename Output>
|
||||
void serialize(Output& out, const sstring& v) {
|
||||
safe_serialize_as_uint32(out, uint32_t(v.size()));
|
||||
out.write(v.begin(), v.size());
|
||||
}
|
||||
template<typename Input>
|
||||
sstring deserialize(Input& in, rpc::type<sstring>) {
|
||||
auto sz = deserialize(in, rpc::type<uint32_t>());
|
||||
sstring v(sstring::initialized_later(), sz);
|
||||
in.read(v.begin(), sz);
|
||||
return v;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
size_type get_sizeof(const T& obj) {
|
||||
rpc::measuring_output_stream ms;
|
||||
serialize(ms, obj);
|
||||
auto size = ms.size();
|
||||
if (size > std::numeric_limits<size_type>::max()) {
|
||||
throw std::runtime_error("Object is too big for get_sizeof");
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user