4df4994b71
When using member name in an idetifer of generated class or method idl compiler should strip the trailing '()'.
997 lines
37 KiB
Python
Executable File
997 lines
37 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# Copyright 2016 ScyllaDB
|
|
#
|
|
|
|
#
|
|
# This file is part of Scylla.
|
|
#
|
|
# Scylla is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Scylla is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
import json
|
|
import sys
|
|
import re
|
|
import glob
|
|
import argparse
|
|
import os
|
|
from string import Template
|
|
import pyparsing as pp
|
|
from functools import reduce
|
|
import textwrap
|
|
from numbers import Number
|
|
from timeit import reindent
|
|
|
|
EXTENSION = '.idl.hh'
|
|
READ_BUFF = 'input_buffer'
|
|
WRITE_BUFF = 'output_buffer'
|
|
SERIALIZER = 'serialize'
|
|
DESERIALIZER = 'deserialize'
|
|
SETSIZE = 'set_size'
|
|
SIZETYPE = 'size_type'
|
|
|
|
parser = argparse.ArgumentParser(description="""Generate serializer helper function""")
|
|
|
|
parser.add_argument('-o', help='Output file', default='')
|
|
parser.add_argument('-f', help='input file', default='')
|
|
parser.add_argument('--ns', help="""namespace, when set function will be created
|
|
under the given namespace""", default='')
|
|
parser.add_argument('file', nargs='*', help="combine one or more file names for the genral include files")
|
|
|
|
config = parser.parse_args()
|
|
|
|
|
|
def reindent(indent, text):
|
|
return textwrap.indent(textwrap.dedent(text), ' ' * indent)
|
|
|
|
def fprint(f, *args):
|
|
for arg in args:
|
|
f.write(arg)
|
|
|
|
def fprintln(f, *args):
|
|
for arg in args:
|
|
f.write(arg)
|
|
f.write('\n')
|
|
|
|
def print_cw(f):
|
|
fprintln(f, """
|
|
/*
|
|
* Copyright 2016 ScyllaDB
|
|
*/
|
|
|
|
/*
|
|
* This file is part of Scylla.
|
|
*
|
|
* Scylla is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU Affero General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* Scylla is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
|
|
/*
|
|
* This is an auto-generated code, do not modify directly.
|
|
*/
|
|
#pragma once
|
|
""")
|
|
|
|
def parse_file(file_name):
|
|
|
|
number = pp.Word(pp.nums)
|
|
identifier = pp.Word(pp.alphas + "_", pp.alphanums + "_")
|
|
|
|
lbrace = pp.Literal('{').suppress()
|
|
rbrace = pp.Literal('}').suppress()
|
|
cls = pp.Keyword('class')
|
|
colon = pp.Literal(":")
|
|
semi = pp.Literal(";").suppress()
|
|
langle = pp.Literal("<")
|
|
rangle = pp.Literal(">")
|
|
equals = pp.Literal("=")
|
|
comma = pp.Literal(",")
|
|
lparen = pp.Literal("(")
|
|
rparen = pp.Literal(")")
|
|
lbrack = pp.Literal("[")
|
|
rbrack = pp.Literal("]")
|
|
mins = pp.Literal("-")
|
|
struct = pp.Keyword('struct')
|
|
template = pp.Keyword('template')
|
|
final = pp.Keyword('final')("final")
|
|
stub = pp.Keyword('stub')("stub")
|
|
with_colon = pp.Word(pp.alphanums + "_" + ":")
|
|
btype = with_colon
|
|
type = pp.Forward()
|
|
nestedParens = pp.nestedExpr('<', '>')
|
|
|
|
tmpl = pp.Group(btype("template_name") + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress())
|
|
type << (tmpl | btype)
|
|
enum_lit = pp.Keyword('enum')
|
|
enum_class = pp.Group(enum_lit + cls)
|
|
ns = pp.Keyword("namespace")
|
|
|
|
enum_init = equals.suppress() + pp.Optional(mins) + number
|
|
enum_value = pp.Group(identifier + pp.Optional(enum_init))
|
|
enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace)
|
|
content = pp.Forward()
|
|
|
|
member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen)))
|
|
attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress())
|
|
opt_attribute = pp.Optional(attrib)("attribute")
|
|
namespace = pp.Group(ns("type") + identifier("name") + lbrace + pp.Group(pp.OneOrMore(content))("content") + rbrace)
|
|
enum = pp.Group(enum_class("type") + identifier("name") + colon.suppress() + identifier("underline_type") + enum_values("enum_values") + pp.Optional(semi).suppress())
|
|
default_value = equals.suppress() + pp.SkipTo(';')
|
|
class_member = pp.Group(type("type") + member_name("name") + opt_attribute + pp.Optional(default_value)("default") + semi.suppress())("member")
|
|
template_param = pp.Group(identifier("type") + identifier("name"))
|
|
template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param))("params") + rangle)
|
|
class_content = pp.Forward()
|
|
class_def = pp.Group(pp.Optional(template_def)("template") + (cls | struct)("type") + with_colon("name") + pp.Optional(final) + pp.Optional(stub) + opt_attribute + lbrace + pp.Group(pp.ZeroOrMore(class_content))("members") + rbrace + pp.Optional(semi))
|
|
content << (enum | class_def | namespace)
|
|
class_content << (enum | class_def | class_member)
|
|
for varname in "enum class_def class_member content namespace template_def".split():
|
|
locals()[varname].setName(varname)
|
|
rt = pp.OneOrMore(content)
|
|
singleLineComment = "//" + pp.restOfLine
|
|
rt.ignore(singleLineComment)
|
|
rt.ignore(pp.cStyleComment)
|
|
return rt.parseFile(file_name, parseAll=True)
|
|
|
|
|
|
def combine_ns(namespaces):
|
|
return "::".join(namespaces)
|
|
|
|
def open_namespaces(namespaces):
|
|
return "".join(map(lambda a: "namespace " + a + " { ", namespaces))
|
|
|
|
def close_namespaces(namespaces):
|
|
return "".join(map(lambda a: "}", namespaces))
|
|
|
|
def set_namespace(namespaces):
|
|
ns = combine_ns(namespaces)
|
|
ns_open = open_namespaces(namespaces)
|
|
ns_close = close_namespaces(namespaces)
|
|
return [ns, ns_open, ns_close]
|
|
|
|
def declare_class(hout, name, ns_open, ns_close):
|
|
clas_def = ns_open + name + ";" + ns_close
|
|
fprintln(hout, "\n", clas_def)
|
|
|
|
def declear_methods(hout, name, template_param = ""):
|
|
if config.ns != '':
|
|
fprintln(hout, "namespace ", config.ns, " {")
|
|
fprintln(hout, Template("""
|
|
template <$tmp_param>
|
|
struct serializer<$name> {
|
|
template <typename Output>
|
|
static void write(Output& buf, const $name& v);
|
|
|
|
template <typename Input>
|
|
static $name read(Input& buf);
|
|
|
|
template <typename Input>
|
|
static void skip(Input& buf);
|
|
};
|
|
""").substitute({'name' : name, 'sizetype' : SIZETYPE, 'tmp_param' : template_param }))
|
|
if config.ns != '':
|
|
fprintln(hout, "}")
|
|
|
|
def handle_enum(enum, hout, cout, namespaces , parent_template_param = []):
|
|
[ns, ns_open, ns_close] = set_namespace(namespaces)
|
|
temp_def = ", ".join(map(lambda a: a[0] + " " + a[1], parent_template_param)) if parent_template_param else ""
|
|
template = "template <"+ temp_def +">" if temp_def else ""
|
|
name = enum["name"] if ns == "" else ns + "::" + enum["name"]
|
|
declear_methods(hout, name, temp_def)
|
|
fprintln(cout, Template("""
|
|
$template
|
|
template <typename Output>
|
|
void serializer<$name>::write(Output& buf, const $name& v) {
|
|
serialize(buf, static_cast<$type>(v));
|
|
}
|
|
|
|
$template
|
|
template<typename Input>
|
|
$name serializer<$name>::read(Input& buf) {
|
|
return static_cast<$name>(deserialize(buf, boost::type<$type>()));
|
|
}""").substitute({'name' : name, 'size_type' : SIZETYPE, 'type': enum['underline_type'], 'template' : template}))
|
|
|
|
|
|
def join_template(lst):
|
|
return "<" + ", ".join([param_type(l) for l in lst]) + ">"
|
|
|
|
def param_type(lst):
|
|
if isinstance(lst, str):
|
|
return lst
|
|
if len(lst) == 1:
|
|
return lst[0]
|
|
return lst[0] + join_template(lst[1])
|
|
|
|
def flat_template(lst):
|
|
return ", ".join([param_type(l) for l in lst])
|
|
|
|
def flat_type(lst):
|
|
if isinstance(lst, str):
|
|
return lst
|
|
if len(lst) == 1:
|
|
return flat_type(lst[0])
|
|
return (lst[0] + "__" + "_".join([flat_type(l) for l in lst[1]])).replace('::', '__')
|
|
|
|
def is_class(obj):
|
|
return obj["type"] == "class" or obj["type"] == "struct"
|
|
|
|
def is_enum(obj):
|
|
try:
|
|
return not isinstance(obj["type"], str) and "".join(obj["type"]) == 'enumclass'
|
|
except:
|
|
return False
|
|
local_types = {}
|
|
|
|
def list_types(lst):
|
|
if isinstance(lst, str):
|
|
return [lst]
|
|
if len(lst) == 1:
|
|
return lst
|
|
lt = reduce(lambda a,b: a + b, [list_types(l) for l in lst[1]])
|
|
return lt
|
|
|
|
def list_local_types(lst):
|
|
return {l for l in list_types(lst) if l in local_types}
|
|
|
|
def is_basic_type(lst):
|
|
if isinstance(lst, str):
|
|
return lst not in local_types
|
|
if len(lst) == 1:
|
|
return isinstance(lst[0], str) and lst[0] not in local_types
|
|
return False
|
|
|
|
def is_local_type(lst):
|
|
return isinstance(lst, str) and lst in local_types or len(lst) == 1 and lst[0] in local_types
|
|
|
|
def get_template_name(lst):
|
|
return lst["template_name"] if not isinstance(lst, str) and len(lst) >1 else None
|
|
|
|
def is_vector(lst):
|
|
return get_template_name(lst) == "std::vector"
|
|
|
|
def is_variant(lst):
|
|
return get_template_name(lst) == "boost::variant"
|
|
|
|
def is_optional(lst):
|
|
return get_template_name(lst) == "std::experimental::optional"
|
|
|
|
created_writers = set()
|
|
|
|
def get_member_name(name):
|
|
return name if not name.endswith('()') else name[:-2]
|
|
|
|
def get_members(cls):
|
|
return [p for p in cls["members"] if not is_class(p) and not is_enum(p)]
|
|
|
|
def is_final(cls):
|
|
return "final" in cls
|
|
def get_variant_type(lst):
|
|
if is_variant(lst):
|
|
return "variant"
|
|
return param_type(lst)
|
|
|
|
def variant_to_member(typ):
|
|
return [{"name" : get_variant_type(x), "type" : x } for x in typ if is_local_type(x) or is_variant(x)]
|
|
|
|
def variant_info(info, typ):
|
|
[cls, namespaces, parent_template_param] = info
|
|
return [{"members" : variant_to_member(typ)}, namespaces, parent_template_param]
|
|
|
|
stubs = set()
|
|
def is_stub(cls):
|
|
return cls in stubs
|
|
|
|
def handle_visitors_state(info, hout, clases = []):
|
|
[cls, namespaces, parent_template_param] = info
|
|
name = "__".join(clases) if clases else cls["name"]
|
|
frame = "empty_frame" if "final" in cls else "frame"
|
|
fprintln(hout, Template("""
|
|
template<typename Output>
|
|
struct state_of_$name {
|
|
$frame<Output> f;""").substitute({'name': name, 'frame': frame }))
|
|
if clases:
|
|
local_state = "state_of_" + "__".join(clases[:-1]) + '<Output>'
|
|
fprintln(hout, Template(" $name _parent;").substitute({'name': local_state}))
|
|
if "final" in cls:
|
|
fprintln(hout, Template(" state_of_$name($state parent) : _parent(parent) {}").substitute({'name': name, 'state' : local_state}))
|
|
fprintln(hout, "};")
|
|
members = get_members(cls)
|
|
member_class = clases if clases else [cls["name"]]
|
|
for param in members:
|
|
if is_local_type(param["type"]):
|
|
handle_visitors_state(local_types[param_type(param["type"])], hout, member_class + [param["name"]])
|
|
if is_variant(param["type"]):
|
|
handle_visitors_state(variant_info(info, param["type"][1]), hout, member_class + [param["name"]])
|
|
|
|
def get_dependency(cls):
|
|
members = get_members(cls)
|
|
return reduce(lambda a,b: a |b, [list_local_types(m["type"]) for m in members], set())
|
|
|
|
def add_vector_node(hout, cls, members, base_state, current_node, ind):
|
|
current = members[ind]
|
|
typ = current["type"][1]
|
|
fprintln(hout, Template("""
|
|
template<typename Output>
|
|
struct $node_name {
|
|
Outout& _out;
|
|
state_of_$base_state<Output> _state;
|
|
place_holder<Output> _size;
|
|
size_type _count = 0;
|
|
$node_name(Output& out, state_of_$base_state<Output> state)
|
|
: _out(out)
|
|
, _state(state)
|
|
, _size(start_place_holder(out))
|
|
{
|
|
}
|
|
$next_state<Output> end_$name() {
|
|
_size.set(_out, _count);
|
|
}""").substitute({'node_name': '', 'name': current["name"] }))
|
|
|
|
fprintln(hout, "};")
|
|
|
|
def optional_add_methods(typ):
|
|
res = reindent(4,"""
|
|
void skip() {
|
|
serialize(_out, false);
|
|
}""")
|
|
if is_basic_type(typ):
|
|
added_type = typ
|
|
elif is_local_type(typ):
|
|
added_type = param_type(typ) + "_view"
|
|
else:
|
|
print("non supported optional type ", typ)
|
|
raise "non supported optional type " + param_type(typ)
|
|
res = res + Template(reindent(4, """
|
|
void write(const $type& obj) {
|
|
serialize(_out, true);
|
|
serialize(_out, obj);
|
|
}""")).substitute({'type' : added_type})
|
|
if is_local_type(typ):
|
|
res = res + Template(reindent(4, """
|
|
writer_of_$type<Output> write() {
|
|
serialize(_out, true);
|
|
return {_out};
|
|
}""")).substitute({'type' : param_type(typ)})
|
|
return res
|
|
|
|
|
|
def vector_add_method(current, base_state):
|
|
typ = current["type"]
|
|
res = ""
|
|
if is_basic_type(typ[1]):
|
|
res = res + Template("""
|
|
void add_$name($type t) {
|
|
serialize(_out, t);
|
|
_count++;
|
|
}""").substitute({'type': param_type(typ[1][0]), 'name': current["name"]})
|
|
else:
|
|
res = res + Template("""
|
|
writer_of_$type<Output> add() {
|
|
_count++;
|
|
return {_out};
|
|
}""").substitute({'type': flat_type(typ[1][0]), 'name': current["name"]})
|
|
res = res + Template("""
|
|
void add(${type} v) {
|
|
serialize(_out, v);
|
|
_count++;
|
|
}""").substitute({'type': param_view_type(typ[1][0])})
|
|
return res + Template("""
|
|
after_${basestate}__$name<Output> end_$name() && {
|
|
_size.set(_out, _count);
|
|
return { _out, std::move(_state) };
|
|
}
|
|
|
|
vector_position pos() const {
|
|
return vector_position{_out.pos(), _count};
|
|
}
|
|
|
|
void rollback(const vector_position& vp) {
|
|
_out.retract(vp.pos);
|
|
_count = vp.count;
|
|
}""").substitute({'name': current["name"], 'basestate' : base_state})
|
|
|
|
def add_param_writer_basic_type(name, base_state, typ, var_type = "", var_index = None, root_node = False):
|
|
if isinstance(var_index, Number):
|
|
var_index = "uint32_t(" + str(var_index) +")"
|
|
set_varient_index = "serialize(_out, " + var_index +");\n" if var_index is not None else ""
|
|
set_command = "_state.f.end(_out);" if var_type is not "" else ""
|
|
return_command = "{ _out, std::move(_state._parent) }" if var_type is not "" and not root_node else "{ _out, std::move(_state) }"
|
|
|
|
if typ in ['bytes', 'sstring']:
|
|
typ += '_view'
|
|
else:
|
|
typ = 'const ' + typ + '&'
|
|
|
|
return Template(reindent(4, """
|
|
after_${base_state}__$name<Output> write_$name$var_type($typ t) && {
|
|
$set_varient_index
|
|
serialize(_out, t);
|
|
$set_command
|
|
return $return_command;
|
|
}""")).substitute(locals())
|
|
|
|
def add_param_writer_object(name, base_state, typ, var_type = "", var_index = None, root_node = False):
|
|
var_type1 = "_" + var_type if var_type != "" else ""
|
|
if isinstance(var_index, Number):
|
|
var_index = "uint32_t(" + str(var_index) +")"
|
|
set_varient_index = "serialize(_out, " + var_index +");\n" if var_index is not None else ""
|
|
ret = Template(reindent(4,"""
|
|
${base_state}__${name}$var_type1<Output> start_${name}$var_type() && {
|
|
$set_varient_index
|
|
return { _out, std::move(_state) };
|
|
}
|
|
""")).substitute(locals())
|
|
if not is_stub(typ) and is_local_type(typ):
|
|
ret += add_param_writer_basic_type(name, base_state, typ, var_type, var_index, root_node)
|
|
if is_stub(typ):
|
|
set_command = "_state.f.end(_out);" if var_type is not "" else ""
|
|
return_command = "{ _out, std::move(_state._parent) }" if var_type is not "" and not root_node else "{ _out, std::move(_state) }"
|
|
ret += Template(reindent(4, """
|
|
template<typename Serializer>
|
|
after_${base_state}__${name}<Output> ${name}$var_type(Serializer&& f) && {
|
|
$set_varient_index
|
|
f(writer_of_$typ<Output>(_out));
|
|
$set_command
|
|
return $return_command;
|
|
}""")).substitute(locals())
|
|
return ret
|
|
|
|
def add_param_write(current, base_state, vector = False, root_node = False):
|
|
typ = current["type"]
|
|
res = ""
|
|
name = get_member_name(current["name"])
|
|
if is_basic_type(typ):
|
|
res = res + add_param_writer_basic_type(name, base_state, typ)
|
|
elif is_optional(typ):
|
|
res = res + Template(reindent(4, """
|
|
after_${basestate}__$name<Output> skip_$name() && {
|
|
serialize(_out, false);
|
|
return { _out, std::move(_state) };
|
|
}""")).substitute({'type': param_type(typ), 'name': name, 'basestate' : base_state})
|
|
if is_basic_type(typ[1][0]):
|
|
res = res + add_param_writer_basic_type(name, base_state, typ[1][0], "", "true")
|
|
elif is_local_type(typ[1][0]):
|
|
res = res + add_param_writer_object(name, base_state[0][1], typ, "", "true")
|
|
else:
|
|
print("non supported optional type ", type[0][1])
|
|
elif is_vector(typ):
|
|
set_size = "_size.set(_out, 0);" if vector else "serialize(_out, size_type(0));"
|
|
|
|
res = res + Template("""
|
|
${basestate}__$name<Output> start_$name() && {
|
|
return { _out, std::move(_state) };
|
|
}
|
|
|
|
after_${basestate}__$name<Output> skip_$name() && {
|
|
$set
|
|
return { _out, std::move(_state) };
|
|
}
|
|
""").substitute({'type': param_type(typ), 'name': name, 'basestate' : base_state, 'set' : set_size})
|
|
elif is_local_type(typ):
|
|
res = res + add_param_writer_object(name, base_state, typ)
|
|
elif is_variant(typ):
|
|
for idx, p in enumerate(typ[1]):
|
|
if is_basic_type(p):
|
|
varient_type = param_type(p)
|
|
res = res + add_param_writer_basic_type(name, base_state, varient_type,"_" + varient_type, idx, root_node)
|
|
elif is_variant(p):
|
|
res = res + add_param_writer_object(name, base_state, p, '_' + "variant", idx, root_node)
|
|
elif is_local_type(p):
|
|
res = res + add_param_writer_object(name, base_state, p, '_' + param_type(p), idx, root_node)
|
|
else:
|
|
print ("something is wrong with type", typ)
|
|
return res;
|
|
|
|
def get_return_struct(variant_node, clases):
|
|
if not variant_node:
|
|
return clases
|
|
if clases[-2] == "variant":
|
|
return clases[:-2]
|
|
return clases[:-1]
|
|
|
|
|
|
def add_variant_end_method(base_state, name, clases):
|
|
|
|
return_struct = "after_" + base_state + '<Output>'
|
|
return Template("""
|
|
$return_struct end_$name() && {
|
|
_state.f.end(_out);
|
|
_state._parent.f.end(_out);
|
|
return { _out, std::move(_state._parent._parent) };
|
|
}
|
|
""").substitute({'name': name, 'return_struct':return_struct})
|
|
|
|
def add_end_method(parents, name, variant_node = False, return_value = True):
|
|
if variant_node:
|
|
return add_variant_end_method(parents, name, return_value)
|
|
base_state = parents + "__" + name
|
|
if return_value:
|
|
return_struct = "after_" + base_state + '<Output>'
|
|
return Template("""
|
|
$return_struct end_$name() && {
|
|
_state.f.end(_out);
|
|
return { _out, std::move(_state._parent) };
|
|
}
|
|
""").substitute({'name': name, 'return_struct':return_struct})
|
|
return Template("""
|
|
void end_$name() {
|
|
_state.f.end(_out);
|
|
}
|
|
""").substitute({'name': name, 'basestate':base_state})
|
|
|
|
def add_vector_placeholder():
|
|
return """ place_holder<Output> _size;
|
|
size_type _count = 0;"""
|
|
|
|
def add_node(hout, name, member, base_state, prefix, parents, fun, is_type_vector = False, is_type_final = False):
|
|
struct_name = prefix + name
|
|
if member and is_type_vector:
|
|
vector_placeholder = add_vector_placeholder()
|
|
vector_init = "\n , _size(start_place_holder(out))"
|
|
else:
|
|
vector_placeholder = ""
|
|
vector_init = ""
|
|
if vector_init != "" or prefix == "":
|
|
state_init = "_state{start_frame(out), std::move(state)}" if parents != base_state and not is_type_final else "_state(state)"
|
|
else:
|
|
if member and is_variant(member) and parents != base_state:
|
|
state_init = "_state{start_frame(out), std::move(state)}"
|
|
sate = name
|
|
else:
|
|
state_init = ""
|
|
if prefix == "writer_of_":
|
|
constructor = Template("""$name(Output& out)
|
|
: _out(out)
|
|
, _state{start_frame(out)}${vector_init}
|
|
{}""").substitute({'name': struct_name, 'vector_init' : vector_init})
|
|
elif state_init != "":
|
|
constructor = Template("""$name(Output& out, state_of_$state<Output> state)
|
|
: _out(out)
|
|
, $state_init${vector_init}
|
|
{}""").substitute({'name': struct_name, 'vector_init' : vector_init, 'state' : parents, 'state_init' : state_init})
|
|
else:
|
|
constructor = ""
|
|
fprintln(hout, Template("""
|
|
template<typename Output>
|
|
struct $name {
|
|
Output& _out;
|
|
state_of_$state<Output> _state;
|
|
${vector_placeholder}
|
|
${constructor}
|
|
$fun
|
|
};""").substitute({'name': struct_name, 'state' : base_state, 'fun' : fun, 'vector_placeholder': vector_placeholder, 'constructor': constructor}))
|
|
|
|
def add_vector_node(hout, member, base_state, parents):
|
|
if get_template_name(member["type"][1][0]):
|
|
add_template_writer_node(hout,member["type"][1][0] )
|
|
add_node(hout, base_state + "__" + member["name"], member["type"], base_state, "", parents, vector_add_method(member, base_state), True)
|
|
|
|
optional_nodes = set()
|
|
|
|
def add_optional_node(hout, typ):
|
|
global optional_nodes
|
|
full_type = flat_type(typ)
|
|
if full_type in optional_nodes:
|
|
return
|
|
optional_nodes.add(full_type)
|
|
fprintln(hout, Template(reindent(0,"""
|
|
template<typename Output>
|
|
struct writer_of_$type {
|
|
Output& _out;
|
|
$add_method
|
|
};""")).substitute({'type': full_type, 'add_method': optional_add_methods(typ[1][0])}))
|
|
|
|
def add_variant_nodes(hout, member, param, base_state, parents, classes):
|
|
vr = False
|
|
par = base_state + "__" + member["name"]
|
|
for typ in param[1]:
|
|
if is_local_type(typ):
|
|
handle_visitors_nodes(local_types[param_type(typ)], hout, True, classes + [member["name"], local_types[param_type(typ)][0]["name"]])
|
|
if is_variant(typ):
|
|
name = base_state + "__" + member["name"] + "__variant"
|
|
new_member = {"type": typ, "name" : "variant"}
|
|
return_struct = "after_" + par
|
|
end_method = Template("""
|
|
$return_struct<Output> end_variant() && {
|
|
_state.f.end(_out);
|
|
return { _out, std::move(_state._parent) };
|
|
}
|
|
""").substitute({'return_struct':return_struct})
|
|
add_node(hout, name, None, base_state + "__" + member["name"], "after_", name, end_method)
|
|
add_variant_nodes(hout, new_member, typ, par, name, classes + [member["name"]])
|
|
add_node(hout, name, typ, name, "", par, add_param_write({"type": typ, "name" : "variant"}, par))
|
|
vr = True
|
|
writers = set()
|
|
|
|
def add_template_writer_node(hout, typ):
|
|
if is_optional(typ):
|
|
add_optional_node(hout, typ)
|
|
|
|
|
|
def add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes):
|
|
if is_vector(member["type"]):
|
|
add_vector_node(hout, member, base_state_name, base_state_name)
|
|
elif is_variant(member["type"]):
|
|
add_variant_nodes(hout, member, member["type"], base_state_name, parents, member_classes)
|
|
elif is_local_type(member["type"]):
|
|
handle_visitors_nodes(local_types[member["type"]], hout, False, member_classes + [member["name"]])
|
|
|
|
def handle_visitors_nodes(info, hout, variant_node = False, clases = []):
|
|
global writers
|
|
[cls, namespaces, parent_template_param] = info
|
|
#for root node, only generate once
|
|
if not clases:
|
|
if cls["name"] in writers:
|
|
return
|
|
writers.add(cls["name"])
|
|
|
|
members = get_members(cls)
|
|
if clases:
|
|
base_state_name = "__".join(clases)
|
|
if variant_node:
|
|
parents = "__".join(clases[:-1])
|
|
else:
|
|
parents = "__".join(clases[:-1])
|
|
current_name = clases[-1]
|
|
else:
|
|
base_state_name = cls["name"]
|
|
current_name = cls["name"]
|
|
parents = ""
|
|
member_classes = clases if clases else [current_name]
|
|
prefix = "" if clases else "writer_of_"
|
|
if not members:
|
|
add_node(hout, base_state_name, None, base_state_name, prefix, parents, add_end_method(parents, current_name, variant_node, clases), False, is_final(cls))
|
|
return
|
|
add_node(hout, base_state_name + "__" + get_member_name(members[-1]["name"]), members[-1]["type"], base_state_name, "after_", base_state_name, add_end_method(parents, current_name, variant_node, clases))
|
|
# Create writer and reader for include class
|
|
if not variant_node:
|
|
for member in get_dependency(cls):
|
|
handle_visitors_nodes(local_types[member], hout)
|
|
for ind in reversed(range(1, len(members))):
|
|
member = members[ind]
|
|
add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes)
|
|
variant_state = base_state_name + "__" + get_member_name(member["name"]) if is_variant(member["type"]) else base_state_name
|
|
is_param_vector = is_vector(member["type"]) and is_basic_type(member["type"][1][0])
|
|
add_node(hout, base_state_name + "__" + get_member_name(members[ind - 1]["name"]), member["type"], variant_state, "after_", base_state_name, add_param_write(member, base_state_name), False)
|
|
member = members[0]
|
|
is_param_vector = is_vector(member["type"]) and is_basic_type(member["type"][1][0])
|
|
add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes)
|
|
add_node(hout, base_state_name, member["type"], base_state_name, prefix, parents, add_param_write(member, base_state_name, False, not clases), False, is_final(cls))
|
|
|
|
def add_to_types(cls, namespaces, parent_template_param ):
|
|
global local_types
|
|
global stubs
|
|
if "attribute" not in cls or cls["attribute"][0][0] != "writable":
|
|
return
|
|
local_types[cls["name"]] = [cls, namespaces, parent_template_param]
|
|
if "stub" in cls:
|
|
stubs.add(cls["name"])
|
|
for param in cls["members"]:
|
|
if is_class(param) or is_enum(param):
|
|
continue
|
|
|
|
def sort_dependencies():
|
|
dep_tree = {}
|
|
res = []
|
|
for k in local_types:
|
|
[cls, namespaces, parent_template_param] = local_types[k]
|
|
dep_tree[k] = get_dependency(cls)
|
|
while (len(dep_tree) > 0):
|
|
found = set()
|
|
found = found | { k for k in dep_tree if not dep_tree[k]}
|
|
res = res + [k for k in found]
|
|
for k in found:
|
|
dep_tree.pop(k)
|
|
for k in dep_tree:
|
|
if dep_tree[k]:
|
|
dep_tree[k].difference_update(found)
|
|
|
|
return res
|
|
|
|
|
|
def join_template_view(lst, more_types = []):
|
|
return "<" + ", ".join([param_view_type(l) for l in lst] + more_types) + ">"
|
|
|
|
def to_view(val):
|
|
if val in local_types:
|
|
return val +"_view"
|
|
return val
|
|
|
|
def param_view_type(lst):
|
|
if isinstance(lst, str):
|
|
return to_view(lst)
|
|
if len(lst) == 1:
|
|
return to_view(lst[0])
|
|
if lst[0] == "boost::variant":
|
|
return lst[0] + join_template_view(lst[1], ["unknown_variant_type"])
|
|
return lst[0] + join_template_view(lst[1])
|
|
|
|
read_sizes = set()
|
|
|
|
def add_variant_read_size(hout, typ):
|
|
global read_sizes
|
|
t = param_view_type(typ)
|
|
if t in read_sizes:
|
|
return
|
|
if not is_variant(typ):
|
|
return
|
|
for p in typ[1]:
|
|
if is_variant(p):
|
|
add_variant_read_size(hout, p)
|
|
read_sizes.add(t)
|
|
fprintln(hout, Template("""
|
|
template<typename Input>
|
|
inline void skip(Input& v, boost::type<${type}>) {
|
|
return seastar::with_serialized_stream(v, [] (auto& v) {
|
|
size_type ln = deserialize(v, boost::type<size_type>());
|
|
v.skip(ln - sizeof(size_type));
|
|
});
|
|
}""").substitute ({'type' : t}))
|
|
|
|
|
|
fprintln(hout, Template("""
|
|
template<typename Input>
|
|
$type deserialize(Input& v, boost::type<${type}>) {
|
|
return seastar::with_serialized_stream(v, [] (auto& v) {
|
|
auto in = v;
|
|
deserialize(in, boost::type<size_type>());
|
|
size_type o = deserialize(in, boost::type<size_type>());
|
|
""").substitute ({'type' : t}))
|
|
for index, param in enumerate(typ[1]):
|
|
fprintln(hout, Template("""
|
|
if (o == $ind) {
|
|
v.skip(sizeof(size_type)*2);
|
|
return $full_type(deserialize(v, boost::type<$type>()));
|
|
}""").substitute({'ind' : index, 'type' : param_view_type(param), 'full_type' : t}))
|
|
fprintln(hout, ' return ' + t + '(deserialize(v, boost::type<unknown_variant_type>()));\n });\n}')
|
|
|
|
def add_view(hout, info):
|
|
[cls, namespaces, parent_template_param] = info
|
|
members = get_members(cls)
|
|
for m in members:
|
|
add_variant_read_size(hout, m["type"])
|
|
|
|
fprintln(hout, Template("""struct ${name}_view {
|
|
utils::input_stream v;
|
|
""").substitute({'name' : cls["name"]}))
|
|
|
|
if not is_stub(cls["name"]) and is_local_type(cls["name"]):
|
|
fprintln(hout, Template(reindent(4, """
|
|
operator $type() const {
|
|
auto in = v;
|
|
return deserialize(in, boost::type<$type>());
|
|
}
|
|
""")).substitute({'type' : cls["name"]}))
|
|
|
|
skip = "" if is_final(cls) else "ser::skip(in, boost::type<size_type>());"
|
|
for m in members:
|
|
full_type = param_view_type(m["type"])
|
|
fprintln(hout, Template(reindent(4, """
|
|
auto $name() const {
|
|
return seastar::with_serialized_stream(v, [] (auto& v) {
|
|
auto in = v;
|
|
$skip
|
|
return deserialize(in, boost::type<$type>());
|
|
});
|
|
}
|
|
""")).substitute({'name' : get_member_name(m["name"]), 'type' : full_type, 'skip' : skip}))
|
|
|
|
skip = skip + Template("\n ser::skip(in, boost::type<${type}>());").substitute({'type': full_type})
|
|
|
|
fprintln(hout, "};")
|
|
skip_impl = "auto& in = v;\n " + skip if is_final(cls) else "v.skip(read_frame_size(v));"
|
|
if skip == "":
|
|
skip_impl = ""
|
|
|
|
fprintln(hout, Template("""
|
|
template<>
|
|
struct serializer<${type}_view> {
|
|
template<typename Input>
|
|
static ${type}_view read(Input& v) {
|
|
return seastar::with_serialized_stream(v, [] (auto& v) {
|
|
auto v_start = v;
|
|
auto start_size = v.size();
|
|
skip(v);
|
|
return ${type}_view{v_start.read_substream(start_size - v.size())};
|
|
});
|
|
}
|
|
template<typename Output>
|
|
static void write(Output& out, ${type}_view v) {
|
|
v.v.copy_to(out);
|
|
}
|
|
template<typename Input>
|
|
static void skip(Input& v) {
|
|
return seastar::with_serialized_stream(v, [] (auto& v) {
|
|
$skip_impl
|
|
});
|
|
}
|
|
};
|
|
""").substitute({'type' : param_type(cls["name"]), 'skip' : skip, 'skip_impl' : skip_impl}))
|
|
|
|
def add_views(hout):
|
|
for k in sort_dependencies():
|
|
add_view(hout, local_types[k])
|
|
|
|
def add_visitors(hout):
|
|
if not local_types:
|
|
return
|
|
add_views(hout)
|
|
fprintln(hout, "\n////// State holders")
|
|
for k in local_types:
|
|
handle_visitors_state(local_types[k], hout)
|
|
fprintln(hout, "\n////// Nodes")
|
|
for k in sort_dependencies():
|
|
handle_visitors_nodes(local_types[k], hout)
|
|
|
|
def handle_class(cls, hout, cout, namespaces=[], parent_template_param = []):
|
|
add_to_types(cls, namespaces, parent_template_param)
|
|
if "stub" in cls:
|
|
return
|
|
[ns, ns_open, ns_close] = set_namespace(namespaces)
|
|
tpl = "template" in cls
|
|
template_param_list = (cls["template"][0]["params"].asList() if tpl else [])
|
|
template_param = ", ".join(map(lambda a: a[0] + " " + a[1], template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else ""
|
|
template = "template <"+ template_param +">" if tpl else ""
|
|
template_class_param = "<" + ",".join(map(lambda a: a[1], template_param_list)) + ">" if tpl else ""
|
|
temp_def = template_param if template_param != "" else ""
|
|
|
|
if ns == "":
|
|
name = cls["name"]
|
|
else:
|
|
name = ns + "::" + cls["name"]
|
|
full_name = name + template_class_param
|
|
for param in cls["members"]:
|
|
if is_class(param):
|
|
handle_class(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list)
|
|
elif is_enum(param):
|
|
handle_enum(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list)
|
|
declear_methods(hout, name + template_class_param, temp_def)
|
|
is_final = "final" in cls
|
|
|
|
fprintln(cout, Template("""
|
|
$template
|
|
template <typename Output>
|
|
void serializer<$name>::write(Output& buf, const $name& obj) {""").substitute({'func' : SERIALIZER, 'name' : full_name, 'template': template}))
|
|
if not is_final:
|
|
fprintln(cout, Template(""" $set_size(buf, obj);""").substitute({'func' : SERIALIZER, 'set_size' : SETSIZE, 'name' : name, 'sizetype' : SIZETYPE}))
|
|
for param in cls["members"]:
|
|
if is_class(param) or is_enum(param):
|
|
continue
|
|
fprintln(cout, Template(""" static_assert(is_equivalent<decltype(obj.$var), $type>::value, "member value has a wrong type");
|
|
$func(buf, obj.$var);""").substitute({'func' : SERIALIZER, 'var' : param["name"], 'type' : param_type(param["type"])}))
|
|
fprintln(cout, "}")
|
|
|
|
fprintln(cout, Template("""
|
|
$template
|
|
template <typename Input>
|
|
$name$temp_param serializer<$name$temp_param>::read(Input& buf) {
|
|
return seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'func' : DESERIALIZER, 'name' : name, 'template': template, 'temp_param' : template_class_param}))
|
|
if not is_final:
|
|
fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>());
|
|
auto in = buf.read_substream(size - sizeof($size_type));""").substitute({'func' : DESERIALIZER, 'size_type' : SIZETYPE}))
|
|
else:
|
|
fprintln(cout, """ auto& in = buf;""")
|
|
params = []
|
|
local_names = {}
|
|
for index, param in enumerate(cls["members"]):
|
|
if is_class(param) or is_enum(param):
|
|
continue
|
|
local_param = "__local_" + str(index)
|
|
local_names[param["name"]] = local_param
|
|
if "attribute" in param:
|
|
deflt = param["default"][0] if "default" in param else param_type(param["type"]) + "()"
|
|
if deflt in local_names:
|
|
deflt = local_names[deflt]
|
|
fprintln(cout, Template(""" auto $local = (in.size()>0) ?
|
|
$func(in, boost::type<$typ>()) : $default;""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param, 'default': deflt}))
|
|
else:
|
|
fprintln(cout, Template(""" auto $local = $func(in, boost::type<$typ>());""").substitute({'func' : DESERIALIZER, 'typ': param_type(param["type"]), 'local' : local_param}))
|
|
params.append("std::move(" + local_param + ")")
|
|
fprintln(cout, Template("""
|
|
$name$temp_param res {$params};
|
|
return res;
|
|
});
|
|
}""").substitute({'name' : name, 'params': ", ".join(params), 'temp_param' : template_class_param}))
|
|
|
|
fprintln(cout, Template("""
|
|
$template
|
|
template <typename Input>
|
|
void serializer<$name$temp_param>::skip(Input& buf) {
|
|
seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'func' : DESERIALIZER, 'name' : name, 'template': template, 'temp_param' : template_class_param}))
|
|
if not is_final:
|
|
fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>());
|
|
buf.skip(size - sizeof($size_type));""").substitute({'func' : DESERIALIZER, 'size_type' : SIZETYPE}))
|
|
else:
|
|
for m in get_members(cls):
|
|
full_type = param_view_type(m["type"])
|
|
fprintln(cout, " ser::skip(buf, boost::type<%s>());" % full_type)
|
|
fprintln(cout, """ });\n}""")
|
|
|
|
|
|
def handle_objects(tree, hout, cout, namespaces=[]):
|
|
for obj in tree:
|
|
if is_class(obj):
|
|
handle_class(obj, hout, cout, namespaces)
|
|
elif is_enum(obj):
|
|
handle_enum(obj, hout, cout, namespaces)
|
|
elif obj["type"] == "namespace":
|
|
handle_objects(obj["content"], hout, cout, namespaces + [obj["name"]])
|
|
else:
|
|
print("unknown type ", obj, obj["type"])
|
|
|
|
def handle_types(tree, namespaces=[]):
|
|
for obj in tree:
|
|
if is_class(obj):
|
|
add_to_types(obj, namespaces, [])
|
|
elif is_enum(obj):
|
|
pass
|
|
elif obj["type"] == "namespace":
|
|
handle_types(obj["content"], namespaces + [obj["name"]])
|
|
else:
|
|
print("unknown type ", obj, obj["type"])
|
|
|
|
def load_file(name):
|
|
if config.o:
|
|
cout = open(config.o.replace('.hh', '.impl.hh'), "w+")
|
|
hout = open(config.o, "w+")
|
|
else:
|
|
cout = open(name.replace(EXTENSION, '.dist.impl.hh'), "w+")
|
|
hout = open(name.replace(EXTENSION, '.dist.hh'), "w+")
|
|
print_cw(hout)
|
|
fprintln(hout, """
|
|
/*
|
|
* The generate code should be included in a header file after
|
|
* The object definition
|
|
*/
|
|
""")
|
|
print_cw(cout)
|
|
fprintln(hout, "#include \"serializer.hh\"\n")
|
|
if config.ns != '':
|
|
fprintln(cout, "namespace ", config.ns, " {")
|
|
data = parse_file(name)
|
|
if data:
|
|
handle_types(data)
|
|
add_visitors(cout)
|
|
if data:
|
|
handle_objects(data, hout, cout)
|
|
if config.ns != '':
|
|
fprintln(cout, "}")
|
|
cout.close()
|
|
hout.close()
|
|
|
|
def general_include(files):
|
|
name = config.o if config.o else "serializer.dist.hh"
|
|
cout = open(name.replace('.hh', '.impl.hh'), "w+")
|
|
hout = open(name, "w+")
|
|
print_cw(cout)
|
|
print_cw(hout)
|
|
for n in files:
|
|
fprintln(hout, '#include "' + n +'"')
|
|
fprintln(cout, '#include "' + n.replace(".dist.hh", '.dist.impl.hh') +'"')
|
|
cout.close()
|
|
hout.close()
|
|
if config.file:
|
|
general_include(config.file)
|
|
elif config.f != '':
|
|
load_file(config.f)
|