mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
76bc30a82d668d541384631892262d2a2454a521
scylladb/idl-compiler.py
Duarte Nunes fa2b0384d2 Replace std::experimental types with C++17 std version. 
Replace stdx::optional and stdx::string_view with the C++ std
counterparts.

Some instances of boost::variant were also replaced with std::variant,
namely those that called seastar::visit.

Scylla now requires GCC 8 to compile.

Signed-off-by: Duarte Nunes <duarte@scylladb.com>
Message-Id: <20190108111141.5369-1-duarte@scylladb.com>
2019-01-08 13:16:36 +02:00

1067 lines
38 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright 2016 ScyllaDB
#
#
# This file is part of Scylla.
#
# Scylla is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Scylla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
import argparse
from string import Template
import pyparsing as pp
from functools import reduce
import textwrap
from numbers import Number
EXTENSION = '.idl.hh'
READ_BUFF = 'input_buffer'
WRITE_BUFF = 'output_buffer'
SERIALIZER = 'serialize'
DESERIALIZER = 'deserialize'
SETSIZE = 'set_size'
SIZETYPE = 'size_type'
parser = argparse.ArgumentParser(description="""Generate serializer helper function""")
parser.add_argument('-o', help='Output file', default='')
parser.add_argument('-f', help='input file', default='')
parser.add_argument('--ns', help="""namespace, when set function will be created
under the given namespace""", default='')
parser.add_argument('file', nargs='*', help="combine one or more file names for the genral include files")
config = parser.parse_args()
def reindent(indent, text):
return textwrap.indent(textwrap.dedent(text), ' ' * indent)
def fprint(f, *args):
for arg in args:
f.write(arg)
def fprintln(f, *args):
for arg in args:
f.write(arg)
f.write('\n')
def print_cw(f):
fprintln(f, """
/*
* Copyright 2016 ScyllaDB
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
/*
* This is an auto-generated code, do not modify directly.
*/
#pragma once
""")
def parse_file(file_name):
number = pp.Word(pp.nums)
identifier = pp.Word(pp.alphas + "_", pp.alphanums + "_")
lbrace = pp.Literal('{').suppress()
rbrace = pp.Literal('}').suppress()
cls = pp.Keyword('class')
colon = pp.Literal(":")
semi = pp.Literal(";").suppress()
langle = pp.Literal("<")
rangle = pp.Literal(">")
equals = pp.Literal("=")
comma = pp.Literal(",")
lparen = pp.Literal("(")
rparen = pp.Literal(")")
lbrack = pp.Literal("[")
rbrack = pp.Literal("]")
mins = pp.Literal("-")
struct = pp.Keyword('struct')
template = pp.Keyword('template')
final = pp.Keyword('final')("final")
stub = pp.Keyword('stub')("stub")
with_colon = pp.Word(pp.alphanums + "_" + ":")
btype = with_colon
type = pp.Forward()
nestedParens = pp.nestedExpr('<', '>')
tmpl = pp.Group(btype("template_name") + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress())
type << (tmpl | btype)
enum_lit = pp.Keyword('enum')
enum_class = pp.Group(enum_lit + cls)
ns = pp.Keyword("namespace")
enum_init = equals.suppress() + pp.Optional(mins) + number
enum_value = pp.Group(identifier + pp.Optional(enum_init))
enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace)
content = pp.Forward()
member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen)))
attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress())
opt_attribute = pp.Optional(attrib)("attribute")
namespace = pp.Group(ns("type") + identifier("name") + lbrace + pp.Group(pp.OneOrMore(content))("content") + rbrace)
enum = pp.Group(enum_class("type") + identifier("name") + colon.suppress() + identifier("underline_type") + enum_values("enum_values") + pp.Optional(semi).suppress())
default_value = equals.suppress() + pp.SkipTo(';')
class_member = pp.Group(type("type") + member_name("name") + opt_attribute + pp.Optional(default_value)("default") + semi.suppress())("member")
template_param = pp.Group(identifier("type") + identifier("name"))
template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param))("params") + rangle)
class_content = pp.Forward()
class_def = pp.Group(pp.Optional(template_def)("template") + (cls | struct)("type") + with_colon("name") + pp.Optional(final) + pp.Optional(stub) + opt_attribute + lbrace + pp.Group(pp.ZeroOrMore(class_content))("members") + rbrace + pp.Optional(semi))
content << (enum | class_def | namespace)
class_content << (enum | class_def | class_member)
for varname in "enum class_def class_member content namespace template_def".split():
locals()[varname].setName(varname)
rt = pp.OneOrMore(content)
singleLineComment = "//" + pp.restOfLine
rt.ignore(singleLineComment)
rt.ignore(pp.cStyleComment)
return rt.parseFile(file_name, parseAll=True)
def combine_ns(namespaces):
return "::".join(namespaces)
def open_namespaces(namespaces):
return "".join(map(lambda a: "namespace " + a + " { ", namespaces))
def close_namespaces(namespaces):
return "".join(map(lambda a: "}", namespaces))
def set_namespace(namespaces):
ns = combine_ns(namespaces)
ns_open = open_namespaces(namespaces)
ns_close = close_namespaces(namespaces)
return [ns, ns_open, ns_close]
def declare_class(hout, name, ns_open, ns_close):
clas_def = ns_open + name + ";" + ns_close
fprintln(hout, "\n", clas_def)
def declear_methods(hout, name, template_param=""):
if config.ns != '':
fprintln(hout, "namespace ", config.ns, " {")
fprintln(hout, Template("""
template <$tmp_param>
struct serializer<$name> {
template <typename Output>
static void write(Output& buf, const $name& v);
template <typename Input>
static $name read(Input& buf);
template <typename Input>
static void skip(Input& buf);
};
""").substitute({'name': name, 'sizetype': SIZETYPE, 'tmp_param': template_param}))
if config.ns != '':
fprintln(hout, "}")
def handle_enum(enum, hout, cout, namespaces, parent_template_param=[]):
[ns, ns_open, ns_close] = set_namespace(namespaces)
temp_def = ", ".join(map(lambda a: a[0] + " " + a[1], parent_template_param)) if parent_template_param else ""
template = "template <" + temp_def + ">" if temp_def else ""
name = enum["name"] if ns == "" else ns + "::" + enum["name"]
declear_methods(hout, name, temp_def)
fprintln(cout, Template("""
$template
template <typename Output>
void serializer<$name>::write(Output& buf, const $name& v) {
serialize(buf, static_cast<$type>(v));
}
$template
template<typename Input>
$name serializer<$name>::read(Input& buf) {
return static_cast<$name>(deserialize(buf, boost::type<$type>()));
}""").substitute({'name': name, 'size_type': SIZETYPE, 'type': enum['underline_type'], 'template': template}))
def join_template(lst):
return "<" + ", ".join([param_type(l) for l in lst]) + ">"
def param_type(lst):
if isinstance(lst, str):
return lst
if len(lst) == 1:
return lst[0]
return lst[0] + join_template(lst[1])
def flat_template(lst):
return ", ".join([param_type(l) for l in lst])
def flat_type(lst):
if isinstance(lst, str):
return lst
if len(lst) == 1:
return flat_type(lst[0])
return (lst[0] + "__" + "_".join([flat_type(l) for l in lst[1]])).replace('::', '__')
def is_class(obj):
return obj["type"] == "class" or obj["type"] == "struct"
def is_enum(obj):
try:
return not isinstance(obj["type"], str) and "".join(obj["type"]) == 'enumclass'
except Exception:
return False
local_types = {}
def list_types(lst):
if isinstance(lst, str):
return [lst]
if len(lst) == 1:
return lst
lt = reduce(lambda a, b: a + b, [list_types(l) for l in lst[1]])
return lt
def list_local_types(lst):
return {l for l in list_types(lst) if l in local_types}
def is_basic_type(lst):
if isinstance(lst, str):
return lst not in local_types
if len(lst) == 1:
return isinstance(lst[0], str) and lst[0] not in local_types
return False
def is_local_type(lst):
return isinstance(lst, str) and lst in local_types or len(lst) == 1 and lst[0] in local_types
def get_template_name(lst):
return lst["template_name"] if not isinstance(lst, str) and len(lst) > 1 else None
def is_vector(lst):
return get_template_name(lst) == "std::vector"
def is_variant(lst):
return get_template_name(lst) == "boost::variant" or get_template_name(lst) == "std::variant"
def is_optional(lst):
return get_template_name(lst) == "std::optional"
created_writers = set()
def get_member_name(name):
return name if not name.endswith('()') else name[:-2]
def get_members(cls):
return [p for p in cls["members"] if not is_class(p) and not is_enum(p)]
def is_final(cls):
return "final" in cls
def get_variant_type(lst):
if is_variant(lst):
return "variant"
return param_type(lst)
def variant_to_member(typ):
return [{"name": get_variant_type(x), "type": x} for x in typ if is_local_type(x) or is_variant(x)]
def variant_info(info, typ):
[cls, namespaces, parent_template_param] = info
return [{"members": variant_to_member(typ)}, namespaces, parent_template_param]
stubs = set()
def is_stub(cls):
return cls in stubs
def handle_visitors_state(info, hout, clases=[]):
[cls, namespaces, parent_template_param] = info
name = "__".join(clases) if clases else cls["name"]
frame = "empty_frame" if "final" in cls else "frame"
fprintln(hout, Template("""
template<typename Output>
struct state_of_$name {
$frame<Output> f;""").substitute({'name': name, 'frame': frame}))
if clases:
local_state = "state_of_" + "__".join(clases[:-1]) + '<Output>'
fprintln(hout, Template(" $name _parent;").substitute({'name': local_state}))
if "final" in cls:
fprintln(hout, Template(" state_of_$name($state parent) : _parent(parent) {}").substitute({'name': name, 'state': local_state}))
fprintln(hout, "};")
members = get_members(cls)
member_class = clases if clases else [cls["name"]]
for param in members:
if is_local_type(param["type"]):
handle_visitors_state(local_types[param_type(param["type"])], hout, member_class + [param["name"]])
if is_variant(param["type"]):
handle_visitors_state(variant_info(info, param["type"][1]), hout, member_class + [param["name"]])
def get_dependency(cls):
members = get_members(cls)
return reduce(lambda a, b: a | b, [list_local_types(m["type"]) for m in members], set())
def optional_add_methods(typ):
res = reindent(4, """
void skip() {
serialize(_out, false);
}""")
if is_basic_type(typ):
added_type = typ
elif is_local_type(typ):
added_type = param_type(typ) + "_view"
else:
print("non supported optional type ", typ)
raise "non supported optional type " + param_type(typ)
res = res + Template(reindent(4, """
void write(const $type& obj) {
serialize(_out, true);
serialize(_out, obj);
}""")).substitute({'type': added_type})
if is_local_type(typ):
res = res + Template(reindent(4, """
writer_of_$type<Output> write() {
serialize(_out, true);
return {_out};
}""")).substitute({'type': param_type(typ)})
return res
def vector_add_method(current, base_state):
typ = current["type"]
res = ""
if is_basic_type(typ[1]):
res = res + Template("""
void add_$name($type t) {
serialize(_out, t);
_count++;
}""").substitute({'type': param_type(typ[1][0]), 'name': current["name"]})
else:
res = res + Template("""
writer_of_$type<Output> add() {
_count++;
return {_out};
}""").substitute({'type': flat_type(typ[1][0]), 'name': current["name"]})
res = res + Template("""
void add(${type} v) {
serialize(_out, v);
_count++;
}""").substitute({'type': param_view_type(typ[1][0])})
return res + Template("""
after_${basestate}__$name<Output> end_$name() && {
_size.set(_out, _count);
return { _out, std::move(_state) };
}
vector_position pos() const {
return vector_position{_out.pos(), _count};
}
void rollback(const vector_position& vp) {
_out.retract(vp.pos);
_count = vp.count;
}""").substitute({'name': current["name"], 'basestate': base_state})
def add_param_writer_basic_type(name, base_state, typ, var_type="", var_index=None, root_node=False):
if isinstance(var_index, Number):
var_index = "uint32_t(" + str(var_index) + ")"
create_variant_state = Template("auto state = state_of_${base_state}__$name<Output> { start_frame(_out), std::move(_state) };").substitute(locals()) if var_index and root_node else ""
set_varient_index = "serialize(_out, " + var_index + ");\n" if var_index is not None else ""
set_command = ("_state.f.end(_out);" if not root_node else "state.f.end(_out);") if var_type is not "" else ""
return_command = "{ _out, std::move(_state._parent) }" if var_type is not "" and not root_node else "{ _out, std::move(_state) }"
allow_fragmented = False
if typ in ['bytes', 'sstring']:
typ += '_view'
allow_fragmented = True
else:
typ = 'const ' + typ + '&'
writer = Template(reindent(4, """
after_${base_state}__$name<Output> write_$name$var_type($typ t) && {
$create_variant_state
$set_varient_index
serialize(_out, t);
$set_command
return $return_command;
}""")).substitute(locals())
if allow_fragmented:
writer += Template(reindent(4, """
template<typename FragmentedBuffer>
GCC6_CONCEPT(requires FragmentRange<FragmentedBuffer>)
after_${base_state}__$name<Output> write_fragmented_$name$var_type(FragmentedBuffer&& fragments) && {
$set_varient_index
serialize_fragmented(_out, std::forward<FragmentedBuffer>(fragments));
$set_command
return $return_command;
}""")).substitute(locals())
return writer
def add_param_writer_object(name, base_state, typ, var_type="", var_index=None, root_node=False):
var_type1 = "_" + var_type if var_type != "" else ""
if isinstance(var_index, Number):
var_index = "uint32_t(" + str(var_index) + ")"
create_variant_state = Template("auto state = state_of_${base_state}__$name<Output> { start_frame(_out), std::move(_state) };").substitute(locals()) if var_index and root_node else ""
set_varient_index = "serialize(_out, " + var_index + ");\n" if var_index is not None else ""
state = "std::move(_state)" if not var_index or not root_node else "std::move(state)"
ret = Template(reindent(4, """
${base_state}__${name}$var_type1<Output> start_${name}$var_type() && {
$create_variant_state
$set_varient_index
return { _out, $state };
}
""")).substitute(locals())
if not is_stub(typ) and is_local_type(typ):
ret += add_param_writer_basic_type(name, base_state, typ, var_type, var_index, root_node)
if is_stub(typ):
set_command = "_state.f.end(_out);" if var_type is not "" else ""
return_command = "{ _out, std::move(_state._parent) }" if var_type is not "" and not root_node else "{ _out, std::move(_state) }"
ret += Template(reindent(4, """
template<typename Serializer>
after_${base_state}__${name}<Output> ${name}$var_type(Serializer&& f) && {
$set_varient_index
f(writer_of_$typ<Output>(_out));
$set_command
return $return_command;
}""")).substitute(locals())
return ret
def add_param_write(current, base_state, vector=False, root_node=False):
typ = current["type"]
res = ""
name = get_member_name(current["name"])
if is_basic_type(typ):
res = res + add_param_writer_basic_type(name, base_state, typ)
elif is_optional(typ):
res = res + Template(reindent(4, """
after_${basestate}__$name<Output> skip_$name() && {
serialize(_out, false);
return { _out, std::move(_state) };
}""")).substitute({'type': param_type(typ), 'name': name, 'basestate': base_state})
if is_basic_type(typ[1][0]):
res = res + add_param_writer_basic_type(name, base_state, typ[1][0], "", "true")
elif is_local_type(typ[1][0]):
res = res + add_param_writer_object(name, base_state[0][1], typ, "", "true")
else:
print("non supported optional type ", type[0][1])
elif is_vector(typ):
set_size = "_size.set(_out, 0);" if vector else "serialize(_out, size_type(0));"
res = res + Template("""
${basestate}__$name<Output> start_$name() && {
return { _out, std::move(_state) };
}
after_${basestate}__$name<Output> skip_$name() && {
$set
return { _out, std::move(_state) };
}
""").substitute({'type': param_type(typ), 'name': name, 'basestate': base_state, 'set': set_size})
elif is_local_type(typ):
res = res + add_param_writer_object(name, base_state, typ)
elif is_variant(typ):
for idx, p in enumerate(typ[1]):
if is_basic_type(p):
varient_type = param_type(p)
res = res + add_param_writer_basic_type(name, base_state, varient_type, "_" + varient_type, idx, root_node)
elif is_variant(p):
res = res + add_param_writer_object(name, base_state, p, '_' + "variant", idx, root_node)
elif is_local_type(p):
res = res + add_param_writer_object(name, base_state, p, '_' + param_type(p), idx, root_node)
else:
print("something is wrong with type", typ)
return res
def get_return_struct(variant_node, clases):
if not variant_node:
return clases
if clases[-2] == "variant":
return clases[:-2]
return clases[:-1]
def add_variant_end_method(base_state, name, clases):
return_struct = "after_" + base_state + '<Output>'
return Template("""
$return_struct end_$name() && {
_state.f.end(_out);
_state._parent.f.end(_out);
return { _out, std::move(_state._parent._parent) };
}
""").substitute({'name': name, 'return_struct': return_struct})
def add_end_method(parents, name, variant_node=False, return_value=True):
if variant_node:
return add_variant_end_method(parents, name, return_value)
base_state = parents + "__" + name
if return_value:
return_struct = "after_" + base_state + '<Output>'
return Template("""
$return_struct end_$name() && {
_state.f.end(_out);
return { _out, std::move(_state._parent) };
}
""").substitute({'name': name, 'return_struct': return_struct})
return Template("""
void end_$name() {
_state.f.end(_out);
}
""").substitute({'name': name, 'basestate': base_state})
def add_vector_placeholder():
return """ place_holder<Output> _size;
size_type _count = 0;"""
def add_node(hout, name, member, base_state, prefix, parents, fun, is_type_vector=False, is_type_final=False):
struct_name = prefix + name
if member and is_type_vector:
vector_placeholder = add_vector_placeholder()
vector_init = "\n , _size(start_place_holder(out))"
else:
vector_placeholder = ""
vector_init = ""
if vector_init != "" or prefix == "":
state_init = "_state{start_frame(out), std::move(state)}" if parents != base_state and not is_type_final else "_state(state)"
else:
if member and is_variant(member) and parents != base_state:
state_init = "_state{start_frame(out), std::move(state)}"
else:
state_init = ""
if prefix == "writer_of_":
constructor = Template("""$name(Output& out)
: _out(out)
, _state{start_frame(out)}${vector_init}
{}""").substitute({'name': struct_name, 'vector_init': vector_init})
elif state_init != "":
constructor = Template("""$name(Output& out, state_of_$state<Output> state)
: _out(out)
, $state_init${vector_init}
{}""").substitute({'name': struct_name, 'vector_init': vector_init, 'state': parents, 'state_init': state_init})
else:
constructor = ""
fprintln(hout, Template("""
template<typename Output>
struct $name {
Output& _out;
state_of_$state<Output> _state;
${vector_placeholder}
${constructor}
$fun
};""").substitute({'name': struct_name, 'state': base_state, 'fun': fun, 'vector_placeholder': vector_placeholder, 'constructor': constructor}))
def add_vector_node(hout, member, base_state, parents):
if get_template_name(member["type"][1][0]):
add_template_writer_node(hout, member["type"][1][0])
add_node(hout, base_state + "__" + member["name"], member["type"], base_state, "", parents, vector_add_method(member, base_state), True)
optional_nodes = set()
def add_optional_node(hout, typ):
global optional_nodes
full_type = flat_type(typ)
if full_type in optional_nodes:
return
optional_nodes.add(full_type)
fprintln(hout, Template(reindent(0, """
template<typename Output>
struct writer_of_$type {
Output& _out;
$add_method
};""")).substitute({'type': full_type, 'add_method': optional_add_methods(typ[1][0])}))
def add_variant_nodes(hout, member, param, base_state, parents, classes):
par = base_state + "__" + member["name"]
for typ in param[1]:
if is_local_type(typ):
handle_visitors_nodes(local_types[param_type(typ)], hout, True, classes + [member["name"], local_types[param_type(typ)][0]["name"]])
if is_variant(typ):
name = base_state + "__" + member["name"] + "__variant"
new_member = {"type": typ, "name": "variant"}
return_struct = "after_" + par
end_method = Template("""
$return_struct<Output> end_variant() && {
_state.f.end(_out);
return { _out, std::move(_state._parent) };
}
""").substitute({'return_struct': return_struct})
add_node(hout, name, None, base_state + "__" + member["name"], "after_", name, end_method)
add_variant_nodes(hout, new_member, typ, par, name, classes + [member["name"]])
add_node(hout, name, typ, name, "", par, add_param_write({"type": typ, "name": "variant"}, par))
writers = set()
def add_template_writer_node(hout, typ):
if is_optional(typ):
add_optional_node(hout, typ)
def add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes):
if is_vector(member["type"]):
add_vector_node(hout, member, base_state_name, base_state_name)
elif is_variant(member["type"]):
add_variant_nodes(hout, member, member["type"], base_state_name, parents, member_classes)
elif is_local_type(member["type"]):
handle_visitors_nodes(local_types[member["type"]], hout, False, member_classes + [member["name"]])
def handle_visitors_nodes(info, hout, variant_node=False, clases=[]):
global writers
[cls, namespaces, parent_template_param] = info
# for root node, only generate once
if not clases:
if cls["name"] in writers:
return
writers.add(cls["name"])
members = get_members(cls)
if clases:
base_state_name = "__".join(clases)
if variant_node:
parents = "__".join(clases[:-1])
else:
parents = "__".join(clases[:-1])
current_name = clases[-1]
else:
base_state_name = cls["name"]
current_name = cls["name"]
parents = ""
member_classes = clases if clases else [current_name]
prefix = "" if clases else "writer_of_"
if not members:
add_node(hout, base_state_name, None, base_state_name, prefix, parents, add_end_method(parents, current_name, variant_node, clases), False, is_final(cls))
return
add_node(hout, base_state_name + "__" + get_member_name(members[-1]["name"]), members[-1]["type"], base_state_name, "after_", base_state_name, add_end_method(parents, current_name, variant_node, clases))
# Create writer and reader for include class
if not variant_node:
for member in get_dependency(cls):
handle_visitors_nodes(local_types[member], hout)
for ind in reversed(range(1, len(members))):
member = members[ind]
add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes)
variant_state = base_state_name + "__" + get_member_name(member["name"]) if is_variant(member["type"]) else base_state_name
add_node(hout, base_state_name + "__" + get_member_name(members[ind - 1]["name"]), member["type"], variant_state, "after_", base_state_name, add_param_write(member, base_state_name), False)
member = members[0]
add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes)
add_node(hout, base_state_name, member["type"], base_state_name, prefix, parents, add_param_write(member, base_state_name, False, not clases), False, is_final(cls))
def add_to_types(cls, namespaces, parent_template_param):
global local_types
global stubs
if "attribute" not in cls or cls["attribute"][0][0] != "writable":
return
local_types[cls["name"]] = [cls, namespaces, parent_template_param]
if "stub" in cls:
stubs.add(cls["name"])
for param in cls["members"]:
if is_class(param) or is_enum(param):
continue
def sort_dependencies():
dep_tree = {}
res = []
for k in local_types:
[cls, namespaces, parent_template_param] = local_types[k]
dep_tree[k] = get_dependency(cls)
while (len(dep_tree) > 0):
found = set()
found = found | {k for k in dep_tree if not dep_tree[k]}
res = res + [k for k in found]
for k in found:
dep_tree.pop(k)
for k in dep_tree:
if dep_tree[k]:
dep_tree[k].difference_update(found)
return res
def join_template_view(lst, more_types=[]):
return "<" + ", ".join([param_view_type(l) for l in lst] + more_types) + ">"
def to_view(val):
if val in local_types:
return val + "_view"
return val
def param_view_type(lst):
if isinstance(lst, str):
return to_view(lst)
if len(lst) == 1:
return to_view(lst[0])
if lst[0] == "boost::variant" or lst[0] == "std::variant":
return lst[0] + join_template_view(lst[1], ["unknown_variant_type"])
return lst[0] + join_template_view(lst[1])
read_sizes = set()
def add_variant_read_size(hout, typ):
global read_sizes
t = param_view_type(typ)
if t in read_sizes:
return
if not is_variant(typ):
return
for p in typ[1]:
if is_variant(p):
add_variant_read_size(hout, p)
read_sizes.add(t)
fprintln(hout, Template("""
template<typename Input>
inline void skip(Input& v, boost::type<${type}>) {
return seastar::with_serialized_stream(v, [] (auto& v) {
size_type ln = deserialize(v, boost::type<size_type>());
v.skip(ln - sizeof(size_type));
});
}""").substitute({'type': t}))
fprintln(hout, Template("""
template<typename Input>
$type deserialize(Input& v, boost::type<${type}>) {
return seastar::with_serialized_stream(v, [] (auto& v) {
auto in = v;
deserialize(in, boost::type<size_type>());
size_type o = deserialize(in, boost::type<size_type>());
""").substitute({'type': t}))
for index, param in enumerate(typ[1]):
fprintln(hout, Template("""
if (o == $ind) {
v.skip(sizeof(size_type)*2);
return $full_type(deserialize(v, boost::type<$type>()));
}""").substitute({'ind': index, 'type': param_view_type(param), 'full_type': t}))
fprintln(hout, ' return ' + t + '(deserialize(v, boost::type<unknown_variant_type>()));\n });\n}')
def add_view(hout, info):
[cls, namespaces, parent_template_param] = info
members = get_members(cls)
for m in members:
add_variant_read_size(hout, m["type"])
fprintln(hout, Template("""struct ${name}_view {
utils::input_stream v;
""").substitute({'name': cls["name"]}))
if not is_stub(cls["name"]) and is_local_type(cls["name"]):
fprintln(hout, Template(reindent(4, """
operator $type() const {
auto in = v;
return deserialize(in, boost::type<$type>());
}
""")).substitute({'type': cls["name"]}))
skip = "" if is_final(cls) else "ser::skip(in, boost::type<size_type>());"
local_names = {}
for m in members:
name = get_member_name(m["name"])
local_names[name] = "this->" + name + "()"
full_type = param_view_type(m["type"])
if "attribute" in m:
deflt = m["default"][0] if "default" in m else param_type(m["type"]) + "()"
if deflt in local_names:
deflt = local_names[deflt]
deser = Template("(in.size()>0) ? $func(in, boost::type<$typ>()) : $default").substitute(
{'func': DESERIALIZER, 'typ': full_type, 'default': deflt})
else:
deser = Template("$func(in, boost::type<$typ>())").substitute({'func': DESERIALIZER, 'typ': full_type})
fprintln(hout, Template(reindent(4, """
auto $name() const {
return seastar::with_serialized_stream(v, [this] (auto& v) -> decltype($func(std::declval<utils::input_stream&>(), boost::type<$type>())) {
auto in = v;
$skip
return $deser;
});
}
""")).substitute({'name': name, 'type': full_type, 'skip': skip, 'deser': deser, 'func': DESERIALIZER}))
skip = skip + Template("\n ser::skip(in, boost::type<${type}>());").substitute({'type': full_type})
fprintln(hout, "};")
skip_impl = "auto& in = v;\n " + skip if is_final(cls) else "v.skip(read_frame_size(v));"
if skip == "":
skip_impl = ""
fprintln(hout, Template("""
template<>
struct serializer<${type}_view> {
template<typename Input>
static ${type}_view read(Input& v) {
return seastar::with_serialized_stream(v, [] (auto& v) {
auto v_start = v;
auto start_size = v.size();
skip(v);
return ${type}_view{v_start.read_substream(start_size - v.size())};
});
}
template<typename Output>
static void write(Output& out, ${type}_view v) {
v.v.copy_to(out);
}
template<typename Input>
static void skip(Input& v) {
return seastar::with_serialized_stream(v, [] (auto& v) {
$skip_impl
});
}
};
""").substitute({'type': param_type(cls["name"]), 'skip': skip, 'skip_impl': skip_impl}))
def add_views(hout):
for k in sort_dependencies():
add_view(hout, local_types[k])
def add_visitors(hout):
if not local_types:
return
add_views(hout)
fprintln(hout, "\n////// State holders")
for k in local_types:
handle_visitors_state(local_types[k], hout)
fprintln(hout, "\n////// Nodes")
for k in sort_dependencies():
handle_visitors_nodes(local_types[k], hout)
def handle_class(cls, hout, cout, namespaces=[], parent_template_param=[]):
add_to_types(cls, namespaces, parent_template_param)
if "stub" in cls:
return
[ns, ns_open, ns_close] = set_namespace(namespaces)
tpl = "template" in cls
template_param_list = (cls["template"][0]["params"].asList() if tpl else [])
template_param = ", ".join(map(lambda a: a[0] + " " + a[1], template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else ""
template = "template <" + template_param + ">" if tpl else ""
template_class_param = "<" + ",".join(map(lambda a: a[1], template_param_list)) + ">" if tpl else ""
temp_def = template_param if template_param != "" else ""
if ns == "":
name = cls["name"]
else:
name = ns + "::" + cls["name"]
full_name = name + template_class_param
for param in cls["members"]:
if is_class(param):
handle_class(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list)
elif is_enum(param):
handle_enum(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list)
declear_methods(hout, name + template_class_param, temp_def)
is_final = "final" in cls
fprintln(cout, Template("""
$template
template <typename Output>
void serializer<$name>::write(Output& buf, const $name& obj) {""").substitute({'func': SERIALIZER, 'name': full_name, 'template': template}))
if not is_final:
fprintln(cout, Template(""" $set_size(buf, obj);""").substitute({'func': SERIALIZER, 'set_size': SETSIZE, 'name': name, 'sizetype': SIZETYPE}))
for param in cls["members"]:
if is_class(param) or is_enum(param):
continue
fprintln(cout, Template(""" static_assert(is_equivalent<decltype(obj.$var), $type>::value, "member value has a wrong type");
$func(buf, obj.$var);""").substitute({'func': SERIALIZER, 'var': param["name"], 'type': param_type(param["type"])}))
fprintln(cout, "}")
fprintln(cout, Template("""
$template
template <typename Input>
$name$temp_param serializer<$name$temp_param>::read(Input& buf) {
return seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'func': DESERIALIZER, 'name': name, 'template': template, 'temp_param': template_class_param}))
if not cls["members"]:
if not is_final:
fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>());
buf.skip(size - sizeof($size_type));""").substitute({'func': DESERIALIZER, 'size_type': SIZETYPE}))
elif not is_final:
fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>());
auto in = buf.read_substream(size - sizeof($size_type));""").substitute({'func': DESERIALIZER, 'size_type': SIZETYPE}))
else:
fprintln(cout, """ auto& in = buf;""")
params = []
local_names = {}
for index, param in enumerate(cls["members"]):
if is_class(param) or is_enum(param):
continue
local_param = "__local_" + str(index)
local_names[param["name"]] = local_param
if "attribute" in param:
deflt = param["default"][0] if "default" in param else param_type(param["type"]) + "()"
if deflt in local_names:
deflt = local_names[deflt]
fprintln(cout, Template(""" auto $local = (in.size()>0) ?
$func(in, boost::type<$typ>()) : $default;""").substitute({'func': DESERIALIZER, 'typ': param_type(param["type"]), 'local': local_param, 'default': deflt}))
else:
fprintln(cout, Template(""" auto $local = $func(in, boost::type<$typ>());""").substitute({'func': DESERIALIZER, 'typ': param_type(param["type"]), 'local': local_param}))
params.append("std::move(" + local_param + ")")
fprintln(cout, Template("""
$name$temp_param res {$params};
return res;
});
}""").substitute({'name': name, 'params': ", ".join(params), 'temp_param': template_class_param}))
fprintln(cout, Template("""
$template
template <typename Input>
void serializer<$name$temp_param>::skip(Input& buf) {
seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'func': DESERIALIZER, 'name': name, 'template': template, 'temp_param': template_class_param}))
if not is_final:
fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>());
buf.skip(size - sizeof($size_type));""").substitute({'func': DESERIALIZER, 'size_type': SIZETYPE}))
else:
for m in get_members(cls):
full_type = param_view_type(m["type"])
fprintln(cout, " ser::skip(buf, boost::type<%s>());" % full_type)
fprintln(cout, """ });\n}""")
def handle_objects(tree, hout, cout, namespaces=[]):
for obj in tree:
if is_class(obj):
handle_class(obj, hout, cout, namespaces)
elif is_enum(obj):
handle_enum(obj, hout, cout, namespaces)
elif obj["type"] == "namespace":
handle_objects(obj["content"], hout, cout, namespaces + [obj["name"]])
else:
print("unknown type ", obj, obj["type"])
def handle_types(tree, namespaces=[]):
for obj in tree:
if is_class(obj):
add_to_types(obj, namespaces, [])
elif is_enum(obj):
pass
elif obj["type"] == "namespace":
handle_types(obj["content"], namespaces + [obj["name"]])
else:
print("unknown type ", obj, obj["type"])
def load_file(name):
if config.o:
cout = open(config.o.replace('.hh', '.impl.hh'), "w+")
hout = open(config.o, "w+")
else:
cout = open(name.replace(EXTENSION, '.dist.impl.hh'), "w+")
hout = open(name.replace(EXTENSION, '.dist.hh'), "w+")
print_cw(hout)
fprintln(hout, """
/*
* The generate code should be included in a header file after
* The object definition
*/
""")
print_cw(cout)
fprintln(hout, "#include \"serializer.hh\"\n")
if config.ns != '':
fprintln(cout, "namespace ", config.ns, " {")
data = parse_file(name)
if data:
handle_types(data)
add_visitors(cout)
if data:
handle_objects(data, hout, cout)
if config.ns != '':
fprintln(cout, "}")
cout.close()
hout.close()
def general_include(files):
name = config.o if config.o else "serializer.dist.hh"
cout = open(name.replace('.hh', '.impl.hh'), "w+")
hout = open(name, "w+")
print_cw(cout)
print_cw(hout)
for n in files:
fprintln(hout, '#include "' + n + '"')
fprintln(cout, '#include "' + n.replace(".dist.hh", '.dist.impl.hh') + '"')
cout.close()
hout.close()
if config.file:
general_include(config.file)
elif config.f != '':
load_file(config.f)
Reference in New Issue View Git Blame Copy Permalink