From 4ab1f7f55d75fd6f87f0a37f6f61bc60d2eb7bda Mon Sep 17 00:00:00 2001 From: Pavel Solodovnikov Date: Thu, 26 Nov 2020 02:22:23 +0300 Subject: [PATCH] idl: Decouple idl-compiler data structures from grammar structure Instead of operating on the raw lists of tokens, transform them into typed structures representation, which makes the code by many orders of magnitude simpler to read, understand and extend. This includes sweeping changes throughout the whole source code of the tool, because almost every function was tightly coupled to the way data was passed down from the parser right to the code generation routines. Tested manually by checking that old generated sources are precisely the same as the new generated sources. Signed-off-by: Pavel Solodovnikov --- idl-compiler.py | 601 +++++++++++++++++++++++++++++++----------------- 1 file changed, 391 insertions(+), 210 deletions(-) diff --git a/idl-compiler.py b/idl-compiler.py index 3570167795..b7266095ff 100755 --- a/idl-compiler.py +++ b/idl-compiler.py @@ -26,6 +26,8 @@ import pyparsing as pp from functools import reduce import textwrap from numbers import Number +from pprint import pformat +from copy import copy EXTENSION = '.idl.hh' READ_BUFF = 'input_buffer' @@ -91,6 +93,182 @@ def print_cw(f): #pragma once """) +class BasicType: + def __init__(self, name): + self.name = name + + def __str__(self): + return f"BasicType(name={self.name})" + + def __repr__(self): + return self.__str__() + + +class TemplateType: + def __init__(self, name, template_parameters): + self.name = name + self.template_parameters = template_parameters + + def __str__(self): + return f"TemplateType(name={self.name}, args={pformat(self.template_parameters)}" + + def __repr__(self): + return self.__str__() + + +class EnumValue: + def __init__(self, name, initializer=None): + self.name = name + self.initializer = initializer + + def __str__(self): + return f"EnumValue(name={self.name}, initializer={self.initializer})" + + def __repr__(self): + return self.__str__() + + +class EnumDef: + def __init__(self, name, underlying_type, members): + self.name = name + self.underlying_type = underlying_type + self.members = members + + def __str__(self): + return f"EnumDef(name={self.name}, underlying_type={self.underlying_type}, members={pformat(self.members)})"; + + def __repr__(self): + return self.__str__() + + +class Attribute: + def __init__(self, name): + self.name = name + + def __str__(self): + return f"[[{self.name}]]" + + def __repr__(self): + return self.__str__() + + +class DataClassMember: + def __init__(self, type, name, attribute=None, default_value=None): + self.type = type + self.name = name + self.attribute = attribute + self.default_value = default_value + + def __str__(self): + return f"DataClassMember(type={self.type}, name={self.name}, attribute={self.attribute}, default_value={self.default_value})" + + def __repr__(self): + return self.__str__() + + +class FunctionClassMember: + def __init__(self, type, name, attribute=None, default_value=None): + self.type = type + self.name = name + self.attribute = attribute + self.default_value = default_value + + def __str__(self): + return f"FunctionClassMember(type={self.type}, name={self.name}, attribute={self.attribute}, default_value={self.default_value})" + + def __repr__(self): + return self.__str__() + + +class ClassTemplateParam: + def __init__(self, typename, name): + self.typename = typename + self.name = name + + def __str__(self): + return f"ClassTemplateParam(typename={self.typename}, name={self.name})" + + def __repr__(self): + return self.__str__() + + +class ClassDef: + def __init__(self, name, members, final, stub, attribute, template_params): + self.name = name + self.members = members + self.final = final + self.stub = stub + self.attribute = attribute + self.template_params = template_params + + def __str__(self): + return f"ClassDef(name={self.name}, members={pformat(self.members)}, final={self.final}, stub={self.stub}, attribute={self.attribute}, template_params={self.template_params})" + + def __repr__(self): + return self.__str__() + + +class NamespaceDef: + def __init__(self, name, members): + self.name = name + self.members = members + + def __str__(self): + return f"NamespaceDef(name={self.name}, members={pformat(self.members)})" + + def __repr__(self): + return self.__str__() + + +def basic_type_parse_action(tokens): + return BasicType(name=tokens[0]) + + +def template_type_parse_action(tokens): + return TemplateType(name=tokens['template_name'], template_parameters=tokens["template_parameters"].asList()) + + +def enum_value_parse_action(tokens): + initializer = None + if len(tokens) == 2: + initializer = tokens[1] + elif len(tokens) == 3: + initializer = '-' + tokens[2] + return EnumValue(name=tokens[0], initializer=initializer) + + +def enum_def_parse_action(tokens): + return EnumDef(name=tokens['name'], underlying_type=tokens['underlying_type'], members=tokens['enum_values'].asList()) + + +def attribute_parse_action(tokens): + return Attribute(name=tokens[0]) + + +def class_member_parse_action(tokens): + member_name = tokens['name'] + attribute = tokens['attribute'][0] if 'attribute' in tokens else None + default = tokens['default'][0] if 'default' in tokens else None + if not isinstance(member_name, str): # accessor function declaration + return FunctionClassMember(type=tokens["type"], name=member_name[0], attribute=attribute, default_value=default) + # data member + return DataClassMember(type=tokens["type"], name=member_name, attribute=attribute, default_value=default) + + +def class_def_parse_action(tokens): + is_final = 'final' in tokens + is_stub = 'stub' in tokens + class_members = tokens['members'].asList() if 'members' in tokens else [] + attribute = tokens['attribute'][0] if 'attribute' in tokens else None + template_params = None + if 'template' in tokens: + template_params = [ClassTemplateParam(typename=tp[0], name=tp[1]) for tp in tokens['template']] + return ClassDef(name=tokens['name'], members=class_members, final=is_final, stub=is_stub, attribute=attribute, template_params=template_params) + + +def namespace_parse_action(tokens): + return NamespaceDef(name=tokens['name'], members=tokens['ns_members'].asList()) + def parse_file(file_name): @@ -102,50 +280,71 @@ def parse_file(file_name): cls = pp.Keyword('class') colon = pp.Literal(":") semi = pp.Literal(";").suppress() - langle = pp.Literal("<") - rangle = pp.Literal(">") - equals = pp.Literal("=") + langle = pp.Literal("<").suppress() + rangle = pp.Literal(">").suppress() + equals = pp.Literal("=").suppress() comma = pp.Literal(",") lparen = pp.Literal("(") rparen = pp.Literal(")") - lbrack = pp.Literal("[") - rbrack = pp.Literal("]") + lbrack = pp.Literal("[").suppress() + rbrack = pp.Literal("]").suppress() mins = pp.Literal("-") struct = pp.Keyword('struct') template = pp.Keyword('template') - final = pp.Keyword('final')("final") - stub = pp.Keyword('stub')("stub") + final = pp.Keyword('final') + stub = pp.Keyword('stub') with_colon = pp.Word(pp.alphanums + "_" + ":") - btype = with_colon - type = pp.Forward() - nestedParens = pp.nestedExpr('<', '>') - tmpl = pp.Group(btype("template_name") + langle.suppress() + pp.Group(pp.delimitedList(type)) + rangle.suppress()) - type << (tmpl | btype) + btype = with_colon.copy() + btype.setParseAction(basic_type_parse_action) + + type = pp.Forward() + + tmpl = with_colon("template_name") + langle.suppress() + pp.Group(pp.delimitedList(type))("template_parameters") + rangle.suppress() + tmpl.setParseAction(template_type_parse_action) + + type <<= tmpl | btype + enum_lit = pp.Keyword('enum') enum_class = pp.Group(enum_lit + cls) ns = pp.Keyword("namespace") - enum_init = equals.suppress() + pp.Optional(mins) + number - enum_value = pp.Group(identifier + pp.Optional(enum_init)) - enum_values = pp.Group(lbrace + pp.delimitedList(enum_value) + pp.Optional(comma) + rbrace) + enum_init = equals + pp.Optional(mins) + number + enum_value = identifier + pp.Optional(enum_init) + enum_value.setParseAction(enum_value_parse_action) + + enum_values = lbrace + pp.delimitedList(enum_value) + pp.Optional(comma).suppress() + rbrace + enum = enum_class.suppress() + identifier("name") + colon.suppress() + identifier("underlying_type") + enum_values("enum_values") + pp.Optional(semi) + enum.setParseAction(enum_def_parse_action) + content = pp.Forward() - member_name = pp.Combine(pp.Group(identifier + pp.Optional(lparen + rparen))) - attrib = pp.Group(lbrack.suppress() + lbrack.suppress() + pp.SkipTo(']') + rbrack.suppress() + rbrack.suppress()) + attrib = lbrack + lbrack + pp.SkipTo(']') + rbrack + rbrack + attrib.setParseAction(attribute_parse_action) opt_attribute = pp.Optional(attrib)("attribute") - namespace = pp.Group(ns("type") + identifier("name") + lbrace + pp.Group(pp.OneOrMore(content))("content") + rbrace) - enum = pp.Group(enum_class("type") + identifier("name") + colon.suppress() + identifier("underline_type") + enum_values("enum_values") + pp.Optional(semi).suppress()) - default_value = equals.suppress() + pp.SkipTo(';') - class_member = pp.Group(type("type") + member_name("name") + opt_attribute + pp.Optional(default_value)("default") + semi.suppress())("member") + + default_value = equals + pp.SkipTo(';') + member_name = pp.Combine(identifier + pp.Optional(lparen + rparen)("function_marker")) + class_member = type("type") + member_name("name") + opt_attribute + pp.Optional(default_value)("default") + semi + class_member.setParseAction(class_member_parse_action) + template_param = pp.Group(identifier("type") + identifier("name")) - template_def = pp.Group(template + langle + pp.Group(pp.delimitedList(template_param))("params") + rangle) + template_def = template.suppress() + langle + pp.delimitedList(template_param)("params") + rangle class_content = pp.Forward() - class_def = pp.Group(pp.Optional(template_def)("template") + (cls | struct)("type") + with_colon("name") + pp.Optional(final) + pp.Optional(stub) + opt_attribute + lbrace + pp.Group(pp.ZeroOrMore(class_content))("members") + rbrace + pp.Optional(semi)) - content << (enum | class_def | namespace) - class_content << (enum | class_def | class_member) - for varname in "enum class_def class_member content namespace template_def".split(): + class_def = pp.Optional(template_def)("template") + (cls | struct).suppress() + with_colon("name") + \ + pp.Optional(final)("final") + pp.Optional(stub)("stub") + opt_attribute + \ + lbrace + pp.ZeroOrMore(class_content)("members") + rbrace + pp.Optional(semi) + class_content <<= enum | class_def | class_member + class_def.setParseAction(class_def_parse_action) + + namespace = ns.suppress() + identifier("name") + lbrace + pp.OneOrMore(content)("ns_members") + rbrace + namespace.setParseAction(namespace_parse_action) + + content <<= enum | class_def | namespace + + for varname in ("enum", "class_def", "class_member", "content", "namespace", "template_def"): locals()[varname].setName(varname) + rt = pp.OneOrMore(content) singleLineComment = "//" + pp.restOfLine rt.ignore(singleLineComment) @@ -192,16 +391,16 @@ struct serializer<$name> { template static void skip(Input& buf); }; -""").substitute({'name': name, 'sizetype': SIZETYPE, 'tmp_param': template_param})) +""").substitute({'name': name, 'tmp_param': template_param})) if config.ns != '': fprintln(hout, "}") def handle_enum(enum, hout, cout, namespaces, parent_template_param=[]): [ns, ns_open, ns_close] = set_namespace(namespaces) - temp_def = ", ".join(map(lambda a: a[0] + " " + a[1], parent_template_param)) if parent_template_param else "" + temp_def = ", ".join(map(lambda a: a.typename + " " + a.name, parent_template_param)) if parent_template_param else "" template = "template <" + temp_def + ">" if temp_def else "" - name = enum["name"] if ns == "" else ns + "::" + enum["name"] + name = enum.name if ns == "" else ns + "::" + enum.name declear_methods(hout, name, temp_def) fprintln(cout, Template(""" $template @@ -214,86 +413,69 @@ $template template $name serializer<$name>::read(Input& buf) { return static_cast<$name>(deserialize(buf, boost::type<$type>())); -}""").substitute({'name': name, 'size_type': SIZETYPE, 'type': enum['underline_type'], 'template': template})) +}""").substitute({'name': name, 'type': enum.underlying_type, 'template': template})) -def join_template(lst): - return "<" + ", ".join([param_type(l) for l in lst]) + ">" +def join_template(template_params): + return "<" + ", ".join([param_type(p) for p in template_params]) + ">" -def param_type(lst): - if isinstance(lst, str): - return lst - if len(lst) == 1: - return lst[0] - return lst[0] + join_template(lst[1]) +def param_type(t): + if isinstance(t, BasicType): + return t.name + elif isinstance(t, TemplateType): + return t.name + join_template(t.template_parameters) def flat_template(lst): return ", ".join([param_type(l) for l in lst]) -def flat_type(lst): - if isinstance(lst, str): - return lst - if len(lst) == 1: - return flat_type(lst[0]) - return (lst[0] + "__" + "_".join([flat_type(l) for l in lst[1]])).replace('::', '__') - - -def is_class(obj): - return obj["type"] == "class" or obj["type"] == "struct" - - -def is_enum(obj): - try: - return not isinstance(obj["type"], str) and "".join(obj["type"]) == 'enumclass' - except Exception: - return False +def flat_type(t): + if isinstance(t, BasicType): + return t.name + elif isinstance(t, TemplateType): + return (t.name + "__" + "_".join([flat_type(p) for p in t.template_parameters])).replace('::', '__') local_types = {} -def list_types(lst): - if isinstance(lst, str): - return [lst] - if len(lst) == 1: - return lst - lt = reduce(lambda a, b: a + b, [list_types(l) for l in lst[1]]) - return lt +def list_types(t): + if isinstance(t, BasicType): + return [t.name] + elif isinstance(t, TemplateType): + return reduce(lambda a, b: a + b, [list_types(p) for p in t.template_parameters]) -def list_local_types(lst): - return {l for l in list_types(lst) if l in local_types} +def list_local_types(t): + return {l for l in list_types(t) if l in local_types} -def is_basic_type(lst): - if isinstance(lst, str): - return lst not in local_types - if len(lst) == 1: - return isinstance(lst[0], str) and lst[0] not in local_types - return False +def is_basic_type(t): + return isinstance(t, BasicType) and t.name not in local_types -def is_local_type(lst): - return isinstance(lst, str) and lst in local_types or len(lst) == 1 and lst[0] in local_types +def is_local_type(t): + if isinstance(t, str): # e.g. `t` is a local class name + return t in local_types + return t.name in local_types def get_template_name(lst): return lst["template_name"] if not isinstance(lst, str) and len(lst) > 1 else None -def is_vector(lst): - return get_template_name(lst) == "std::vector" +def is_vector(t): + return isinstance(t, TemplateType) and t.name == "std::vector" -def is_variant(lst): - return get_template_name(lst) == "boost::variant" or get_template_name(lst) == "std::variant" +def is_variant(t): + return isinstance(t, TemplateType) and (t.name == "boost::variant" or t.name == "std::variant") -def is_optional(lst): - return get_template_name(lst) == "std::optional" +def is_optional(t): + return isinstance(t, TemplateType) and t.name == "std::optional" created_writers = set() @@ -304,26 +486,28 @@ def get_member_name(name): def get_members(cls): - return [p for p in cls["members"] if not is_class(p) and not is_enum(p)] + return [p for p in cls.members if not isinstance(p, ClassDef) and not isinstance(p, EnumDef)] def is_final(cls): - return "final" in cls + return cls.final -def get_variant_type(lst): - if is_variant(lst): +def get_variant_type(t): + if is_variant(t): return "variant" - return param_type(lst) + return param_type(t) -def variant_to_member(typ): - return [{"name": get_variant_type(x), "type": x} for x in typ if is_local_type(x) or is_variant(x)] +def variant_to_member(template_parameters): + return [DataClassMember(name=get_variant_type(x), type=x) for x in template_parameters if is_local_type(x) or is_variant(x)] -def variant_info(info, typ): +def variant_info(info, template_parameters): [cls, namespaces, parent_template_param] = info - return [{"members": variant_to_member(typ)}, namespaces, parent_template_param] + variant_info_cls = copy(cls) # shallow copy of cls + variant_info_cls.members = variant_to_member(template_parameters) + return [variant_info_cls, namespaces, parent_template_param] stubs = set() @@ -335,8 +519,8 @@ def is_stub(cls): def handle_visitors_state(info, hout, clases=[]): [cls, namespaces, parent_template_param] = info - name = "__".join(clases) if clases else cls["name"] - frame = "empty_frame" if "final" in cls else "frame" + name = "__".join(clases) if clases else cls.name + frame = "empty_frame" if cls.final else "frame" fprintln(hout, Template(""" template struct state_of_$name { @@ -344,21 +528,21 @@ struct state_of_$name { if clases: local_state = "state_of_" + "__".join(clases[:-1]) + '' fprintln(hout, Template(" $name _parent;").substitute({'name': local_state})) - if "final" in cls: + if cls.final: fprintln(hout, Template(" state_of_$name($state parent) : _parent(parent) {}").substitute({'name': name, 'state': local_state})) fprintln(hout, "};") members = get_members(cls) - member_class = clases if clases else [cls["name"]] - for param in members: - if is_local_type(param["type"]): - handle_visitors_state(local_types[param_type(param["type"])], hout, member_class + [param["name"]]) - if is_variant(param["type"]): - handle_visitors_state(variant_info(info, param["type"][1]), hout, member_class + [param["name"]]) + member_class = clases if clases else [cls.name] + for m in members: + if is_local_type(m.type): + handle_visitors_state(local_types[param_type(m.type)], hout, member_class + [m.name]) + if is_variant(m.type): + handle_visitors_state(variant_info(info, m.type.template_parameters), hout, member_class + [m.name]) def get_dependency(cls): members = get_members(cls) - return reduce(lambda a, b: a | b, [list_local_types(m["type"]) for m in members], set()) + return reduce(lambda a, b: a | b, [list_local_types(m.type) for m in members], set()) def optional_add_methods(typ): @@ -388,25 +572,25 @@ def optional_add_methods(typ): def vector_add_method(current, base_state): - typ = current["type"] + typ = current.type res = "" - if is_basic_type(typ[1]): + if is_basic_type(typ.template_parameters[0]): res = res + Template(""" void add_$name($type t) { serialize(_out, t); _count++; - }""").substitute({'type': param_type(typ[1][0]), 'name': current["name"]}) + }""").substitute({'type': param_type(typ.template_parameters[0]), 'name': current.name}) else: res = res + Template(""" writer_of_$type add() { _count++; return {_out}; - }""").substitute({'type': flat_type(typ[1][0]), 'name': current["name"]}) + }""").substitute({'type': flat_type(typ.template_parameters[0]), 'name': current.name}) res = res + Template(""" void add(${type} v) { serialize(_out, v); _count++; - }""").substitute({'type': param_view_type(typ[1][0])}) + }""").substitute({'type': param_view_type(typ.template_parameters[0])}) return res + Template(""" after_${basestate}__$name end_$name() && { _size.set(_out, _count); @@ -420,7 +604,7 @@ def vector_add_method(current, base_state): void rollback(const vector_position& vp) { _out.retract(vp.pos); _count = vp.count; - }""").substitute({'name': current["name"], 'basestate': base_state}) + }""").substitute({'name': current.name, 'basestate': base_state}) def add_param_writer_basic_type(name, base_state, typ, var_type="", var_index=None, root_node=False): @@ -432,14 +616,14 @@ def add_param_writer_basic_type(name, base_state, typ, var_type="", var_index=No return_command = "{ _out, std::move(_state._parent) }" if var_type != "" and not root_node else "{ _out, std::move(_state) }" allow_fragmented = False - if typ in ['bytes', 'sstring']: - typ += '_view' + if typ.name in ['bytes', 'sstring']: + typename = typ.name + '_view' allow_fragmented = True else: - typ = 'const ' + typ + '&' + typename = 'const ' + typ.name + '&' writer = Template(reindent(4, """ - after_${base_state}__$name write_$name$var_type($typ t) && { + after_${base_state}__$name write_$name$var_type($typename t) && { $create_variant_state $set_varient_index serialize(_out, t); @@ -473,16 +657,17 @@ def add_param_writer_object(name, base_state, typ, var_type="", var_index=None, return { _out, $state }; } """)).substitute(locals()) - if not is_stub(typ) and is_local_type(typ): + if not is_stub(typ.name) and is_local_type(typ): ret += add_param_writer_basic_type(name, base_state, typ, var_type, var_index, root_node) - if is_stub(typ): + if is_stub(typ.name): + typename = typ.name set_command = "_state.f.end(_out);" if var_type != "" else "" return_command = "{ _out, std::move(_state._parent) }" if var_type != "" and not root_node else "{ _out, std::move(_state) }" ret += Template(reindent(4, """ template after_${base_state}__${name} ${name}$var_type(Serializer&& f) && { $set_varient_index - f(writer_of_$typ(_out)); + f(writer_of_$typename(_out)); $set_command return $return_command; }""")).substitute(locals()) @@ -490,9 +675,9 @@ def add_param_writer_object(name, base_state, typ, var_type="", var_index=None, def add_param_write(current, base_state, vector=False, root_node=False): - typ = current["type"] + typ = current.type res = "" - name = get_member_name(current["name"]) + name = get_member_name(current.name) if is_basic_type(typ): res = res + add_param_writer_basic_type(name, base_state, typ) elif is_optional(typ): @@ -501,12 +686,12 @@ def add_param_write(current, base_state, vector=False, root_node=False): serialize(_out, false); return { _out, std::move(_state) }; }""")).substitute({'type': param_type(typ), 'name': name, 'basestate': base_state}) - if is_basic_type(typ[1][0]): - res = res + add_param_writer_basic_type(name, base_state, typ[1][0], "", "true") - elif is_local_type(typ[1][0]): + if is_basic_type(typ.template_parameters[0]): + res = res + add_param_writer_basic_type(name, base_state, typ.template_parameters[0], "", "true") + elif is_local_type(typ.template_parameters[0]): res = res + add_param_writer_object(name, base_state[0][1], typ, "", "true") else: - print("non supported optional type ", type[0][1]) + print("non supported optional type ", typ.template_parameters[0]) elif is_vector(typ): set_size = "_size.set(_out, 0);" if vector else "serialize(_out, size_type(0));" @@ -523,10 +708,9 @@ def add_param_write(current, base_state, vector=False, root_node=False): elif is_local_type(typ): res = res + add_param_writer_object(name, base_state, typ) elif is_variant(typ): - for idx, p in enumerate(typ[1]): + for idx, p in enumerate(typ.template_parameters): if is_basic_type(p): - varient_type = param_type(p) - res = res + add_param_writer_basic_type(name, base_state, varient_type, "_" + varient_type, idx, root_node) + res = res + add_param_writer_basic_type(name, base_state, p, "_" + param_type(p), idx, root_node) elif is_variant(p): res = res + add_param_writer_object(name, base_state, p, '_' + "variant", idx, root_node) elif is_local_type(p): @@ -619,9 +803,9 @@ struct $name { def add_vector_node(hout, member, base_state, parents): - if get_template_name(member["type"][1][0]): - add_template_writer_node(hout, member["type"][1][0]) - add_node(hout, base_state + "__" + member["name"], member["type"], base_state, "", parents, vector_add_method(member, base_state), True) + if member.type.template_parameters[0].name: + add_template_writer_node(hout, member.type.template_parameters[0]) + add_node(hout, base_state + "__" + member.name, member.type, base_state, "", parents, vector_add_method(member, base_state), True) optional_nodes = set() @@ -638,17 +822,19 @@ template struct writer_of_$type { Output& _out; $add_method -};""")).substitute({'type': full_type, 'add_method': optional_add_methods(typ[1][0])})) +};""")).substitute({'type': full_type, 'add_method': optional_add_methods(typ.template_parameters[0])})) def add_variant_nodes(hout, member, param, base_state, parents, classes): - par = base_state + "__" + member["name"] - for typ in param[1]: + par = base_state + "__" + member.name + for typ in param.template_parameters: if is_local_type(typ): - handle_visitors_nodes(local_types[param_type(typ)], hout, True, classes + [member["name"], local_types[param_type(typ)][0]["name"]]) + handle_visitors_nodes(local_types[param_type(typ)], hout, True, classes + [member.name, local_types[param_type(typ)][0].name]) if is_variant(typ): - name = base_state + "__" + member["name"] + "__variant" - new_member = {"type": typ, "name": "variant"} + name = base_state + "__" + member.name + "__variant" + new_member = copy(member) # shallow copy + new_member.type = typ + new_member.name = "variant" return_struct = "after_" + par end_method = Template(""" $return_struct end_variant() && { @@ -656,9 +842,9 @@ def add_variant_nodes(hout, member, param, base_state, parents, classes): return { _out, std::move(_state._parent) }; } """).substitute({'return_struct': return_struct}) - add_node(hout, name, None, base_state + "__" + member["name"], "after_", name, end_method) - add_variant_nodes(hout, new_member, typ, par, name, classes + [member["name"]]) - add_node(hout, name, typ, name, "", par, add_param_write({"type": typ, "name": "variant"}, par)) + add_node(hout, name, None, base_state + "__" + member.name, "after_", name, end_method) + add_variant_nodes(hout, new_member, typ, par, name, classes + [member.name]) + add_node(hout, name, typ, name, "", par, add_param_write(new_member, par)) writers = set() @@ -670,12 +856,12 @@ def add_template_writer_node(hout, typ): def add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes): - if is_vector(member["type"]): + if is_vector(member.type): add_vector_node(hout, member, base_state_name, base_state_name) - elif is_variant(member["type"]): - add_variant_nodes(hout, member, member["type"], base_state_name, parents, member_classes) - elif is_local_type(member["type"]): - handle_visitors_nodes(local_types[member["type"]], hout, False, member_classes + [member["name"]]) + elif is_variant(member.type): + add_variant_nodes(hout, member, member.type, base_state_name, parents, member_classes) + elif is_local_type(member.type): + handle_visitors_nodes(local_types[member.type.name], hout, False, member_classes + [member.name]) def handle_visitors_nodes(info, hout, variant_node=False, clases=[]): @@ -683,9 +869,9 @@ def handle_visitors_nodes(info, hout, variant_node=False, clases=[]): [cls, namespaces, parent_template_param] = info # for root node, only generate once if not clases: - if cls["name"] in writers: + if cls.name in writers: return - writers.add(cls["name"]) + writers.add(cls.name) members = get_members(cls) if clases: @@ -696,15 +882,15 @@ def handle_visitors_nodes(info, hout, variant_node=False, clases=[]): parents = "__".join(clases[:-1]) current_name = clases[-1] else: - base_state_name = cls["name"] - current_name = cls["name"] + base_state_name = cls.name + current_name = cls.name parents = "" member_classes = clases if clases else [current_name] prefix = "" if clases else "writer_of_" if not members: add_node(hout, base_state_name, None, base_state_name, prefix, parents, add_end_method(parents, current_name, variant_node, clases), False, is_final(cls)) return - add_node(hout, base_state_name + "__" + get_member_name(members[-1]["name"]), members[-1]["type"], base_state_name, "after_", base_state_name, add_end_method(parents, current_name, variant_node, clases)) + add_node(hout, base_state_name + "__" + get_member_name(members[-1].name), members[-1].type, base_state_name, "after_", base_state_name, add_end_method(parents, current_name, variant_node, clases)) # Create writer and reader for include class if not variant_node: for member in get_dependency(cls): @@ -712,24 +898,21 @@ def handle_visitors_nodes(info, hout, variant_node=False, clases=[]): for ind in reversed(range(1, len(members))): member = members[ind] add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes) - variant_state = base_state_name + "__" + get_member_name(member["name"]) if is_variant(member["type"]) else base_state_name - add_node(hout, base_state_name + "__" + get_member_name(members[ind - 1]["name"]), member["type"], variant_state, "after_", base_state_name, add_param_write(member, base_state_name), False) + variant_state = base_state_name + "__" + get_member_name(member.name) if is_variant(member.type) else base_state_name + add_node(hout, base_state_name + "__" + get_member_name(members[ind - 1].name), member.type, variant_state, "after_", base_state_name, add_param_write(member, base_state_name), False) member = members[0] add_nodes_when_needed(hout, info, member, base_state_name, parents, member_classes) - add_node(hout, base_state_name, member["type"], base_state_name, prefix, parents, add_param_write(member, base_state_name, False, not clases), False, is_final(cls)) + add_node(hout, base_state_name, member.type, base_state_name, prefix, parents, add_param_write(member, base_state_name, False, not clases), False, is_final(cls)) def add_to_types(cls, namespaces, parent_template_param): global local_types global stubs - if "attribute" not in cls or cls["attribute"][0][0] != "writable": + if not cls.attribute or cls.attribute.name != 'writable': return - local_types[cls["name"]] = [cls, namespaces, parent_template_param] - if "stub" in cls: - stubs.add(cls["name"]) - for param in cls["members"]: - if is_class(param) or is_enum(param): - continue + local_types[cls.name] = [cls, namespaces, parent_template_param] + if cls.stub: + stubs.add(cls.name) def sort_dependencies(): @@ -760,14 +943,14 @@ def to_view(val): return val -def param_view_type(lst): - if isinstance(lst, str): - return to_view(lst) - if len(lst) == 1: - return to_view(lst[0]) - if lst[0] == "boost::variant" or lst[0] == "std::variant": - return lst[0] + join_template_view(lst[1], ["unknown_variant_type"]) - return lst[0] + join_template_view(lst[1]) +def param_view_type(t): + if isinstance(t, BasicType): + return to_view(t.name) + elif isinstance(t, TemplateType): + additional_types = [] + if t.name == "boost::variant" or t.name == "std::variant": + additional_types.append("unknown_variant_type") + return t.name + join_template_view(t.template_parameters, additional_types) read_sizes = set() @@ -780,7 +963,7 @@ def add_variant_read_size(hout, typ): return if not is_variant(typ): return - for p in typ[1]: + for p in typ.template_parameters: if is_variant(p): add_variant_read_size(hout, p) read_sizes.add(t) @@ -801,7 +984,7 @@ $type deserialize(Input& v, boost::type<${type}>) { deserialize(in, boost::type()); size_type o = deserialize(in, boost::type()); """).substitute({'type': t})) - for index, param in enumerate(typ[1]): + for index, param in enumerate(typ.template_parameters): fprintln(hout, Template(""" if (o == $ind) { v.skip(sizeof(size_type)*2); @@ -814,28 +997,28 @@ def add_view(hout, info): [cls, namespaces, parent_template_param] = info members = get_members(cls) for m in members: - add_variant_read_size(hout, m["type"]) + add_variant_read_size(hout, m.type) fprintln(hout, Template("""struct ${name}_view { utils::input_stream v; - """).substitute({'name': cls["name"]})) + """).substitute({'name': cls.name})) - if not is_stub(cls["name"]) and is_local_type(cls["name"]): + if not is_stub(cls.name) and is_local_type(cls.name): fprintln(hout, Template(reindent(4, """ operator $type() const { auto in = v; return deserialize(in, boost::type<$type>()); } - """)).substitute({'type': cls["name"]})) + """)).substitute({'type': cls.name})) skip = "" if is_final(cls) else "ser::skip(in, boost::type());" local_names = {} for m in members: - name = get_member_name(m["name"]) + name = get_member_name(m.name) local_names[name] = "this->" + name + "()" - full_type = param_view_type(m["type"]) - if "attribute" in m: - deflt = m["default"][0] if "default" in m else param_type(m["type"]) + "()" + full_type = param_view_type(m.type) + if m.attribute: + deflt = m.default_value if m.default_value else param_type(m.type) + "()" if deflt in local_names: deflt = local_names[deflt] deser = Template("(in.size()>0) ? $func(in, boost::type<$typ>()) : $default").substitute( @@ -883,7 +1066,7 @@ struct serializer<${type}_view> { }); } }; -""").substitute({'type': param_type(cls["name"]), 'skip': skip, 'skip_impl': skip_impl})) +""").substitute({'type': cls.name, 'skip': skip, 'skip_impl': skip_impl})) def add_views(hout): @@ -905,28 +1088,25 @@ def add_visitors(hout): def handle_class(cls, hout, cout, namespaces=[], parent_template_param=[]): add_to_types(cls, namespaces, parent_template_param) - if "stub" in cls: + if cls.stub: return [ns, ns_open, ns_close] = set_namespace(namespaces) - tpl = "template" in cls - template_param_list = (cls["template"][0]["params"].asList() if tpl else []) - template_param = ", ".join(map(lambda a: a[0] + " " + a[1], template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else "" - template = "template <" + template_param + ">" if tpl else "" - template_class_param = "<" + ",".join(map(lambda a: a[1], template_param_list)) + ">" if tpl else "" + is_tpl = cls.template_params is not None + template_param_list = (cls.template_params if is_tpl else []) + template_param = ", ".join(map(lambda a: a.typename + " " + a.name, template_param_list + parent_template_param)) if (template_param_list + parent_template_param) else "" + template = "template <" + template_param + ">" if is_tpl else "" + template_class_param = "<" + ",".join(map(lambda a: a.name, template_param_list)) + ">" if is_tpl else "" temp_def = template_param if template_param != "" else "" - if ns == "": - name = cls["name"] - else: - name = ns + "::" + cls["name"] + name = cls.name if ns == "" else ns + "::" + cls.name full_name = name + template_class_param - for param in cls["members"]: - if is_class(param): - handle_class(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list) - elif is_enum(param): - handle_enum(param, hout, cout, namespaces + [cls["name"] + template_class_param], parent_template_param + template_param_list) + for member in cls.members: + if isinstance(member, ClassDef): + handle_class(member, hout, cout, namespaces + [cls.name + template_class_param], parent_template_param + template_param_list) + elif isinstance(member, EnumDef): + handle_enum(member, hout, cout, namespaces + [cls.name + template_class_param], parent_template_param + template_param_list) declear_methods(hout, name + template_class_param, temp_def) - is_final = "final" in cls + is_final = cls.final fprintln(cout, Template(""" $template @@ -934,11 +1114,11 @@ template void serializer<$name>::write(Output& buf, const $name& obj) {""").substitute({'func': SERIALIZER, 'name': full_name, 'template': template})) if not is_final: fprintln(cout, Template(""" $set_size(buf, obj);""").substitute({'func': SERIALIZER, 'set_size': SETSIZE, 'name': name, 'sizetype': SIZETYPE})) - for param in cls["members"]: - if is_class(param) or is_enum(param): + for member in cls.members: + if isinstance(member, ClassDef) or isinstance(member, EnumDef): continue fprintln(cout, Template(""" static_assert(is_equivalent::value, "member value has a wrong type"); - $func(buf, obj.$var);""").substitute({'func': SERIALIZER, 'var': param["name"], 'type': param_type(param["type"])})) + $func(buf, obj.$var);""").substitute({'func': SERIALIZER, 'var': member.name, 'type': param_type(member.type)})) fprintln(cout, "}") fprintln(cout, Template(""" @@ -946,7 +1126,7 @@ $template template $name$temp_param serializer<$name$temp_param>::read(Input& buf) { return seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'func': DESERIALIZER, 'name': name, 'template': template, 'temp_param': template_class_param})) - if not cls["members"]: + if not cls.members: if not is_final: fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>()); buf.skip(size - sizeof($size_type));""").substitute({'func': DESERIALIZER, 'size_type': SIZETYPE})) @@ -957,19 +1137,21 @@ $name$temp_param serializer<$name$temp_param>::read(Input& buf) { fprintln(cout, """ auto& in = buf;""") params = [] local_names = {} - for index, param in enumerate(cls["members"]): - if is_class(param) or is_enum(param): + for index, param in enumerate(cls.members): + if isinstance(param, ClassDef) or isinstance(param, EnumDef): continue local_param = "__local_" + str(index) - local_names[param["name"]] = local_param - if "attribute" in param: - deflt = param["default"][0] if "default" in param else param_type(param["type"]) + "()" + local_names[param.name] = local_param + if param.attribute: + deflt = param_type(param.type) + "()" + if param.default_value: + deflt = param.default_value if deflt in local_names: deflt = local_names[deflt] fprintln(cout, Template(""" auto $local = (in.size()>0) ? - $func(in, boost::type<$typ>()) : $default;""").substitute({'func': DESERIALIZER, 'typ': param_type(param["type"]), 'local': local_param, 'default': deflt})) + $func(in, boost::type<$typ>()) : $default;""").substitute({'func': DESERIALIZER, 'typ': param_type(param.type), 'local': local_param, 'default': deflt})) else: - fprintln(cout, Template(""" auto $local = $func(in, boost::type<$typ>());""").substitute({'func': DESERIALIZER, 'typ': param_type(param["type"]), 'local': local_param})) + fprintln(cout, Template(""" auto $local = $func(in, boost::type<$typ>());""").substitute({'func': DESERIALIZER, 'typ': param_type(param.type), 'local': local_param})) params.append("std::move(" + local_param + ")") fprintln(cout, Template(""" $name$temp_param res {$params}; @@ -981,39 +1163,39 @@ $name$temp_param serializer<$name$temp_param>::read(Input& buf) { $template template void serializer<$name$temp_param>::skip(Input& buf) { - seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'func': DESERIALIZER, 'name': name, 'template': template, 'temp_param': template_class_param})) + seastar::with_serialized_stream(buf, [] (auto& buf) {""").substitute({'name': name, 'template': template, 'temp_param': template_class_param})) if not is_final: fprintln(cout, Template(""" $size_type size = $func(buf, boost::type<$size_type>()); buf.skip(size - sizeof($size_type));""").substitute({'func': DESERIALIZER, 'size_type': SIZETYPE})) else: for m in get_members(cls): - full_type = param_view_type(m["type"]) + full_type = param_view_type(m.type) fprintln(cout, " ser::skip(buf, boost::type<%s>());" % full_type) fprintln(cout, """ });\n}""") def handle_objects(tree, hout, cout, namespaces=[]): for obj in tree: - if is_class(obj): + if isinstance(obj, ClassDef): handle_class(obj, hout, cout, namespaces) - elif is_enum(obj): + elif isinstance(obj, EnumDef): handle_enum(obj, hout, cout, namespaces) - elif obj["type"] == "namespace": - handle_objects(obj["content"], hout, cout, namespaces + [obj["name"]]) + elif isinstance(obj, NamespaceDef): + handle_objects(obj.members, hout, cout, namespaces + [obj.name]) else: - print("unknown type ", obj, obj["type"]) + print(f"Unknown type: {obj}") def handle_types(tree, namespaces=[]): for obj in tree: - if is_class(obj): + if isinstance(obj, ClassDef): add_to_types(obj, namespaces, []) - elif is_enum(obj): + elif isinstance(obj, EnumDef): pass - elif obj["type"] == "namespace": - handle_types(obj["content"], namespaces + [obj["name"]]) + elif isinstance(obj, NamespaceDef): + handle_types(obj.members, namespaces + [obj.name]) else: - print("unknown type ", obj, obj["type"]) + print(f"Unknown object type: {obj}") def load_file(name): @@ -1037,7 +1219,6 @@ def load_file(name): data = parse_file(name) if data: handle_types(data) - if data: handle_objects(data, hout, cout) add_visitors(cout) if config.ns != '':