refactor proto-structs: prettify IO and bundles

1. rename `forward_declarations.inc.jinja` -\> `types_forward_declarations.inc.jinja`
2. `VanillaType` -\> `VanillaTypeDeclaration`
3. `VanillaClass` -\> `VanillaClassDeclaration`
4. don't use `trim` in jinja templates
5. not append includes that are in bundles
commit_hash:471f9699744675f724988f08431957920e9a342b

Author: ArkadyRudenko

.mapping.json

@@ -2592,8 +2592,8 @@
   "libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/not_recommended_field_names/basic.proto",
   "libraries/proto-structs/codegen-tests/proto/oneof/basic.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof/basic.proto",
   "libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/oneof/proto2.proto",
+  "libraries/proto-structs/codegen-tests/proto/simple/base.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/base.proto",
   "libraries/proto-structs/codegen-tests/proto/simple/cycles.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/cycles.proto",
-  "libraries/proto-structs/codegen-tests/proto/simple/file1.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/file1.proto",
   "libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/separate_enum.proto",
   "libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/proto/simple/subdirectory/subdirectory.proto",
   "libraries/proto-structs/codegen-tests/src/cycles_test.cpp":"taxi/uservices/userver/libraries/proto-structs/codegen-tests/src/cycles_test.cpp",
@@ -4268,6 +4268,7 @@
   "scripts/proto_structs/models/__init__.py":"taxi/uservices/userver/scripts/proto_structs/models/__init__.py",
   "scripts/proto_structs/models/gen_node.py":"taxi/uservices/userver/scripts/proto_structs/models/gen_node.py",
   "scripts/proto_structs/models/includes.py":"taxi/uservices/userver/scripts/proto_structs/models/includes.py",
+  "scripts/proto_structs/models/includes_bundles.py":"taxi/uservices/userver/scripts/proto_structs/models/includes_bundles.py",
   "scripts/proto_structs/models/io.py":"taxi/uservices/userver/scripts/proto_structs/models/io.py",
   "scripts/proto_structs/models/names.py":"taxi/uservices/userver/scripts/proto_structs/models/names.py",
   "scripts/proto_structs/models/options.py":"taxi/uservices/userver/scripts/proto_structs/models/options.py",
@@ -4276,12 +4277,12 @@
   "scripts/proto_structs/models/type_ref.py":"taxi/uservices/userver/scripts/proto_structs/models/type_ref.py",
   "scripts/proto_structs/models/type_ref_consts.py":"taxi/uservices/userver/scripts/proto_structs/models/type_ref_consts.py",
   "scripts/proto_structs/models/vanilla.py":"taxi/uservices/userver/scripts/proto_structs/models/vanilla.py",
-  "scripts/proto_structs/templates/forward_declarations.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/forward_declarations.inc.jinja",
   "scripts/proto_structs/templates/io.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/io.inc.jinja",
   "scripts/proto_structs/templates/io_forward_declarations.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/io_forward_declarations.inc.jinja",
   "scripts/proto_structs/templates/structs.usrv.cpp.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/structs.usrv.cpp.jinja",
   "scripts/proto_structs/templates/structs.usrv.hpp.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/structs.usrv.hpp.jinja",
   "scripts/proto_structs/templates/types.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/types.inc.jinja",
+  "scripts/proto_structs/templates/types_forward_declarations.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/types_forward_declarations.inc.jinja",
   "scripts/proto_structs/templates/utils.inc.jinja":"taxi/uservices/userver/scripts/proto_structs/templates/utils.inc.jinja",
   "scripts/rabbitmq/ubuntu_install_rabbitmq_dev.sh":"taxi/uservices/userver/scripts/rabbitmq/ubuntu_install_rabbitmq_dev.sh",
   "scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh":"taxi/uservices/userver/scripts/rabbitmq/ubuntu_install_rabbitmq_server.sh",

libraries/proto-structs/codegen-tests/proto/simple/file1.proto renamed to libraries/proto-structs/codegen-tests/proto/simple/base.proto

@@ -4,8 +4,8 @@
 
 #include <userver/proto-structs/convert.hpp>
 
-#include <simple/file1.pb.h>
-#include <simple/file1.structs.usrv.pb.hpp>
+#include <simple/base.pb.h>
+#include <simple/base.structs.usrv.pb.hpp>
 
 namespace ss = simple::structs;
 

libraries/proto-structs/codegen-tests/src/simple_test.cpp

@@ -80,6 +80,8 @@ def __init__(self, *, proto_relative_path: pathlib.Path, children: Sequence[Code
     def collect_includes(self) -> Iterable[includes.Include]:
         for child in self.children:
             yield from child.collect_includes()
+        yield includes.Include(path='cstddef', kind=includes.IncludeKind.FOR_HPP)
+        yield includes.Include(path='utility', kind=includes.IncludeKind.FOR_HPP)
 
     def gen_path(self, *, ext: str) -> pathlib.Path:
         """Path to the generated userver proto structs file."""
@@ -191,6 +193,18 @@ def own_includes(self) -> Iterable[includes.Include]:
         )
         for field in self.fields:
             yield from field.field_type.collect_includes()
+        yield includes.Include(path='userver/proto-structs/io/fwd.hpp', kind=includes.IncludeKind.FOR_HPP)
+
+        # For IO.
+        yield from [
+            includes.Include(path=path, kind=includes.IncludeKind.FOR_CPP)
+            for path in [
+                'userver/proto-structs/io/impl/field_accessor.hpp',
+                'userver/proto-structs/io/impl/read.hpp',
+                'userver/proto-structs/io/impl/write.hpp',
+                'userver/proto-structs/type_mapping.hpp',
+            ]
+        ]
 
     @override
     def type_dependencies(self) -> Iterable[type_ref.TypeDependency]:
@@ -315,6 +329,7 @@ def kind(self) -> str:
     def own_includes(self) -> Iterable[includes.Include]:
         for field in self.fields:
             yield from field.field_type.collect_includes()
+        yield includes.Include(path='userver/proto-structs/impl/oneof_codegen.hpp', kind=includes.IncludeKind.FOR_HPP)
 
     @override
     def type_dependencies(self) -> Iterable[type_ref.TypeDependency]:
@@ -323,7 +338,7 @@ def type_dependencies(self) -> Iterable[type_ref.TypeDependency]:
             yield from field.field_type.type_dependencies()
 
 
-class VanillaType(TypeNode):
+class VanillaTypeDeclaration(TypeNode):
     """A C++ vanilla type."""
 
     def __init__(
@@ -336,7 +351,7 @@ def __init__(
         self.vanilla_name: Final[names.TypeName] = vanilla_name
 
 
-class VanillaClass(VanillaType):
+class VanillaClassDeclaration(VanillaTypeDeclaration):
     """A C++ vanilla class."""
 
     @property

scripts/proto_structs/models/gen_node.py

@@ -9,6 +9,7 @@
 from typing import List
 from typing import Optional
 
+from proto_structs.models import includes_bundles
 from proto_structs.models import names
 
 
@@ -21,7 +22,7 @@ class IncludeKind(enum.Enum):
     FOR_CPP = enum.auto()
 
 
-@dataclasses.dataclass
+@dataclasses.dataclass(order=True, frozen=True)
 class Include:
     """A C++ include"""
 
@@ -30,12 +31,6 @@ class Include:
     #: Include type.
     kind: IncludeKind
 
-    def __hash__(self):
-        return hash(self.path)
-
-    def __lt__(self, other: 'Include') -> bool:
-        return self.path < other.path
-
 
 class HasCppIncludes(abc.ABC):
     """A C++ entity or entity group that may require includes to define."""
@@ -53,8 +48,16 @@ def sorted_includes(entity: HasCppIncludes, *, current_hpp: Optional[str] = None
         includes_set.discard(Include(path=current_hpp, kind=IncludeKind.FOR_HPP))
     sorted_includes = sorted(includes_set)
     return {
-        IncludeKind.FOR_HPP: [include.path for include in sorted_includes if include.kind == IncludeKind.FOR_HPP],
-        IncludeKind.FOR_CPP: [include.path for include in sorted_includes if include.kind == IncludeKind.FOR_CPP],
+        IncludeKind.FOR_HPP: [
+            include.path
+            for include in sorted_includes
+            if include.kind == IncludeKind.FOR_HPP and include.path not in includes_bundles.BUNDLE_HPP
+        ],
+        IncludeKind.FOR_CPP: [
+            include.path
+            for include in sorted_includes
+            if include.kind == IncludeKind.FOR_CPP and include.path not in includes_bundles.BUNDLE_CPP
+        ],
     }
 
 
@@ -68,6 +71,11 @@ def proto_path_to_vanilla_pb_h(path: pathlib.Path) -> str:
     return str(path.with_suffix('.pb.h'))
 
 
+def structs_path_to_vanilla_pb_h(path: str) -> str:
+    """Returns the path of vanilla C++ `.pb.h` file (relative to source dir)."""
+    return path.removesuffix('.structs.usrv.pb.hpp') + '.pb.h'
+
+
 def io_includes_by_full_name(name: str, *, prefix: str) -> List[Include]:
     """Returns includes for IO."""
 

scripts/proto_structs/models/includes.py

@@ -0,0 +1,31 @@
+"""Sets of includes that are in bundles."""
+
+from typing import Set
+
+BUNDLE_HPP: Set[str] = {
+    'cstddef',
+    'cstdint',
+    'limits',
+    'optional',
+    'string',
+    'utility',
+    'vector',
+    'userver/proto-structs/impl/oneof_codegen.hpp',
+    'userver/proto-structs/io/fwd.hpp',
+}
+
+BUNDLE_CPP: Set[str] = {
+    'userver/proto-structs/io/impl/field_accessor.hpp',
+    'userver/proto-structs/io/impl/read.hpp',
+    'userver/proto-structs/io/impl/write.hpp',
+    'userver/proto-structs/type_mapping.hpp',
+    'userver/proto-structs/io/std/optional_conv.hpp',
+    'userver/proto-structs/io/std/string_conv.hpp',
+    'userver/proto-structs/io/std/vector_conv.hpp',
+    'userver/proto-structs/io/std/int32_t_conv.hpp',
+    'userver/proto-structs/io/std/int64_t_conv.hpp',
+    'userver/proto-structs/io/std/size_t_conv.hpp',
+    'userver/proto-structs/io/std/uint32_t_conv.hpp',
+    'userver/proto-structs/io/std/uint64_t_conv.hpp',
+    'userver/proto-structs/io/std/scalar_conv.hpp',
+}

scripts/proto_structs/models/includes_bundles.py

@@ -123,10 +123,7 @@ def full_cpp_name_segments(self) -> Sequence[str]:
 
     @override
     def collect_includes(self) -> Iterable[includes.Include]:
-        # For IO.
-        yield includes.Include(
-            path='userver/proto-structs/impl/bundles/structs_cpp.hpp', kind=includes.IncludeKind.FOR_CPP
-        )
+        yield includes.Include(path='userver/proto-structs/io/std/scalar_conv.hpp', kind=includes.IncludeKind.FOR_CPP)
 
 
 class BuiltinType(TypeReference, names.HasCppNameImpl):
@@ -194,7 +191,7 @@ def full_cpp_name_segments(self) -> Sequence[str]:
     def collect_includes(self) -> Iterable[includes.Include]:
         yield includes.Include(path=self._include, kind=includes.IncludeKind.FOR_HPP)
         yield includes.Include(
-            path=self._include.removesuffix('.structs.usrv.pb.hpp') + '.pb.h', kind=includes.IncludeKind.FOR_CPP
+            path=includes.structs_path_to_vanilla_pb_h(self._include), kind=includes.IncludeKind.FOR_CPP
         )
 
 

scripts/proto_structs/models/type_ref.py

@@ -6,11 +6,11 @@
 from proto_structs.models import names
 
 
-def collect_vanilla_types(*, types: List[gen_node.TypeNode]) -> List[gen_node.VanillaType]:
-    vanillas: List[gen_node.VanillaType] = []
+def collect_vanilla_types(*, types: List[gen_node.TypeNode]) -> List[gen_node.VanillaTypeDeclaration]:
+    vanillas: List[gen_node.VanillaTypeDeclaration] = []
     for type_node in types:
         for child in type_node.iter_all_children():
             if isinstance(child, gen_node.StructNode):
                 vanilla_name: names.TypeName = names.get_vanilla_type_name(name=child.vanilla_name)
-                vanillas.append(gen_node.VanillaClass(vanilla_name=vanilla_name))
+                vanillas.append(gen_node.VanillaClassDeclaration(vanilla_name=vanilla_name))
     return vanillas

scripts/proto_structs/models/vanilla.py

@@ -35,40 +35,41 @@ namespace {{ namespace.short_name }} {
 {% endmacro -%}
 
 {%- macro do_render_vanilla(struct) %}
-::{{ struct.vanilla_full_name }}
-{% endmacro -%}
+::{{ struct.vanilla_full_name -}}
+{%- endmacro -%}
 
 {%- macro do_render_field_number(struct, field) %}
-{{ do_render_vanilla(struct) | trim }}::{{ field.field_number_name }}
-{% endmacro -%}
+{{ do_render_vanilla(struct) -}}::{{ field.field_number_name }}
+{%- endmacro -%}
 
 {%- macro do_render_field_set(struct, field) %}
-&{{ do_render_vanilla(struct) | trim }}::set_{{ field.short_vanilla_name }}
-{% endmacro -%}
+&{{ do_render_vanilla(struct) -}}::set_{{ field.short_vanilla_name }}
+{%- endmacro -%}
 
 {%- macro do_render_field_clear(struct, field) %}
-&{{ do_render_vanilla(struct) | trim }}::clear_{{ field.short_vanilla_name }}
-{% endmacro -%}
+&{{ do_render_vanilla(struct)  -}}::clear_{{ field.short_vanilla_name }}
+{%- endmacro -%}
 
 {%- macro do_render_field_mutable(struct, field) %}
-&{{ do_render_vanilla(struct) | trim }}::mutable_{{ field.short_vanilla_name }}
-{% endmacro -%}
+&{{ do_render_vanilla(struct) -}}::mutable_{{ field.short_vanilla_name }}
+{%- endmacro -%}
 
 {%- macro do_render_ptr_getter(struct, field) %}
-&{{ do_render_vanilla(struct) | trim }}::{{ field.short_vanilla_name }}
-{% endmacro -%}
+&{{ do_render_vanilla(struct) -}}::{{ field.short_vanilla_name }}
+{%- endmacro -%}
 
 {%- macro do_render_ptr_has(struct, field) %}
-&{{ do_render_vanilla(struct) | trim }}::has_{{ field.short_vanilla_name }}
-{% endmacro -%}
+&{{ do_render_vanilla(struct) -}}::has_{{ field.short_vanilla_name }}
+{%- endmacro -%}
 
 {%- macro do_render_field_getter(struct, field, pass_has) %}
 USERVER_NAMESPACE::proto_structs::io::impl::CreateFieldGetter(msg, ::{{ struct.vanilla_full_name }}::{{ field.field_number_name }},
 {% filter indent(width=4, first=True) %}
-{{ do_render_ptr_getter(struct, field) }} 
-{%- if pass_has %},{{ do_render_ptr_has(struct, field) | trim }}{% endif %})
+{{ do_render_ptr_getter(struct, field) -}} 
+{%- if pass_has -%},{{ do_render_ptr_has(struct, field) -}}{% endif %}
+)
 {% endfilter %}
-{% endmacro -%}
+{%- endmacro -%}
 
 {%- macro do_render_read_field(struct, context, field) %}
 .{{ field.short_name }} = USERVER_NAMESPACE::proto_structs::io::impl::ReadField<decltype({{ struct.contextual_cpp_name(context=context) }}::{{ field.short_name }})>(ctx, 
@@ -80,7 +81,7 @@ USERVER_NAMESPACE::proto_structs::io::impl::CreateFieldGetter(msg, ::{{ struct.v
 {%- elif field.read_kind == "oneof" %}
 {%- for oneof_field in field.oneof_fields %} 
 {{ do_render_field_getter(struct, oneof_field, True) }}
-{%- if not loop.last %},{% endif %}
+{%- if not loop.last -%},{%- endif %}
 {% endfor %}
 {% endif %}
 ),
@@ -109,9 +110,9 @@ return {
 USERVER_NAMESPACE::proto_structs::io::impl::CreateFieldSetter(
 {% filter indent(width=4, first=True) %}
 msg,
-{{ do_render_field_number(struct, field) | trim }},
-{{ do_render_field_set(struct, field) | trim }}<const std::string&>,
-{{ do_render_field_clear(struct, field) | trim }}
+{{ do_render_field_number(struct, field) }},
+{{ do_render_field_set(struct, field) }}<const std::string&>,
+{{ do_render_field_clear(struct, field) }}
 {% endfilter %}
 )
 {% endmacro -%}
@@ -120,9 +121,9 @@ msg,
 USERVER_NAMESPACE::proto_structs::io::impl::CreateFieldSetter(
 {% filter indent(width=4, first=True) %}
 msg, 
-{{ do_render_field_number(struct, field) | trim }},
-{{ do_render_field_set(struct, field) | trim }},
-{{ do_render_field_clear(struct, field) | trim}}
+{{ do_render_field_number(struct, field) }},
+{{ do_render_field_set(struct, field) }},
+{{ do_render_field_clear(struct, field)}}
 {% endfilter %}
 )
 {% endmacro -%}
@@ -131,9 +132,9 @@ msg,
 USERVER_NAMESPACE::proto_structs::io::impl::CreateFieldSetter(
 {% filter indent(width=4, first=True) %}
 msg, 
-{{ do_render_field_number(struct, field) | trim }},
-{{ do_render_field_mutable(struct, field) | trim }},
-{{ do_render_field_clear(struct, field) | trim }}
+{{ do_render_field_number(struct, field) }},
+{{ do_render_field_mutable(struct, field) }},
+{{ do_render_field_clear(struct, field) }}
 {% endfilter %}
 )
 {% endmacro -%}

scripts/proto_structs/templates/io.inc.jinja

@@ -1,4 +1,4 @@
-{% import "forward_declarations.inc.jinja" as forward_declarations %}
+{% import "types_forward_declarations.inc.jinja" as types_forward_declarations %}
 {% import "types.inc.jinja" as types with context %}
 {% import "utils.inc.jinja" as utils %}
 {{ utils.do_not_edit(file_name_wo_ext) }}
@@ -12,7 +12,7 @@
 {% endfor %}
 
 {% for node in gen_nodes %}
-{{ forward_declarations.render_gen_node(node) }}
+{{ types_forward_declarations.render_gen_node(node) }}
 {%- endfor %}
 
 {% for node in gen_nodes %}