Merge pull request #123 from emqx/master

Add map type schema store

Author: mikpe

.github/workflows/erlang.yml

@@ -0,0 +1,24 @@
+name: Erlang CI
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+permissions:
+  contents: read
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    container:
+      image: erlang:26
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Run tests
+      run: make eunit && make cover

changelog.md

@@ -1,3 +1,6 @@
+* 2.10.0
+   - Add map as avro store, and use it as default.
+   - Changed to store type aliases as type's full name index, so the type store map (or dict) is less bloated.
 * 2.9.10
    - Optimize avro:is_same_type/2
    - Upgrade jsone to 1.8.1

src/avro_json_encoder.erl

@@ -347,7 +347,10 @@ do_encode_value(Array) when ?IS_ARRAY_VALUE(Array) ->
   lists:map(fun do_encode_value/1, ?AVRO_VALUE_DATA(Array));
 do_encode_value(Map) when ?IS_MAP_VALUE(Map) ->
   L = avro_map:to_list(Map),
-  lists:map(fun encode_field_with_value/1, L);
+  lists:foldl(fun (X, Acc) ->
+                  {Key, Value} = encode_field_with_value(X),
+                  maps:put(Key, Value, Acc)
+              end, #{}, L);
 do_encode_value(Fixed) when ?IS_FIXED_VALUE(Fixed) ->
   %% jsone treats binary as utf8 string
   ?INLINE(encode_binary(?AVRO_VALUE_DATA(Fixed)));

src/avro_schema_store.erl

@@ -62,8 +62,8 @@
 
 -include("avro_internal.hrl").
 
--opaque store() :: ets:tab() | {dict, dict:dict()}.
--type option_key() :: access | name | dict.
+-type store() :: ets:tab() | {dict, dict:dict()} | map().
+-type option_key() :: access | name | dict | map.
 -type options() :: [option_key() | {option_key(), term()}].
 -type filename() :: file:filename_all().
 
@@ -81,12 +81,20 @@ new() -> new([]).
 %%    mode in ets:new and defines what processes can have access to
 %%  * `{name, atom()}' - used to create a named ets table.
 %%  * `dict' - use dict as store backend, ignore `access' and `name' options
+%%  * `map' - use map as store backend, ignore `access' and `name' options
 %% @end
 -spec new(options()) -> store().
 new(Options) ->
   case proplists:get_bool(dict, Options) of
-    true -> {dict, dict:new()};
-    false -> new_ets(Options)
+    true ->
+          {dict, dict:new()};
+    false ->
+          case proplists:get_bool(map, Options) of
+              true ->
+                  #{};
+              false ->
+                new_ets(Options)
+          end
   end.
 
 %% @doc Create a new schema store and improt the given schema JSON files.
@@ -98,6 +106,7 @@ new(Options, Files) ->
 %% @doc Return true if the given arg is a schema store.
 -spec is_store(term()) -> boolean().
 is_store({dict, _}) -> true;
+is_store(Map) when is_map(Map) -> true;
 is_store(T) -> is_integer(T) orelse is_atom(T) orelse is_reference(T).
 
 %% @doc Make a schema lookup function from store.
@@ -143,12 +152,13 @@ import_schema_json(Json, Store) ->
 %% @doc Delete the ets table.
 -spec close(store()) -> ok.
 close({dict, _}) -> ok;
+close(Map) when is_map(Map) -> ok;
 close(Store) ->
   ets:delete(Store),
   ok.
 
 %% @doc To make dialyzer happy.
--spec ensure_store(atom() | integer() | reference() | {dict, dict:dict()}) ->
+-spec ensure_store(atom() | integer() | reference() | store()) ->
         store().
 ensure_store(Store) ->
   true = is_store(Store),
@@ -194,6 +204,7 @@ get_all_types(Store) ->
 
 -spec to_list(store()) -> [{name(), avro_type()}].
 to_list({dict, Dict}) -> dict:to_list(Dict);
+to_list(Map) when is_map(Map) -> maps:to_list(Map);
 to_list(Store) -> ets:tab2list(Store).
 
 -spec new_ets(options()) -> store().
@@ -212,8 +223,8 @@ new_ets(Options) ->
 -spec add_by_assigned_name(undefined | name_raw(),
                            type_or_name(), store()) -> store().
 add_by_assigned_name(undefined, _Type, Store) -> Store;
-add_by_assigned_name(AssignedName, Type, Store) ->
-  do_add_type_by_names([?NAME(AssignedName)], Type, Store).
+add_by_assigned_name(AssignedName, TypeOrName, Store) ->
+  add_type_by_name(?NAME(AssignedName), TypeOrName, Store).
 
 %% @private Parse file basename. try to strip ".avsc" or ".json" extension.
 -spec parse_basename(filename()) -> name().
@@ -241,13 +252,19 @@ import_schema_json(AssignedName, Json, Store) ->
 do_add_type(Type, Store) ->
   FullName = avro:get_type_fullname(Type),
   Aliases = avro:get_aliases(Type),
-  do_add_type_by_names([FullName|Aliases], Type, Store).
+  Store1 = add_type_by_name(FullName, Type, Store),
+  add_aliases(Aliases, FullName, Store1).
+
+add_aliases([], _FullName, Store) ->
+  Store;
+add_aliases([Alias | More], FullName, Store) ->
+  NewStore = put_type_to_store(Alias, FullName, Store),
+  add_aliases(More, FullName, NewStore).
 
 %% @private
--spec do_add_type_by_names([fullname()], avro_type(), store()) ->
+-spec add_type_by_name([fullname()], avro_type(), store()) ->
         store() | no_return().
-do_add_type_by_names([], _Type, Store) -> Store;
-do_add_type_by_names([Name|Rest], Type, Store) ->
+add_type_by_name(Name, Type, Store) ->
   case get_type_from_store(Name, Store) of
     {ok, Type} ->
       Store;
@@ -257,27 +274,45 @@ do_add_type_by_names([Name|Rest], Type, Store) ->
       %% old / new types.
       erlang:error({name_clash, Name, Type, OtherType});
     false   ->
-      Store1 = put_type_to_store(Name, Type, Store),
-      do_add_type_by_names(Rest, Type, Store1)
+      put_type_to_store(Name, Type, Store)
   end.
 
 %% @private
--spec put_type_to_store(fullname(), avro_type(), store()) -> store().
+-spec put_type_to_store(fullname(), name() | avro_type(), store()) -> store().
 put_type_to_store(Name, Type, {dict, Dict}) ->
   NewDict = dict:store(Name, Type, Dict),
   {dict, NewDict};
+put_type_to_store(Name, Type, Map) when is_map(Map) ->
+  Map#{Name => Type};
 put_type_to_store(Name, Type, Store) ->
   true = ets:insert(Store, {Name, Type}),
   Store.
 
-%% @private
+%% @private Get type by name or alias.
 -spec get_type_from_store(fullname(), store()) -> false | {ok, avro_type()}.
-get_type_from_store(Name, {dict, Dict}) ->
+get_type_from_store(NameRef, Store) ->
+  case do_get_type_from_store(NameRef, Store) of
+    false ->
+      false;
+    {ok, FullName} when is_binary(FullName) ->
+      do_get_type_from_store(FullName, Store);
+    {ok, Type} ->
+      {ok, Type}
+  end.
+
+%% @private
+-spec do_get_type_from_store(fullname(), store()) -> false | {ok, fullname() | avro_type()}.
+do_get_type_from_store(Name, {dict, Dict}) ->
   case dict:find(Name, Dict) of
     error -> false;
     {ok, Type} -> {ok, Type}
   end;
-get_type_from_store(Name, Store) ->
+do_get_type_from_store(Name, Map) when is_map(Map) ->
+  case maps:find(Name, Map) of
+    error -> false;
+    {ok, Type} -> {ok, Type}
+  end;
+do_get_type_from_store(Name, Store) ->
   case ets:lookup(Store, Name) of
     []             -> false;
     [{Name, Type}] -> {ok, Type}

src/avro_util.erl

@@ -83,10 +83,10 @@ ensure_lkup_fun(Sc) ->
     false -> make_lkup_fun(?ASSIGNED_NAME, Sc)
   end.
 
-%% @doc Make a schema store (dict based) and wrap it in a lookup fun.
+%% @doc Make a schema store (map based) and wrap it in a lookup fun.
 -spec make_lkup_fun(name_raw(), avro_type()) -> lkup().
 make_lkup_fun(AssignedName, Type) ->
-  Store0 = avro_schema_store:new([dict]),
+  Store0 = avro_schema_store:new([map]),
   Store = avro_schema_store:add_type(AssignedName, Type, Store0),
   avro_schema_store:to_lookup_fun(Store).
 

test/avro_json_encoder_tests.erl

@@ -249,6 +249,16 @@ encode_map_test() ->
   ?assertEqual(<<"{\"v1\":{\"int\":1},\"v2\":null,\"v3\":{\"int\":2}}">>,
                Json1).
 
+encode_empty_map_test() ->
+  MapType = avro_map:type(avro_union:type([int, null])),
+  MapValue0 = avro_map:new(MapType, []),
+  Json0 = encode_value(MapValue0),
+  ?assertEqual(<<"{}">>, Json0),
+
+  MapValue1 = avro_map:new(MapType, #{}),
+  Json1 = encode_value(MapValue1),
+  ?assertEqual(<<"{}">>, Json1).
+
 encode_fixed_type_test() ->
   Type = avro_fixed:type("FooBar", 2,
                          [ {namespace, "name.space"}

test/avro_schema_store_tests.erl

@@ -35,11 +35,13 @@ get_all_types_test() ->
       ok = avro_schema_store:close(Store)
     end,
   ok = TestFun(avro_schema_store:new([{name, ?MODULE}])),
-  ok = TestFun(avro_schema_store:new([dict])).
+  ok = TestFun(avro_schema_store:new([dict])),
+  ok = TestFun(avro_schema_store:new([map])).
 
 is_store_test() ->
   ?assertNot(avro_schema_store:is_store(<<"json">>)),
   ?assert(avro_schema_store:is_store(avro_schema_store:new([dict]))),
+  ?assert(avro_schema_store:is_store(avro_schema_store:new([map]))),
   ?assert(avro_schema_store:is_store(avro_schema_store:new([]))).
 
 ensure_store_test() ->
@@ -52,7 +54,8 @@ ensure_store_test() ->
   ?assertEqual(1, avro_schema_store:ensure_store(1)),
   ?assertEqual(atom, avro_schema_store:ensure_store(atom)),
   ?assertEqual({dict, dict:new()},
-               avro_schema_store:ensure_store({dict, dict:new()})).
+               avro_schema_store:ensure_store({dict, dict:new()})),
+  ?assertEqual(#{}, avro_schema_store:ensure_store(#{})).
 
 flatten_type_test() ->
   Type = avro_array:type(test_record()),
@@ -81,7 +84,8 @@ add_type_test() ->
         ok = avro_schema_store:close(Store1)
     end,
   ok = TestFun(avro_schema_store:new()),
-  ok = TestFun(avro_schema_store:new([dict])).
+  ok = TestFun(avro_schema_store:new([dict])),
+  ok = TestFun(avro_schema_store:new([map])).
 
 lookup(Name, Store) ->
   avro_schema_store:lookup_type(Name, Store).

test/avro_tests.erl

@@ -337,7 +337,7 @@ wrapped_union_cast_test() ->
   Rec1 = avro_record:type("rec1", [avro_record:define_field("f1", int)]),
   Rec2 = avro_record:type("rec2", [avro_record:define_field("f1", int)]),
   Union = avro_union:type(["rec1", "rec2"]),
-  Store0 = avro_schema_store:new([dict]),
+  Store0 = avro_schema_store:new([map]),
   Store1 = avro_schema_store:add_type(Rec1, Store0),
   Store = avro_schema_store:add_type(Rec2, Store1),
   Lkup = avro_util:ensure_lkup_fun(Store),