Merge pull request #278 from lafirest/upgrade_to_40

Upgrade to 40

SCHEMA.md

@@ -37,7 +37,7 @@ to_ip_port(String) ->
 
 ### Complex types
 
-HOCON schema supports 3 different complext types: `struct`, `array`, and `union`.
+HOCON schema supports 3 different complex types: `struct`, `array`, and `union`.
 NOTE: to make it easier for future extensions, it's recommended to use `hoconsc` module APIs to define schema.
 
 #### Structs
@@ -224,7 +224,7 @@ Environment variables are not parsed as plain string, rather as HOCON values.
 This creates the flexibility for overriding config values in different ways:
 
 * Set individual object paths, for example `export EMQX_MY__KEY__name=zz; export EMQX_MY__KEY__fingers=10`
-* Set the the engire object as escaped HOCON value: `export EMQX_MY__KEY="{name = \"zz\", fingers = 10}"`
+* Set the the entire object as escaped HOCON value: `export EMQX_MY__KEY="{name = \"zz\", fingers = 10}"`
 * Load the object from another file `export EMQX_MY__KEY="{\"include /config/my-key-override.conf\"}"`
 
 Using `{include "path/to/file"}` is extremely useful to override a value with large object or an array.

include/hocon_types.hrl

@@ -18,7 +18,8 @@
 -define(HOCONSC_TYPES_HRL, true).
 
 -define(ARRAY(OfTYpe), {array, OfTYpe}).
--define(UNION(OfTypes), {union, OfTypes}).
+-define(UNION(OfTypes), {union, OfTypes, undefined}).
+-define(UNION(OfTypes, DisplayName), {union, OfTypes, DisplayName}).
 -define(ENUM(OfSymbols), {enum, OfSymbols}).
 -define(REF(Name), {ref, Name}).
 % remote ref

sample-schemas/demo_schema.erl

@@ -38,7 +38,7 @@ fields("a_b") ->
     ];
 
 fields("b") ->
-    [ {"u", fun (type) -> {union, ["priv.bool", "priv.int"]};
+    [ {"u", fun (type) -> hoconsc:union(["priv.bool", "priv.int"]);
                 (mapping) -> "app_foo.u";
                 (_) -> undefined end}
     , {"arr", fun (type) -> hoconsc:array("priv.int");

sample-schemas/demo_schema6.erl

@@ -4,7 +4,7 @@
 
 -behaviour(hocon_schema).
 
--export([namespace/0, roots/0, fields/1]).
+-export([namespace/0, roots/0, fields/1, desc/1]).
 
 namespace() -> undefined.
 
@@ -18,3 +18,8 @@ fields(foo) ->
     ];
 fields(im_hidden) ->
     [{i_should_be_hidden, integer()}].
+
+desc(foo) ->
+    {foo, invalid};
+desc(_) ->
+    undefined.

src/hocon_maps.erl

@@ -50,6 +50,10 @@
 
 -type flatten_opts() :: #{rich_value => boolean()}.
 
+-elvis([
+    {elvis_style, atom_naming_convention, #{regex => "^([a-z][a-z0-9]*_?)([a-z0-9]*_?)*(_SUITE)?$"}}
+]).
+
 %% @doc put unboxed value to the richmap box
 %% this function is called places where there is no boxing context
 %% so it has to accept unboxed value.
@@ -89,13 +93,17 @@ update_map_field(Opts, Map, FieldName, GoDeep) ->
     Map1 = maps:without([FieldName], Map),
     Map1#{maybe_atom(Opts, FieldName) => FieldV}.
 
+maybe_atom(#{atom_key := true}, Name) when is_binary(Name), size(Name) > 255 ->
+    error({name_longer_than_255_bytes, Name});
 maybe_atom(#{atom_key := true}, Name) when is_binary(Name) ->
     try
         binary_to_existing_atom(Name, utf8)
     catch
         _:_ ->
             error({non_existing_atom, Name})
     end;
+maybe_atom(#{atom_key := {true, unsafe}}, Name) when is_binary(Name), size(Name) > 255 ->
+    error({name_longer_than_255_bytes, Name});
 maybe_atom(#{atom_key := {true, unsafe}}, Name) when is_binary(Name) ->
     binary_to_atom(Name, utf8);
 maybe_atom(_Opts, Name) ->

src/hocon_schema.erl

@@ -71,9 +71,7 @@
 -callback tags() -> [tag()].
 
 -optional_callbacks([
-    namespace/0,
     roots/0,
-    fields/1,
     translations/0,
     translation/1,
     validations/0,
@@ -97,13 +95,13 @@
     %% array of
     | ?ARRAY(type())
     %% one-of
-    | ?UNION(union_members())
+    | ?UNION(union_members(), _)
     %% one-of atoms, data is allowed to be binary()
     | ?ENUM([atom()]).
 
 -type field_schema() ::
     typerefl:type()
-    | ?UNION(union_members())
+    | ?UNION(union_members(), _)
     | ?ARRAY(type())
     | ?ENUM(type())
     | field_schema_map()
@@ -342,7 +340,7 @@ find_structs_per_type(Schema, ?LAZY(Type), Acc, Stack, TStack, Opts) ->
     find_structs_per_type(Schema, Type, Acc, Stack, TStack, Opts);
 find_structs_per_type(Schema, ?ARRAY(Type), Acc, Stack, TStack, Opts) ->
     find_structs_per_type(Schema, Type, Acc, ["$INDEX" | Stack], TStack, Opts);
-find_structs_per_type(Schema, ?UNION(Types0), Acc, Stack, TStack, Opts) ->
+find_structs_per_type(Schema, ?UNION(Types0, _), Acc, Stack, TStack, Opts) ->
     Types = hoconsc:union_members(Types0),
     lists:foldl(
         fun(T, AccIn) ->
@@ -458,7 +456,7 @@ field_schema(?R_REF(_, _) = Ref, SchemaKey) ->
     field_schema(hoconsc:mk(Ref), SchemaKey);
 field_schema(?ARRAY(_) = Array, SchemaKey) ->
     field_schema(hoconsc:mk(Array), SchemaKey);
-field_schema(?UNION(_) = Union, SchemaKey) ->
+field_schema(?UNION(_, _) = Union, SchemaKey) ->
     field_schema(hoconsc:mk(Union), SchemaKey);
 field_schema(?ENUM(_) = Enum, SchemaKey) ->
     field_schema(hoconsc:mk(Enum), SchemaKey);
@@ -492,10 +490,11 @@ fmt_type(Ns, ?ARRAY(T)) ->
         kind => array,
         elements => fmt_type(Ns, T)
     };
-fmt_type(Ns, ?UNION(Ts)) ->
+fmt_type(Ns, ?UNION(Ts, DisplayName)) ->
     #{
         kind => union,
-        members => [fmt_type(Ns, T) || T <- hoconsc:union_members(Ts)]
+        members => [fmt_type(Ns, T) || T <- hoconsc:union_members(Ts)],
+        display_name => DisplayName
     };
 fmt_type(_Ns, ?ENUM(Symbols)) ->
     #{

src/hocon_schema_example.erl

@@ -90,7 +90,7 @@ fmt_field(
                 case maps:get(examples, Field, #{}) of
                     #{} = Example ->
                         fmt_field_with_example(Path, SubFields, Example, Opts2);
-                    {union, UnionExamples} ->
+                    ?UNION(UnionExamples, _) ->
                         Examples1 = filter_union_example(UnionExamples, SubFields),
                         fmt_field_with_example(Path, SubFields, Examples1, Opts2);
                     {array, ArrayExamples} ->

src/hocon_schema_json.erl

@@ -94,8 +94,15 @@ gen_struct(_RootNs, Ns, Name, #{fields := Fields} = Meta, Opts) ->
         fields => fmt_fields(Ns, Fields, Opts)
     },
     case Meta of
-        #{desc := StructDoc} -> S0#{desc => fmt_desc(StructDoc, Opts)};
-        _ -> S0
+        #{desc := StructDoc} ->
+            case fmt_desc(StructDoc, Opts) of
+                undefined ->
+                    S0;
+                Text ->
+                    S0#{desc => Text}
+            end;
+        _ ->
+            S0
     end.
 
 assert_unique_names(FullName, Fields) ->
@@ -173,7 +180,23 @@ fmt_type(Ns, T) ->
     hocon_schema:fmt_type(Ns, T).
 
 fmt_desc(Desc, #{desc_resolver := F}) when Desc =/= undefined ->
-    F(Desc);
+    case F(Desc) of
+        undefined ->
+            undefined;
+        IoData ->
+            try
+                B = unicode:characters_to_binary(IoData, utf8),
+                true = is_binary(B),
+                B
+            catch
+                _:_ ->
+                    throw(#{
+                        reason => bad_desc_resolution,
+                        reference => Desc,
+                        resolution => IoData
+                    })
+            end
+    end;
 fmt_desc(_Desc, _) ->
     %% no resolver, no description needed at all for this schema dump
     undefined.

src/hocon_tconf.erl

@@ -585,7 +585,7 @@ map_field(?REF(Ref), FieldSchema, Value, #{schema := Schema} = Opts) ->
 map_field(Ref, FieldSchema, Value, #{schema := Schema} = Opts) when is_list(Ref) ->
     Fields = hocon_schema:fields(Schema, Ref),
     do_map(Fields, Value, Opts, FieldSchema);
-map_field(?UNION(Types0), Schema0, Value, Opts) ->
+map_field(?UNION(Types0, _), Schema0, Value, Opts) ->
     try select_union_members(Types0, Value, Opts) of
         Types ->
             F = fun(Type) ->
@@ -896,7 +896,7 @@ is_path(Schema, ?ARRAY(Type), [Name | Path]) ->
         {true, _} -> is_path(Schema, Type, Path);
         false -> false
     end;
-is_path(Schema, ?UNION(Types), Path) ->
+is_path(Schema, ?UNION(Types, _), Path) ->
     lists:any(fun(T) -> is_path(Schema, T, Path) end, hoconsc:union_members(Types));
 is_path(Schema, ?MAP(_, Type), [_ | Path]) ->
     is_path(Schema, Type, Path);
@@ -1337,7 +1337,15 @@ default_map_key_name_validator(Name) ->
                 got => Name
             }};
         _ ->
-            ok
+            case string:length(Name) > 255 of
+                true ->
+                    {error, #{
+                        hint => "map keys must have less than 255 bytes",
+                        got => Name
+                    }};
+                _ ->
+                    ok
+            end
     end.
 
 fmt_field_names(Names) ->

src/hoconsc.erl

@@ -19,7 +19,7 @@
 
 -export([mk/1, mk/2]).
 -export([ref/1, ref/2]).
--export([array/1, union/1, enum/1]).
+-export([array/1, union/1, union/2, enum/1]).
 -export([lazy/1, map/2]).
 -export([is_schema/1]).
 -export([union_members/1]).
@@ -59,7 +59,11 @@ array(OfType) -> ?ARRAY(OfType).
 %%    `({value, #{<<"kind">> := <<"foo">>}) -> [ref(foo)];'
 %%    `({value, #{<<"kind">> := <<"bar">>}} -> [ref(bar)].'
 -spec union(hocon_schema:union_members()) -> ?UNION(hocon_schema:union_members()).
-union(OfTypes) when is_list(OfTypes) orelse is_function(OfTypes, 1) -> ?UNION(OfTypes).
+union(OfTypes) when is_list(OfTypes) orelse is_function(OfTypes, 1) ->
+    ?UNION(OfTypes, undefined).
+
+union(OfTypes, DisplayName) when is_list(OfTypes) orelse is_function(OfTypes, 1) ->
+    ?UNION(OfTypes, DisplayName).
 
 %% @doc make a enum type.
 enum(OfSymbols) when is_list(OfSymbols) -> ?ENUM(OfSymbols).
@@ -71,7 +75,7 @@ lazy(HintType) -> ?LAZY(HintType).
 map(Name, Type) -> ?MAP(Name, Type).
 
 %% @doc Check Type is a hocon type.
-is_schema(?UNION(Members)) -> lists:all(fun is_schema/1, union_members(Members));
+is_schema(?UNION(Members, _)) -> lists:all(fun is_schema/1, union_members(Members));
 is_schema(?ARRAY(ElemT)) -> is_schema(ElemT);
 is_schema(?LAZY(HintT)) -> is_schema(HintT);
 is_schema(?REF(_)) -> true;
@@ -86,7 +90,7 @@ is_schema(_) -> false.
 assert_type(S) when is_function(S) -> error({expecting_type_but_got_schema, S});
 assert_type(#{type := _} = S) ->
     error({expecting_type_but_got_schema, S});
-assert_type(?UNION(Members)) ->
+assert_type(?UNION(Members, _)) ->
     lists:foreach(fun assert_type/1, union_members(Members));
 assert_type(?ENUM(Symbols)) ->
     lists:foreach(

test/hocon_schema_json_tests.erl

@@ -131,7 +131,7 @@ hidden_structs1_test() ->
     ).
 
 hidden_structs2_test() ->
-    Json = gen(demo_schema6),
+    Json = gen(demo_schema6, #{desc_resolver => fun(_) -> undefined end}),
     ?assertMatch(
         [
             #{
@@ -147,5 +147,19 @@ hidden_structs2_test() ->
         Json
     ).
 
+hidden_structs3_test() ->
+    ?assertThrow(
+        #{reason := bad_desc_resolution, resolution := invalid},
+        gen(demo_schema6, #{desc_resolver => fun(_) -> invalid end})
+    ).
+
+bad_desc_test() ->
+    Throw = fun(_) -> throw({foo, ?FUNCTION_NAME}) end,
+    ?assertThrow({foo, ?FUNCTION_NAME}, Throw(a)),
+    ?assertThrow({foo, ?FUNCTION_NAME}, gen(demo_schema6, #{desc_resolver => Throw})).
+
 gen(Schema) ->
     hocon_schema_json:gen(Schema).
+
+gen(Schema, Opts) ->
+    hocon_schema_json:gen(Schema, Opts).