Merge pull request #165 from zmstone/feat-support-apply-envs-without-…

…defatuls

Feat support apply envs without defatuls

SCHEMA.md

@@ -192,12 +192,12 @@ HOCON schema also supports below field metadata.
   the value is as expected. NOTE: the input to validator after convert (if present) is applied.
 * `default`: default value of the field. NOTE that default values are to be treated as raw inputs,
   meaning they are put hrough  the `converter`s and `validator`s etc, and then type-checked.
-* `override_env`: special environment variable name to override this field value.
-  NOTE: For generic override, see [below](#default_override_rule) for more info
 * `nullable`: set to `true` if this field is allowed to be `undefined`.
   NOTE: there is no point setting it to `true` if fields has a default value.
 * `sensitive`: set to `true` if this field's value is sensitive so we will obfuscate the log
   with `********` when logging.
+* `desc`: text for document generation
+* `hidden`: a boolean flag to hide it from appearing in config document
 
 <a name="default_override_rule"></a>
 

sample-schemas/demo_schema.erl

@@ -79,7 +79,6 @@ translation("app_foo") ->
 
 setting(mapping) -> "app_foo.setting";
 setting(type) -> string();
-setting(override_env) -> "MY_OVERRIDE";
 setting(_) -> undefined.
 
 range(Conf) ->

sample-schemas/demo_schema3.erl

@@ -20,15 +20,24 @@
 
 -export([namespace/0, roots/0, fields/1]).
 
-namespace() -> undefined.
+namespace() -> ?MODULE.
 
 roots() ->
     [ {foo, hoconsc:array(hoconsc:ref(foo))}
+    , {bar, hoconsc:ref(parent)}
     ].
 
 fields(foo) ->
     [ {bar, hoconsc:ref(bar)}
     ];
 fields(bar) ->
-    Sc = hoconsc:union([hoconsc:ref(foo), null]),
-    [{"baz", Sc}].
+    Sc = hoconsc:union([hoconsc:ref(foo), null]), %% cyclic refs
+    [{"baz", Sc}];
+%% below fields are to cover the test where identical paths for the same name
+%% i.e. bar.f1.subf.baz can be reached from both "sub1" and "sub2"
+fields(parent) ->
+    [{"f1", hoconsc:union([hoconsc:ref("sub1"), hoconsc:ref("sub2")])}];
+fields("sub1") ->
+    [{"sub_f", hoconsc:ref(bar)}]; %% both sub1 and sub2 refs to bar
+fields("sub2") ->
+    [{"sub_f", hoconsc:ref(bar)}]. %% both sub1 and sub2 refs to bar

sample-schemas/emqx_schema.erl

@@ -34,7 +34,7 @@
                 comma_separated_list/0, bar_separated_list/0, ip_port/0]).
 
 -export([roots/0, fields/1, translations/0, translation/1]).
--export([t/1, t/3, t/4, ref/1]).
+-export([t/1, t/3, ref/1]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 -export([ssl/2, tr_ssl/2, tr_password_hash/2]).
 
@@ -96,14 +96,14 @@ fields("k8s") ->
     ];
 
 fields("node") ->
-    [ {"name", t(string(), "vm_args.-name", "[email protected]", "NODE_NAME")}
+    [ {"name", t(string(), "vm_args.-name", "[email protected]")}
     , {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)}
-    , {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "NODE_COOKIE")}
+    , {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie")}
     , {"data_dir", t(string(), "emqx.data_dir", undefined)}
     , {"heartbeat", t(flag(), undefined, false)}
     , {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)}
     , {"process_limit", t(integer(), "vm_args.+P", undefined)}
-    , {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined, "MAX_PORTS")}
+    , {"max_ports", t(range(1024, 134217727), "vm_args.+Q", undefined)}
     , {"dist_buffer_size", fun node__dist_buffer_size/1}
     , {"global_gc_interval", t(duration_s(), "emqx.global_gc_interval", undefined)}
     , {"fullsweep_after", t(non_neg_integer(),
@@ -194,7 +194,7 @@ fields("acl") ->
     ];
 
 fields("mqtt") ->
-    [ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB", "MAX_PACKET_SIZE")}
+    [ {"max_packet_size", t(bytesize(), "emqx.max_packet_size", "1MB")}
     , {"max_clientid_len", t(integer(), "emqx.max_clientid_len", 65535)}
     , {"max_topic_levels", t(integer(), "emqx.max_topic_levels", 0)}
     , {"max_qos_allowed", t(range(0, 2), "emqx.max_qos_allowed", 2)}
@@ -1154,13 +1154,6 @@ t(T) ->
 t(T, M, D) ->
     fun (type) -> T; (mapping) -> M; (default) -> D; (_) -> undefined end.
 
-t(T, M, D, O) ->
-    fun (type) -> T;
-        (mapping) -> M;
-        (default) -> D;
-        (override_env) -> O;
-        (_) -> undefined end.
-
 ref(Field) ->
     fun (type) -> Field; (_) -> undefined end.
 

src/hocon.erl

@@ -53,7 +53,8 @@ load(Filename0, Opts) ->
     Ctx = hocon_util:stack_multiple_push(CtxList, #{}),
     try
         Bytes = hocon_token:read(Filename),
-        Conf = transform(do_binary(Bytes, Ctx), Opts),
+        Map = do_binary(Bytes, Ctx),
+        Conf = transform(Map, Opts),
         {ok, apply_opts(Conf, Opts)}
     catch
         throw:Reason -> {error, Reason}
@@ -102,10 +103,27 @@ binary(Binary, Opts) ->
     end.
 
 %% @doc Recursively merge two maps.
-%% NOTE: arrays are not merged.
+%% There are two array representations supported
+%% * as list, e.g. foo=[{bar=1},{bar=2}]
+%% * as indexed-map: e.g. foo={"1"={bar=1},"2"={bar=2}}
+%% When merging two values, if the `Base' is a list
+%% indexed map elements respects the base representation, that is
+%% the result of such merge is also a list.
+%% Otherwise indexed-map is used.
+%%
+%% When both `Base' and `Override' are lists, the elements
+%% are not merged, rather, the overriding-array replaces the whole base-array.
+%%
+%% NOTE: when merging indexed-map to list, the index boundary
+%% is checked to ensure the elements are consecutive
 -spec deep_merge(undefined | map(), map()) -> map().
 deep_merge(Base, Override) -> hocon_util:deep_merge(Base, Override).
 
+%% @doc Recursively merge two maps.
+%% Arrays are not merged as deep_merge/2.
+-spec deep_map_merge(undefined | map(), map()) -> map().
+deep_map_merge(Base, Override) -> hocon_util:deep_map_merge(Base, Override).
+
 do_binary(String, Ctx) when is_list(String) ->
     do_binary(iolist_to_binary(String), Ctx);
 do_binary(Binary, Ctx) when is_binary(Binary) ->
@@ -281,15 +299,15 @@ do_concat(Concat, Location) ->
 do_concat([], _, []) ->
     nothing;
 do_concat([], MetaKey, [{#{?METADATA := MetaFirstElem}, _V} = F | _Fs] = Acc) when ?IS_FIELD(F) ->
-    Metadata = deep_merge(MetaFirstElem, MetaKey),
+    Metadata = deep_map_merge(MetaFirstElem, MetaKey),
     case lists:all(fun (F0) -> ?IS_FIELD(F0) end, Acc) of
         true ->
             #{?HOCON_T => object, ?HOCON_V => lists:reverse(Acc), ?METADATA => Metadata};
         false ->
             concat_error(lists:reverse(Acc), #{?METADATA => Metadata})
     end;
 do_concat([], MetaKey, [#{?HOCON_T := string, ?METADATA := MetaFirstElem} | _] = Acc) ->
-    Metadata = deep_merge(MetaFirstElem, MetaKey),
+    Metadata = deep_map_merge(MetaFirstElem, MetaKey),
     case lists:all(fun (A) -> type_of(A) =:= string end, Acc) of
         true ->
             BinList = lists:map(fun(M) -> maps:get(?HOCON_V , M) end, lists:reverse(Acc)),
@@ -298,7 +316,7 @@ do_concat([], MetaKey, [#{?HOCON_T := string, ?METADATA := MetaFirstElem} | _] =
             concat_error(lists:reverse(Acc), #{?METADATA => Metadata})
     end;
 do_concat([], MetaKey, [#{?HOCON_T := array, ?METADATA := MetaFirstElem} | _] = Acc) ->
-    Metadata = deep_merge(MetaFirstElem, MetaKey),
+    Metadata = deep_map_merge(MetaFirstElem, MetaKey),
     case lists:all(fun (A) -> type_of(A) =:= array end, Acc) of
         true ->
             NewValue = lists:append(lists:reverse(lists:map(fun value_of/1, Acc))),
@@ -335,7 +353,9 @@ transform(#{?HOCON_T := object, ?HOCON_V := V}, Opts) ->
 
 do_transform([], Map, _Opts) -> Map;
 do_transform([{Key, Value} | More], Map, Opts) ->
-    do_transform(More, merge(hd(paths(hocon_token:value_of(Key))), unpack(Value, Opts), Map), Opts).
+    [KeyReal] = paths(hocon_token:value_of(Key)),
+    ValueReal = unpack(Value, Opts),
+    do_transform(More, merge(KeyReal, ValueReal, Map), Opts).
 
 unpack(#{?HOCON_T := object, ?HOCON_V := V} = O, #{format := richmap} = Opts) ->
     O#{?HOCON_V => do_transform(remove_nothing(V), #{}, Opts)};
@@ -361,9 +381,10 @@ paths(Key) when is_list(Key) ->
 
 merge(Key, Val, Map) when is_map(Val) ->
     case maps:find(Key, Map) of
-        {ok, MVal} when is_map(MVal) ->
-            maps:put(Key, deep_merge(MVal, Val), Map);
-        _Other -> maps:put(Key, Val, Map)
+        {ok, MVal} ->
+            maps:put(Key, hocon_util:deep_merge(MVal, Val), Map);
+        _Other ->
+            maps:put(Key, Val, Map)
     end;
 merge(Key, Val, Map) -> maps:put(Key, Val, Map).
 

src/hocon_cli.erl

@@ -143,8 +143,8 @@ get(ParsedArgs, [Query | _]) ->
     [RootName0 | _] = string:tokens(Query, "."),
     RootName = hocon_schema:resolve_struct_name(Schema, RootName0),
     %% do not log anything for `get` commands
-    DummyLogger = #{logger => fun(_, _) -> ok end},
-    {_, NewConf} = hocon_schema:map(Schema, Conf, [RootName], DummyLogger),
+    Opts = #{logger => fun(_, _) -> ok end, apply_override_envs => true},
+    {_, NewConf} = hocon_schema:map(Schema, Conf, [RootName], Opts),
     ?STDOUT("~0p", [hocon_schema:get_value(Query, NewConf)]),
     stop_ok().
 
@@ -251,7 +251,8 @@ generate(ParsedArgs) ->
                  true -> fun log_for_generator/2;
                  false -> fun(_, _) -> ok end
              end,
-    try hocon_schema:generate(Schema, Conf, #{logger => LogFun}) of
+    Opts = #{logger => LogFun, apply_override_envs => true},
+    try hocon_schema:generate(Schema, Conf, Opts) of
         NewConfig ->
             AppConfig = proplists:delete(vm_args, NewConfig),
             VmArgs = stringify(proplists:get_value(vm_args, NewConfig)),