rabbitmq_prelaunch: Run rabbit_prelaunch_conf:setup/1 earlier

This fixes the issue where applications' configuration is applied after
they are started (and they do not read their environment again after
startup). This is the case of applications such as `ra` or
`sysmon_handler`: they are dependencies of `rabbit` and the Erlang
application controller will start them before.

Now, the configuration is loaded during the first prelaunch phase by
rabbitmq_prelaunch, hopefully before those applications are started.

To permit this change, the code updating the `enabled_plugins_file` was
moved to its own module. This one can't be moved to the
rabbitmq_prelaunch application because it depends on `rabbit_plugins`.

While here, add a couple assertions by checking return values.

2 files changed, 493 insertions, 1 deletions

diff --git a/apps/rabbitmq_prelaunch/src/rabbit_prelaunch.erl b/apps/rabbitmq_prelaunch/src/rabbit_prelaunch.erl
index 5c3d56cd50..aba76c197c 100644
--- a/apps/rabbitmq_prelaunch/src/rabbit_prelaunch.erl
+++ b/apps/rabbitmq_prelaunch/src/rabbit_prelaunch.erl
@@ -105,7 +105,10 @@ do_run() ->
     %% 2. Erlang distribution check + start.
     ok = rabbit_prelaunch_dist:setup(Context),
 
-    %% 3. Write PID file.
+    %% 3. Configuration check + loading.
+    ok = rabbit_prelaunch_conf:setup(Context),
+
+    %% 4. Write PID file.
     rabbit_log_prelaunch:debug(""),
     _ = write_pid_file(Context),
     ignore.
diff --git a/apps/rabbitmq_prelaunch/src/rabbit_prelaunch_conf.erl b/apps/rabbitmq_prelaunch/src/rabbit_prelaunch_conf.erl
new file mode 100644
index 0000000000..928cc45e14
--- /dev/null
+++ b/apps/rabbitmq_prelaunch/src/rabbit_prelaunch_conf.erl
@@ -0,0 +1,489 @@
+-module(rabbit_prelaunch_conf).
+
+-include_lib("kernel/include/file.hrl").
+-include_lib("stdlib/include/zip.hrl").
+
+-include_lib("rabbit_common/include/rabbit.hrl").
+
+-export([setup/1,
+         get_config_state/0,
+         generate_config_from_cuttlefish_files/3,
+         decrypt_config/1]).
+
+-ifdef(TEST).
+-export([decrypt_config/2]).
+-endif.
+
+setup(Context) ->
+    rabbit_log_prelaunch:debug(""),
+    rabbit_log_prelaunch:debug("== Configuration =="),
+
+    %% TODO: Check if directories/files are inside Mnesia dir.
+
+    %% TODO: Support glob patterns & directories in RABBITMQ_CONFIG_FILE.
+    %% TODO: Always try parsing of both erlang and cuttlefish formats.
+
+    ok = set_default_config(),
+
+    AdvancedConfigFile = find_actual_advanced_config_file(Context),
+    State = case find_actual_main_config_file(Context) of
+                {MainConfigFile, erlang} ->
+                    Config = load_erlang_term_based_config_file(
+                               MainConfigFile),
+                    Apps = [App || {App, _} <- Config],
+                    decrypt_config(Apps),
+                    #{config_type => erlang,
+                      config_files => [MainConfigFile],
+                      config_advanced_file => undefined};
+                {MainConfigFile, cuttlefish} ->
+                    ConfigFiles = [MainConfigFile],
+                    Config = load_cuttlefish_config_file(Context,
+                                                         ConfigFiles,
+                                                         AdvancedConfigFile),
+                    Apps = [App || {App, _} <- Config],
+                    decrypt_config(Apps),
+                    #{config_type => cuttlefish,
+                      config_files => ConfigFiles,
+                      config_advanced_file => AdvancedConfigFile};
+                undefined when AdvancedConfigFile =/= undefined ->
+                    rabbit_log_prelaunch:warning(
+                      "Using RABBITMQ_ADVANCED_CONFIG_FILE: ~s",
+                      [AdvancedConfigFile]),
+                    Config = load_erlang_term_based_config_file(
+                               AdvancedConfigFile),
+                    Apps = [App || {App, _} <- Config],
+                    decrypt_config(Apps),
+                    #{config_type => erlang,
+                      config_files => [AdvancedConfigFile],
+                      config_advanced_file => AdvancedConfigFile};
+                undefined ->
+                    #{config_type => undefined,
+                      config_files => [],
+                      config_advanced_file => undefined}
+            end,
+    ok = override_with_hard_coded_critical_config(),
+    rabbit_log_prelaunch:debug(
+      "Saving config state to application env: ~p", [State]),
+    store_config_state(State).
+
+store_config_state(ConfigState) ->
+    persistent_term:put({rabbitmq_prelaunch, config_state}, ConfigState).
+
+get_config_state() ->
+    persistent_term:get({rabbitmq_prelaunch, config_state}, undefined).
+
+%% -------------------------------------------------------------------
+%% Configuration loading.
+%% -------------------------------------------------------------------
+
+set_default_config() ->
+    rabbit_log_prelaunch:debug("Setting default config"),
+    Config = [
+              {ra,
+               [
+                %% Use a larger segments size for queues.
+                {segment_max_entries, 32768},
+                {wal_max_size_bytes, 536870912} %% 5 * 2 ^ 20
+               ]},
+              {sysmon_handler,
+               [{process_limit, 100},
+                {port_limit, 100},
+                {gc_ms_limit, 0},
+                {schedule_ms_limit, 0},
+                {heap_word_limit, 0},
+                {busy_port, false},
+                {busy_dist_port, true}]}
+             ],
+    apply_erlang_term_based_config(Config).
+
+find_actual_main_config_file(#{main_config_file := File}) ->
+    case filelib:is_regular(File) of
+        true ->
+            Format = case filename:extension(File) of
+                ".conf"   -> cuttlefish;
+                ".config" -> erlang;
+                _         -> determine_config_format(File)
+            end,
+            {File, Format};
+        false ->
+            OldFormatFile = File ++ ".config",
+            NewFormatFile = File ++ ".conf",
+            case filelib:is_regular(OldFormatFile) of
+                true ->
+                    case filelib:is_regular(NewFormatFile) of
+                        true ->
+                            rabbit_log_prelaunch:warning(
+                              "Both old (.config) and new (.conf) format config "
+                              "files exist."),
+                            rabbit_log_prelaunch:warning(
+                              "Using the old format config file: ~s",
+                              [OldFormatFile]),
+                            rabbit_log_prelaunch:warning(
+                              "Please update your config files to the new format "
+                              "and remove the old file."),
+                            ok;
+                        false ->
+                            ok
+                    end,
+                    {OldFormatFile, erlang};
+                false ->
+                    case filelib:is_regular(NewFormatFile) of
+                        true  -> {NewFormatFile, cuttlefish};
+                        false -> undefined
+                    end
+            end
+    end.
+
+find_actual_advanced_config_file(#{advanced_config_file := File}) ->
+    case filelib:is_regular(File) of
+        true  -> File;
+        false -> undefined
+    end.
+
+determine_config_format(File) ->
+    case filelib:file_size(File) of
+        0 ->
+            cuttlefish;
+        _ ->
+            case file:consult(File) of
+                {ok, _} -> erlang;
+                _       -> cuttlefish
+            end
+    end.
+
+load_erlang_term_based_config_file(ConfigFile) ->
+    rabbit_log_prelaunch:debug(
+      "Loading configuration file \"~ts\" (Erlang terms based)", [ConfigFile]),
+    case file:consult(ConfigFile) of
+        {ok, [Config]} when is_list(Config) ->
+            apply_erlang_term_based_config(Config),
+            Config;
+        {ok, OtherTerms} ->
+            rabbit_log_prelaunch:error(
+              "Failed to load configuration file \"~ts\", "
+              "incorrect format: ~p",
+              [ConfigFile, OtherTerms]),
+            throw({error, failed_to_parse_configuration_file});
+        {error, Reason} ->
+            rabbit_log_prelaunch:error(
+              "Failed to load configuration file \"~ts\": ~ts",
+              [ConfigFile, file:format_error(Reason)]),
+            throw({error, failed_to_read_configuration_file})
+    end.
+
+load_cuttlefish_config_file(Context,
+                            ConfigFiles,
+                            AdvancedConfigFile) ->
+    Config = generate_config_from_cuttlefish_files(
+               Context, ConfigFiles, AdvancedConfigFile),
+    apply_erlang_term_based_config(Config),
+    Config.
+
+generate_config_from_cuttlefish_files(Context,
+                                      ConfigFiles,
+                                      AdvancedConfigFile) ->
+    %% Load schemas.
+    SchemaFiles = find_cuttlefish_schemas(Context),
+    case SchemaFiles of
+        [] ->
+            rabbit_log_prelaunch:error(
+              "No configuration schema found~n", []),
+            throw({error, no_configuration_schema_found});
+        _ ->
+            rabbit_log_prelaunch:debug(
+              "Configuration schemas found:~n", []),
+            [rabbit_log_prelaunch:debug("  - ~ts", [SchemaFile])
+             || SchemaFile <- SchemaFiles],
+            ok
+    end,
+    Schema = cuttlefish_schema:files(SchemaFiles),
+
+    %% Load configuration.
+    rabbit_log_prelaunch:debug(
+      "Loading configuration files (Cuttlefish based):"),
+    [rabbit_log_prelaunch:debug(
+       "  - ~ts", [ConfigFile]) || ConfigFile <- ConfigFiles],
+    case cuttlefish_conf:files(ConfigFiles) of
+        {errorlist, Errors} ->
+            rabbit_log_prelaunch:error("Error generating configuration:", []),
+            [rabbit_log_prelaunch:error(
+               "  - ~ts",
+               [cuttlefish_error:xlate(Error)])
+             || Error <- Errors],
+            throw({error, failed_to_generate_configuration_file});
+        Config0 ->
+            %% Finalize configuration, based on the schema.
+            Config = case cuttlefish_generator:map(Schema, Config0) of
+                         {error, Phase, {errorlist, Errors}} ->
+                             %% TODO
+                             rabbit_log_prelaunch:error(
+                               "Error generating configuration in phase ~ts:",
+                               [Phase]),
+                             [rabbit_log_prelaunch:error(
+                                "  - ~ts",
+                                [cuttlefish_error:xlate(Error)])
+                              || Error <- Errors],
+                             throw(
+                               {error, failed_to_generate_configuration_file});
+                         ValidConfig ->
+                             proplists:delete(vm_args, ValidConfig)
+                     end,
+
+            %% Apply advanced configuration overrides, if any.
+            override_with_advanced_config(Config, AdvancedConfigFile)
+    end.
+
+find_cuttlefish_schemas(Context) ->
+    Apps = list_apps(Context),
+    rabbit_log_prelaunch:debug(
+      "Looking up configuration schemas in the following applications:"),
+    find_cuttlefish_schemas(Apps, []).
+
+find_cuttlefish_schemas([App | Rest], AllSchemas) ->
+    Schemas = list_schemas_in_app(App),
+    find_cuttlefish_schemas(Rest, AllSchemas ++ Schemas);
+find_cuttlefish_schemas([], AllSchemas) ->
+    lists:sort(fun(A,B) -> A < B end, AllSchemas).
+
+list_apps(#{os_type := {win32, _}, plugins_path := PluginsPath}) ->
+    PluginsDirs = string:lexemes(PluginsPath, ";"),
+    list_apps1(PluginsDirs, []);
+list_apps(#{plugins_path := PluginsPath}) ->
+    PluginsDirs = string:lexemes(PluginsPath, ":"),
+    list_apps1(PluginsDirs, []).
+
+
+list_apps1([Dir | Rest], Apps) ->
+    case file:list_dir(Dir) of
+        {ok, Filenames} ->
+            NewApps = [list_to_atom(
+                         hd(
+                           string:split(filename:basename(F, ".ex"), "-")))
+                       || F <- Filenames],
+            Apps1 = lists:umerge(Apps, lists:sort(NewApps)),
+            list_apps1(Rest, Apps1);
+        {error, Reason} ->
+            rabbit_log_prelaunch:debug(
+              "Failed to list directory \"~ts\" content: ~ts",
+              [Dir, file:format_error(Reason)]),
+            list_apps1(Rest, Apps)
+    end;
+list_apps1([], AppInfos) ->
+    AppInfos.
+
+list_schemas_in_app(App) ->
+    {Loaded, Unload} = case application:load(App) of
+                           ok                           -> {true, true};
+                           {error, {already_loaded, _}} -> {true, false};
+                           {error, _}                   -> {false, false}
+                       end,
+    List = case Loaded of
+               true ->
+                   case code:priv_dir(App) of
+                       {error, bad_name} ->
+                           rabbit_log_prelaunch:debug(
+                             "  [ ] ~s (no readable priv dir)", [App]),
+                           [];
+                       PrivDir ->
+                           SchemaDir = filename:join([PrivDir, "schema"]),
+                           do_list_schemas_in_app(App, SchemaDir)
+                   end;
+               false ->
+                   rabbit_log_prelaunch:debug(
+                     "  [ ] ~s (failed to load application)", [App]),
+                   []
+           end,
+    case Unload of
+        true  -> application:unload(App);
+        false -> ok
+    end,
+    List.
+
+do_list_schemas_in_app(App, SchemaDir) ->
+    case erl_prim_loader:list_dir(SchemaDir) of
+        {ok, Files} ->
+            rabbit_log_prelaunch:debug("  [x] ~s", [App]),
+            [filename:join(SchemaDir, File)
+             || [C | _] = File <- Files,
+                C =/= $.];
+        error ->
+            rabbit_log_prelaunch:debug(
+              "  [ ] ~s (no readable schema dir)", [App]),
+            []
+    end.
+
+override_with_advanced_config(Config, undefined) ->
+    Config;
+override_with_advanced_config(Config, AdvancedConfigFile) ->
+    rabbit_log_prelaunch:debug(
+      "Override with advanced configuration file \"~ts\"",
+      [AdvancedConfigFile]),
+    case file:consult(AdvancedConfigFile) of
+        {ok, [AdvancedConfig]} ->
+            cuttlefish_advanced:overlay(Config, AdvancedConfig);
+        {ok, OtherTerms} ->
+            rabbit_log_prelaunch:error(
+              "Failed to load advanced configuration file \"~ts\", "
+              "incorrect format: ~p",
+              [AdvancedConfigFile, OtherTerms]),
+            throw({error, failed_to_parse_advanced_configuration_file});
+        {error, Reason} ->
+            rabbit_log_prelaunch:error(
+              "Failed to load advanced configuration file \"~ts\": ~ts",
+              [AdvancedConfigFile, file:format_error(Reason)]),
+            throw({error, failed_to_read_advanced_configuration_file})
+    end.
+
+override_with_hard_coded_critical_config() ->
+    rabbit_log_prelaunch:debug("Override with hard-coded critical config"),
+    Config = [
+              {ra,
+               %% Make Ra use a custom logger that dispatches to lager
+               %% instead of the default OTP logger
+               [{logger_module, rabbit_log_ra_shim}]}
+             ],
+    apply_erlang_term_based_config(Config).
+
+apply_erlang_term_based_config([{_, []} | Rest]) ->
+    apply_erlang_term_based_config(Rest);
+apply_erlang_term_based_config([{App, Vars} | Rest]) ->
+    rabbit_log_prelaunch:debug("  Applying configuration for '~s':", [App]),
+    ok = apply_app_env_vars(App, Vars),
+    apply_erlang_term_based_config(Rest);
+apply_erlang_term_based_config([]) ->
+    ok.
+
+apply_app_env_vars(App, [{Var, Value} | Rest]) ->
+    rabbit_log_prelaunch:debug(
+      "    - ~s = ~p",
+      [Var, Value]),
+    ok = application:set_env(App, Var, Value, [{persistent, true}]),
+    apply_app_env_vars(App, Rest);
+apply_app_env_vars(_, []) ->
+    ok.
+
+%% -------------------------------------------------------------------
+%% Config decryption.
+%% -------------------------------------------------------------------
+
+decrypt_config(Apps) ->
+    rabbit_log_prelaunch:debug("Decoding encrypted config values (if any)"),
+    ConfigEntryDecoder = application:get_env(rabbit, config_entry_decoder, []),
+    decrypt_config(Apps, ConfigEntryDecoder).
+
+decrypt_config([], _) ->
+    ok;
+decrypt_config([App | Apps], Algo) ->
+    Algo1 = decrypt_app(App, application:get_all_env(App), Algo),
+    decrypt_config(Apps, Algo1).
+
+decrypt_app(_, [], Algo) ->
+    Algo;
+decrypt_app(App, [{Key, Value} | Tail], Algo) ->
+    Algo2 = try
+                case decrypt(Value, Algo) of
+                    {Value, Algo1} ->
+                        Algo1;
+                    {NewValue, Algo1} ->
+                        rabbit_log_prelaunch:debug(
+                          "Value of `~s` decrypted", [Key]),
+                        ok = application:set_env(App, Key, NewValue,
+                                                 [{persistent, true}]),
+                        Algo1
+                end
+            catch
+                throw:{bad_config_entry_decoder, _} = Error ->
+                    throw(Error);
+                _:Msg ->
+                    throw({config_decryption_error, {key, Key}, Msg})
+            end,
+    decrypt_app(App, Tail, Algo2).
+
+decrypt({encrypted, EncValue},
+        {Cipher, Hash, Iterations, PassPhrase} = Algo) ->
+    {rabbit_pbe:decrypt_term(Cipher, Hash, Iterations, PassPhrase, EncValue),
+     Algo};
+decrypt({encrypted, _} = Value,
+        ConfigEntryDecoder)
+  when is_list(ConfigEntryDecoder) ->
+    Algo = config_entry_decoder_to_algo(ConfigEntryDecoder),
+    decrypt(Value, Algo);
+decrypt(List, Algo) when is_list(List) ->
+    decrypt_list(List, Algo, []);
+decrypt(Value, Algo) ->
+    {Value, Algo}.
+
+%% We make no distinction between strings and other lists.
+%% When we receive a string, we loop through each element
+%% and ultimately return the string unmodified, as intended.
+decrypt_list([], Algo, Acc) ->
+    {lists:reverse(Acc), Algo};
+decrypt_list([{Key, Value} | Tail], Algo, Acc)
+  when Key =/= encrypted ->
+    {Value1, Algo1} = decrypt(Value, Algo),
+    decrypt_list(Tail, Algo1, [{Key, Value1} | Acc]);
+decrypt_list([Value | Tail], Algo, Acc) ->
+    {Value1, Algo1} = decrypt(Value, Algo),
+    decrypt_list(Tail, Algo1, [Value1 | Acc]).
+
+config_entry_decoder_to_algo(ConfigEntryDecoder) ->
+    case get_passphrase(ConfigEntryDecoder) of
+        undefined ->
+            throw({bad_config_entry_decoder, missing_passphrase});
+        PassPhrase ->
+            {
+             proplists:get_value(
+               cipher, ConfigEntryDecoder, rabbit_pbe:default_cipher()),
+             proplists:get_value(
+               hash, ConfigEntryDecoder, rabbit_pbe:default_hash()),
+             proplists:get_value(
+               iterations, ConfigEntryDecoder, rabbit_pbe:default_iterations()),
+             PassPhrase
+            }
+    end.
+
+get_passphrase(ConfigEntryDecoder) ->
+    rabbit_log_prelaunch:debug("Getting encrypted config passphrase"),
+    case proplists:get_value(passphrase, ConfigEntryDecoder) of
+        prompt ->
+            IoDevice = get_input_iodevice(),
+            io:setopts(IoDevice, [{echo, false}]),
+            PP = lists:droplast(io:get_line(IoDevice,
+                "\nPlease enter the passphrase to unlock encrypted "
+                "configuration entries.\n\nPassphrase: ")),
+            io:setopts(IoDevice, [{echo, true}]),
+            io:format(IoDevice, "~n", []),
+            PP;
+        {file, Filename} ->
+            {ok, File} = file:read_file(Filename),
+            [PP|_] = binary:split(File, [<<"\r\n">>, <<"\n">>]),
+            PP;
+        PP ->
+            PP
+    end.
+
+%% This function retrieves the correct IoDevice for requesting
+%% input. The problem with using the default IoDevice is that
+%% the Erlang shell prevents us from getting the input.
+%%
+%% Instead we therefore look for the io process used by the
+%% shell and if it can't be found (because the shell is not
+%% started e.g with -noshell) we use the 'user' process.
+%%
+%% This function will not work when either -oldshell or -noinput
+%% options are passed to erl.
+get_input_iodevice() ->
+    case whereis(user) of
+        undefined ->
+            user;
+        User ->
+            case group:interfaces(User) of
+                [] ->
+                    user;
+                [{user_drv, Drv}] ->
+                    case user_drv:interfaces(Drv) of
+                        []                          -> user;
+                        [{current_group, IoDevice}] -> IoDevice
+                    end
+            end
+    end.