New "Feature flags" subsystem to ease upgrades with breaking changes

The goal is that each breaking change is implemented in way that a
RabbitMQ node having that change can work in the same cluster as other
nodes which do not know about it.

The feature flags are here to indicate which breaking changes are
"available" (i.e. the node knows about the breaking changes, but it
still runs in a backward-compatible mode) and which ones are currently
activated (i.e. the node uses the new code and older node can not be
part of the cluster anymore).

Enabling a feature flag (i.e. activating a breaking change) is a manual
operation where the user validates that the cluster is ready to switch
to the new world after the breaking change. Therefore the subsystem
ensures RabbitMQ nodes can talk to each other even if they don't have
the same code as long as they agree on using the common subset, and
prevents RabbitMQ nodes from talking to each other once the new code is
being used and one node does not understand it.

The consequence is that if a breaking change is implemented using this
new subsystem, a cluster can be upgraded one node at a time instead of
shutting down the entire cluster to upgrade.

Of course, the ability to implement a breaking change in such a way
entirely depends on the nature of that change. This new subsystem does
not guarantee that a cluster shutdown will never be required again.

[#159298729]

5 files changed, 40 insertions, 4 deletions

diff --git a/scripts/rabbitmq-env b/scripts/rabbitmq-env
index f4b06c171b..2b06dec2cd 100755
--- a/scripts/rabbitmq-env
+++ b/scripts/rabbitmq-env
@@ -278,6 +278,10 @@ rmq_normalize_path_var RABBITMQ_PID_FILE
 
 [ "x" = "x$RABBITMQ_BOOT_MODULE" ] && RABBITMQ_BOOT_MODULE=${BOOT_MODULE}
 
+[ "x" != "x$RABBITMQ_FEATURE_FLAGS_FILE" ] && RABBITMQ_FEATURE_FLAGS_FILE_source=environment
+[ "x" = "x$RABBITMQ_FEATURE_FLAGS_FILE" ] && RABBITMQ_FEATURE_FLAGS_FILE=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-feature_flags
+rmq_normalize_path_var RABBITMQ_FEATURE_FLAGS_FILE
+
 [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${PLUGINS_EXPAND_DIR}
 [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-plugins-expand
 rmq_normalize_path_var RABBITMQ_PLUGINS_EXPAND_DIR
@@ -310,6 +314,7 @@ rmq_check_if_shared_with_mnesia \
     RABBITMQ_CONFIG_FILE \
     RABBITMQ_LOG_BASE \
     RABBITMQ_PID_FILE \
+    RABBITMQ_FEATURE_FLAGS_FILE \
     RABBITMQ_PLUGINS_EXPAND_DIR \
     RABBITMQ_ENABLED_PLUGINS_FILE \
     RABBITMQ_PLUGINS_DIR \
@@ -320,21 +325,29 @@ rmq_check_if_shared_with_mnesia \
 
 ## Development-specific environment.
 if [ "${RABBITMQ_DEV_ENV}" ]; then
-    if [ "$RABBITMQ_PLUGINS_DIR_source" != 'environment' -o \
+    if [ "$RABBITMQ_FEATURE_FLAGS_FILE_source" != 'environment' -o \
+         "$RABBITMQ_PLUGINS_DIR_source" != 'environment' -o \
          "$RABBITMQ_ENABLED_PLUGINS_FILE_source" != 'environment' ]; then
         # We need to query the running node for the plugins directory
         # and the "enabled plugins" file.
         eval $( (${RABBITMQ_SCRIPTS_DIR}/rabbitmqctl eval \
-            '{ok, P} = application:get_env(rabbit, plugins_dir),
+            '{ok, F} = application:get_env(rabbit, feature_flags_file),
+             {ok, P} = application:get_env(rabbit, plugins_dir),
              {ok, E} = application:get_env(rabbit, enabled_plugins_file),
              B = os:getenv("RABBITMQ_MNESIA_BASE"),
              M = os:getenv("RABBITMQ_MNESIA_DIR"),
              io:format(
+               "feature_flags_file=\"~s\"~n"
                "plugins_dir=\"~s\"~n"
                "enabled_plugins_file=\"~s\"~n"
                "mnesia_base=\"~s\"~n"
-               "mnesia_dir=\"~s\"~n", [P, E, B, M]).' \
-            2>/dev/null | grep -E '^(plugins_dir|enabled_plugins_file|mnesia_base|mnesia_dir)=') || :)
+               "mnesia_dir=\"~s\"~n", [F, P, E, B, M]).' \
+            2>/dev/null | grep -E '^(feature_flags_file|plugins_dir|enabled_plugins_file|mnesia_base|mnesia_dir)=') || :)
+
+        if [ "${feature_flags_file}" -a \
+             "$RABBITMQ_FEATURE_FLAGS_FILE_source" != 'environment' ]; then
+            RABBITMQ_FEATURE_FLAGS_FILE="${feature_flags_file}"
+        fi
         if [ "${plugins_dir}" -a \
              "$RABBITMQ_PLUGINS_DIR_source" != 'environment' ]; then
             RABBITMQ_PLUGINS_DIR="${plugins_dir}"
diff --git a/scripts/rabbitmq-env.bat b/scripts/rabbitmq-env.bat
index 912668c8ac..783fc55287 100644
--- a/scripts/rabbitmq-env.bat
+++ b/scripts/rabbitmq-env.bat
@@ -284,6 +284,15 @@ if "!RABBITMQ_BOOT_MODULE!"=="" (
     )

 )

 

+REM [ "x" = "x$RABBITMQ_FEATURE_FLAGS_FILE" ] && RABBITMQ_FEATURE_FLAGS_FILE=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-feature_flags

+if "!RABBITMQ_FEATURE_FLAGS_FILE!"=="" (

+    if "!FEATURE_FLAGS_FILE!"=="" (

+        set RABBITMQ_FEATURE_FLAGS_FILE=!RABBITMQ_MNESIA_BASE!\!RABBITMQ_NODENAME!-feature_flags

+    ) else (

+        set RABBITMQ_FEATURE_FLAGS_FILE=!FEATURE_FLAGS_FILE!

+    )

+)

+

 REM [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${PLUGINS_EXPAND_DIR}

 REM [ "x" = "x$RABBITMQ_PLUGINS_EXPAND_DIR" ] && RABBITMQ_PLUGINS_EXPAND_DIR=${RABBITMQ_MNESIA_BASE}/${RABBITMQ_NODENAME}-plugins-expand

 if "!RABBITMQ_PLUGINS_EXPAND_DIR!"=="" (

@@ -397,6 +406,13 @@ if defined RABBITMQ_DEV_ENV (
     if "!SCRIPT_NAME!" == "rabbitmq-plugins" (

         REM We may need to query the running node for the plugins directory

         REM and the "enabled plugins" file.

+        if not "%RABBITMQ_FEATURE_FLAGS_FILE_source%" == "environment" (

+            for /f "delims=" %%F in ('!SCRIPT_DIR!\rabbitmqctl eval "{ok, P} = application:get_env(rabbit, feature_flags_file), io:format(""~s~n"", [P])."') do @set feature_flags_file=%%F

+            if exist "!feature_flags_file!" (

+                set RABBITMQ_FEATURE_FLAGS_FILE=!feature_flags_file!

+            )

+            REM set feature_flags_file=

+        )

         if not "%RABBITMQ_PLUGINS_DIR_source%" == "environment" (

             for /f "delims=" %%F in ('!SCRIPT_DIR!\rabbitmqctl eval "{ok, P} = application:get_env(rabbit, plugins_dir), io:format(""~s~n"", [P])."') do @set plugins_dir=%%F

             if exist "!plugins_dir!" (

@@ -476,6 +492,7 @@ exit /b
 REM Environment cleanup

 set BOOT_MODULE=

 set CONFIG_FILE=

+set FEATURE_FLAGS_FILE=

 set ENABLED_PLUGINS_FILE=

 set LOG_BASE=

 set MNESIA_BASE=

diff --git a/scripts/rabbitmq-server b/scripts/rabbitmq-server
index efff38c3af..aeac38e46f 100755
--- a/scripts/rabbitmq-server
+++ b/scripts/rabbitmq-server
@@ -195,6 +195,7 @@ RABBITMQ_DIST_PORT=$RABBITMQ_DIST_PORT \
     -s rabbit_prelaunch \
     ${RABBITMQ_NAME_TYPE} ${RABBITMQ_PRELAUNCH_NODENAME} \
     -conf_advanced "${RABBITMQ_ADVANCED_CONFIG_FILE}" \
+    -rabbit feature_flags_file "\"$RABBITMQ_FEATURE_FLAGS_FILE\"" \
     -rabbit enabled_plugins_file "\"$RABBITMQ_ENABLED_PLUGINS_FILE\"" \
     -rabbit plugins_dir "\"$RABBITMQ_PLUGINS_DIR\"" \
     -extra "${RABBITMQ_NODENAME}"
@@ -302,6 +303,7 @@ start_rabbitmq_server() {
         -rabbit lager_log_root "\"$RABBITMQ_LOG_BASE\"" \
         -rabbit lager_default_file "$RABBIT_LAGER_HANDLER" \
         -rabbit lager_upgrade_file "$RABBITMQ_LAGER_HANDLER_UPGRADE" \
+        -rabbit feature_flags_file "\"$RABBITMQ_FEATURE_FLAGS_FILE\"" \
         -rabbit enabled_plugins_file "\"$RABBITMQ_ENABLED_PLUGINS_FILE\"" \
         -rabbit plugins_dir "\"$RABBITMQ_PLUGINS_DIR\"" \
         -rabbit plugins_expand_dir "\"$RABBITMQ_PLUGINS_EXPAND_DIR\"" \
diff --git a/scripts/rabbitmq-server.bat b/scripts/rabbitmq-server.bat
index 9039243c62..d417091732 100644
--- a/scripts/rabbitmq-server.bat
+++ b/scripts/rabbitmq-server.bat
@@ -146,6 +146,7 @@ if "!RABBITMQ_CONFIG_FILE_NOEX!.conf" == "!RABBITMQ_CONFIG_FILE!" (
         -s rabbit_prelaunch ^

         !RABBITMQ_NAME_TYPE! rabbitmqprelaunch!RANDOM!!TIME:~9!@localhost ^

         -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^

+        -rabbit feature_flags_file "!RABBITMQ_FEATURE_FLAGS_FILE!" ^

         -rabbit enabled_plugins_file "!RABBITMQ_ENABLED_PLUGINS_FILE!" ^

         -rabbit plugins_dir "!RABBITMQ_PLUGINS_DIR!" ^

         -extra "!RABBITMQ_NODENAME!"

@@ -247,6 +248,7 @@ if "!ENV_OK!"=="false" (
 -rabbit lager_log_root \""!RABBITMQ_LOG_BASE:\=/!"\" ^

 -rabbit lager_default_file !RABBIT_LAGER_HANDLER! ^

 -rabbit lager_upgrade_file !RABBITMQ_LAGER_HANDLER_UPGRADE! ^

+-rabbit feature_flags_file \""!RABBITMQ_FEATURE_FLAGS_FILE:\=/!"\" ^

 -rabbit enabled_plugins_file \""!RABBITMQ_ENABLED_PLUGINS_FILE:\=/!"\" ^

 -rabbit plugins_dir \""!RABBITMQ_PLUGINS_DIR:\=/!"\" ^

 -rabbit plugins_expand_dir \""!RABBITMQ_PLUGINS_EXPAND_DIR:\=/!"\" ^

diff --git a/scripts/rabbitmq-service.bat b/scripts/rabbitmq-service.bat
index 08cc29f2c8..7bb1f124b5 100644
--- a/scripts/rabbitmq-service.bat
+++ b/scripts/rabbitmq-service.bat
@@ -229,6 +229,7 @@ if "!RABBITMQ_CONFIG_FILE_NOEX!.conf" == "!RABBITMQ_CONFIG_FILE!" (
         -s rabbit_prelaunch ^

         !RABBITMQ_NAME_TYPE! rabbitmqprelaunch!RANDOM!!TIME:~9!@localhost ^

         -conf_advanced "!RABBITMQ_ADVANCED_CONFIG_FILE!" ^

+        -rabbit feature_flags_file "!RABBITMQ_FEATURE_FLAGS_FILE!" ^

         -rabbit enabled_plugins_file "!RABBITMQ_ENABLED_PLUGINS_FILE!" ^

         -rabbit plugins_dir "!RABBITMQ_PLUGINS_DIR!" ^

         -extra "!RABBITMQ_NODENAME!"

@@ -320,6 +321,7 @@ set ERLANG_SERVICE_ARGUMENTS= ^
 -rabbit lager_log_root \""!RABBITMQ_LOG_BASE:\=/!"\" ^

 -rabbit lager_default_file !RABBIT_LAGER_HANDLER! ^

 -rabbit lager_upgrade_file !RABBITMQ_LAGER_HANDLER_UPGRADE! ^

+-rabbit feature_flags_file \""!RABBITMQ_FEATURE_FLAGS_FILE:\=/!"\" ^

 -rabbit enabled_plugins_file \""!RABBITMQ_ENABLED_PLUGINS_FILE:\=/!"\" ^

 -rabbit plugins_dir \""!RABBITMQ_PLUGINS_DIR:\=/!"\" ^

 -rabbit plugins_expand_dir \""!RABBITMQ_PLUGINS_EXPAND_DIR:\=/!"\" ^