Merge remote-tracking branch 'origin/master' into stream-timestamp-offsetstream-timestamp-offset

12 files changed, 1046 insertions, 0 deletions

diff --git a/workflow_sources/base_image/workflow.yml b/workflow_sources/base_image/workflow.yml
new file mode 100644
index 0000000000..e1b32c708b
--- /dev/null
+++ b/workflow_sources/base_image/workflow.yml
@@ -0,0 +1,49 @@
+#@ load("@ytt:data", "data")
+
+#@ def jobs_map_from_list(list_of_jobs):
+#@   jobs = {}
+#@   for job in list_of_jobs:
+#@     name = job['name']
+#@     jobs[name] = job
+#@   end
+#@   return jobs
+#@ end
+
+#@ def to_build_args(d):
+#@   return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
+#@ end
+
+#@ def base_image_job(erlang_version):
+name: #@ ("ci-base-" + erlang_version).replace(".", "_")
+runs-on: ubuntu-18.04
+steps:
+  - name: CHECKOUT REPOSITORY
+    uses: actions/checkout@v2
+  - name: #@ 'CREATE ERLANG+ELIXIR IMAGE ({})'.format(erlang_version)
+    uses: docker/build-push-action@v1
+    with:
+      username: _json_key
+      password: ${{ secrets.GCR_JSON_KEY }}
+      registry: eu.gcr.io
+      repository: cf-rabbitmq-core/erlang_elixir
+      dockerfile: #@ 'ci/dockerfiles/{}/erlang_elixir'.format(erlang_version)
+      tags: #@ erlang_version
+  - name: #@ 'CREATE BASE CI IMAGE ({})'.format(erlang_version)
+    uses: docker/build-push-action@v1
+    with:
+      username: _json_key
+      password: ${{ secrets.GCR_JSON_KEY }}
+      registry: eu.gcr.io
+      repository: cf-rabbitmq-core/ci-base
+      dockerfile: ci/dockerfiles/ci-base
+      #@ build_args = {}
+      #@ build_args["ERLANG_VERSION"] = erlang_version
+      #@ build_args["SECONDARY_UMBRELLA_GITREFS"] = ' '.join(data.values.secondary_umbrella_gitrefs)
+      build_args: #@ to_build_args(build_args)
+      tags: #@ erlang_version
+#@ end
+---
+name: Workflow Base Images
+#! https://github.com/k14s/ytt/issues/189
+a_magic_string_that_we_will_sed_to_on: [workflow_dispatch]
+jobs: #@ jobs_map_from_list([base_image_job(v) for v in data.values.erlang_versions])
diff --git a/workflow_sources/base_values.yml b/workflow_sources/base_values.yml
new file mode 100644
index 0000000000..e40c75bb16
--- /dev/null
+++ b/workflow_sources/base_values.yml
@@ -0,0 +1,10 @@
+#@data/values
+---
+base_rmq_ref: master
+secondary_umbrella_gitrefs:
+- v3.7.28
+- v3.8.9
+#! these values are injected at template time from the cli,
+#! but must be declared first here
+erlang_versions: []
+erlang_version: null
diff --git a/workflow_sources/deps.yml b/workflow_sources/deps.yml
new file mode 100644
index 0000000000..12f0699332
--- /dev/null
+++ b/workflow_sources/deps.yml
@@ -0,0 +1,431 @@
+#@data/values
+---
+#@overlay/match missing_ok=True
+deps:
+- name: rabbit_common
+  suites:
+  - name: unit_priority_queue
+  - name: unit
+  - name: worker_pool
+  - name: rabbit_env
+- name: rabbit
+  test_suites_in_parallel: true
+  suites:
+  - name: amqqueue_backward_compatibility
+    time: 63
+  - name: backing_queue
+    time: 320
+  - name: channel_interceptor
+    time: 103
+  - name: channel_operation_timeout
+    time: 104
+  - name: cluster
+    time: 85
+  - name: cluster_rename
+    time: 284
+  - name: clustering_management
+    time: 708
+  - name: config_schema
+    time: 81
+  - name: confirms_rejects
+    time: 138
+  - name: consumer_timeout
+    time: 123
+  - name: crashing_queues
+    time: 124
+  - name: dead_lettering
+    time: 173
+  - name: definition_import
+    time: 84
+  - name: disconnect_detected_during_alarm
+    time: 88
+  - name: dynamic_ha
+    time: 900
+  - name: dynamic_qq
+    time: 238
+  - name: eager_sync
+    time: 452
+  - name: feature_flags
+    time: 102
+  - name: lazy_queue
+    time: 125
+  - name: list_consumers_sanity_check
+    time: 86
+  - name: list_queues_online_and_offline
+    time: 85
+  - name: maintenance_mode
+    time: 185
+  - name: many_node_ha
+    time: 112
+  - name: message_size_limit
+    time: 78
+  - name: metrics
+    time: 160
+  - name: mirrored_supervisor
+    time: 82
+  - name: msg_store
+    time: 65
+  - name: peer_discovery_classic_config
+    time: 136
+  - name: peer_discovery_dns
+    time: 76
+  - name: per_user_connection_channel_limit
+    time: 69
+  - name: per_user_connection_channel_limit_partitions
+    time: 56
+  - name: per_user_connection_channel_tracking
+    time: 89
+  - name: per_user_connection_tracking
+    time: 132
+  - name: per_vhost_connection_limit
+    time: 252
+  - name: per_vhost_connection_limit_partitions
+    time: 123
+  - name: per_vhost_msg_store
+    time: 66
+  - name: per_vhost_queue_limit
+    time: 137
+  - name: policy
+    time: 86
+  - name: priority_queue
+    time: 243
+  - name: priority_queue_recovery
+    time: 90
+  - name: product_info
+    time: 93
+  - name: proxy_protocol
+    time: 78
+  - name: publisher_confirms_parallel
+    time: 126
+  - name: queue_length_limits
+    time: 84
+  - name: queue_master_location
+    time: 401
+  - name: queue_parallel
+    time: 260
+  - name: queue_type
+    time: 56
+  - name: quorum_queue
+    time: 687
+  - name: rabbit_confirms
+    time: 4
+  - name: rabbit_core_metrics_gc
+    time: 221
+  - name: rabbit_fifo
+    time: 79
+  - name: rabbit_fifo_int
+    time: 86
+  - name: rabbit_fifo_prop
+    time: 169
+  - name: rabbit_fifo_v0
+    time: 6
+  - name: rabbit_msg_record
+    time: 4
+  - name: rabbit_stream_queue
+    time: 592
+  - name: rabbitmq_queues_cli_integration
+    time: 85
+  - name: rabbitmqctl_integration
+    time: 100
+  - name: rabbitmqctl_shutdown
+    time: 67
+  - name: signal_handling
+    time: 164
+  - name: simple_ha
+    time: 426
+  - name: single_active_consumer
+    time: 78
+  - name: sync_detection
+    time: 125
+  - name: term_to_binary_compat_prop
+    time: 66
+  - name: topic_permission
+    time: 77
+  - name: unit_access_control
+    time: 90
+  - name: unit_access_control_authn_authz_context_propagation
+    time: 87
+  - name: unit_access_control_credential_validation
+    time: 74
+  - name: unit_amqp091_content_framing
+    time: 67
+  - name: unit_amqp091_server_properties
+    time: 95
+  - name: unit_app_management
+    time: 79
+  - name: unit_cluster_formation_locking_mocks
+    time: 56
+  - name: unit_collections
+    time: 61
+  - name: unit_config_value_encryption
+    time: 99
+  - name: unit_connection_tracking
+    time: 16
+  - name: unit_credit_flow
+    time: 81
+  - name: unit_disk_monitor
+    time: 80
+  - name: unit_disk_monitor_mocks
+    time: 76
+  - name: unit_file_handle_cache
+    time: 93
+  - name: unit_gen_server2
+    time: 80
+  - name: unit_gm
+    time: 65
+  - name: unit_log_config
+    time: 75
+  - name: unit_log_management
+    time: 104
+  - name: unit_operator_policy
+    time: 66
+  - name: unit_pg_local
+    time: 68
+  - name: unit_plugin_directories
+    time: 52
+  - name: unit_plugin_versioning
+    time: 69
+  - name: unit_policy_validators
+    time: 68
+  - name: unit_priority_queue
+    time: 70
+  - name: unit_queue_consumers
+    time: 73
+  - name: unit_stats_and_metrics
+    time: 88
+  - name: unit_supervisor2
+    time: 71
+  - name: unit_vm_memory_monitor
+    time: 81
+  - name: upgrade_preparation
+    time: 117
+  - name: vhost
+    time: 211
+- name: amqp_client
+  skip_dialyzer: true
+  suites:
+  - name: unit
+  - name: system
+- name: amqp10_client
+  skip_tests: true
+  suites:
+  - name: system
+  - name: msg
+- name: amqp10_common
+  suites:
+  - name: binary_generator
+- name: rabbitmq_amqp1_0
+  skip_dialyzer: true
+  skip_tests: true
+  suites:
+  - name: amqp10_client
+  - name: proxy_protocol
+  - name: command
+  - name: unit
+  - name: system
+- name: rabbitmq_auth_backend_cache
+  skip_dialyzer: true
+  suites:
+  - name: rabbit_auth_backend_cache
+  - name: rabbit_auth_cache
+  - name: config_schema
+- name: rabbitmq_auth_backend_http
+  skip_dialyzer: true
+  suites:
+  - name: unit
+  - name: auth
+  - name: config_schema
+- name: rabbitmq_auth_backend_ldap
+  skip_dialyzer: true
+  skip_tests: true
+  suites:
+  - name: unit
+  - name: system
+  - name: config_schema
+- name: rabbitmq_auth_backend_oauth2
+  skip_dialyzer: true
+  suites:
+  - name: wildcard_match
+  - name: unit
+  - name: scope
+  - name: system
+  - name: add_uaa_key_command
+- name: rabbitmq_auth_mechanism_ssl
+  suites: []
+- name: rabbitmq_aws
+  skip_dialyzer: true
+  suites: []
+- name: rabbitmq_cli
+  skip_dialyzer: true
+  suites: []
+- name: rabbitmq_codegen
+  skip_xref: true
+  skip_dialyzer: true
+  skip_tests: true
+  suites: []
+- name: rabbitmq_consistent_hash_exchange
+  skip_dialyzer: true
+  suites:
+  - name: rabbit_exchange_type_consistent_hash
+- name: rabbitmq_event_exchange
+  suites:
+  - name: unit
+  - name: system
+  - name: config_schema
+- name: rabbitmq_federation
+  skip_dialyzer: true
+  suites:
+  - name: unit_inbroker
+  - name: restart_federation_link_command
+  - name: rabbit_federation_status
+  - name: federation_status_command
+  - name: unit
+  - name: queue
+  - name: exchange
+- name: rabbitmq_federation_management
+  suites:
+  - name: federation_mgmt
+- name: rabbitmq_jms_topic_exchange
+  skip_dialyzer: true
+  suites:
+  - name: sjx_evaluation
+  - name: rjms_topic_selector
+  - name: rjms_topic_selector_unit
+- name: rabbitmq_management
+  skip_dialyzer: true
+  suites:
+  - name: cache
+  - name: stats
+  - name: rabbit_mgmt_rabbitmqadmin
+  - name: listener_config
+  - name: rabbit_mgmt_test_db
+  - name: rabbit_mgmt_only_http
+  - name: rabbit_mgmt_stats
+  - name: rabbit_mgmt_test_unit
+  - name: rabbit_mgmt_http
+  - name: clustering_prop
+  - name: clustering
+  - name: config_schema
+- name: rabbitmq_management_agent
+  skip_dialyzer: true
+  suites:
+  - name: metrics
+  - name: exometer_slide
+  - name: rabbit_mgmt_gc
+  - name: rabbit_mgmt_slide
+- name: rabbitmq_mqtt
+  skip_dialyzer: true
+  skip_tests: true
+  suites:
+  - name: util
+  - name: retainer
+  - name: proxy_protocol
+  - name: command
+  - name: reader
+  - name: mqtt_machine
+  - name: processor
+  - name: java
+  - name: auth
+  - name: cluster
+  - name: config_schema
+- name: rabbitmq_peer_discovery_common
+  suites:
+  - name: config_schema
+- name: rabbitmq_peer_discovery_aws
+  skip_dialyzer: true
+  skip_tests: true
+  suites:
+  - name: rabbitmq_peer_discovery_aws
+  - name: integration
+  - name: config_schema
+- name: rabbitmq_peer_discovery_k8s
+  skip_dialyzer: true
+  suites:
+  - name: rabbitmq_peer_discovery_k8s
+  - name: config_schema
+- name: rabbitmq_peer_discovery_consul
+  skip_dialyzer: true
+  suites:
+  - name: rabbitmq_peer_discovery_consul
+  - name: config_schema
+- name: rabbitmq_peer_discovery_etcd
+  skip_dialyzer: true
+  skip_tests: true
+  suites:
+  - name: unit
+  - name: system
+  - name: config_schema
+- name: rabbitmq_prometheus
+  skip_dialyzer: true
+  suites:
+  - name: rabbit_prometheus_http
+  - name: config_schema
+- name: rabbitmq_random_exchange
+  suites: []
+- name: rabbitmq_recent_history_exchange
+  skip_dialyzer: true
+  suites:
+  - name: system
+- name: rabbitmq_sharding
+  skip_dialyzer: true
+  suites: []
+- name: rabbitmq_shovel
+  skip_dialyzer: true
+  suites:
+  - name: amqp10_shovel
+  - name: configuration
+  - name: dynamic
+  - name: parameters
+  - name: config
+  - name: shovel_status_command
+  - name: delete_shovel_command
+  - name: amqp10_dynamic
+  - name: amqp10
+- name: rabbitmq_shovel_management
+  suites:
+  - name: http
+- name: rabbitmq_stomp
+  skip_dialyzer: true
+  suites:
+  - name: util
+  - name: proxy_protocol
+  - name: amqqueue
+  - name: command
+  - name: topic
+  - name: frame
+  - name: python
+  - name: connections
+  - name: config_schema
+- name: rabbitmq_top
+  suites: []
+- name: rabbitmq_tracing
+  suites:
+  - name: rabbit_tracing
+- name: rabbitmq_trust_store
+  skip_dialyzer: true
+  suites:
+  - name: system
+  - name: config_schema
+- name: rabbitmq_web_dispatch
+  skip_dialyzer: true
+  suites:
+  - name: rabbit_web_dispatch
+  - name: rabbit_web_dispatch_unit
+- name: rabbitmq_web_mqtt
+  skip_dialyzer: true
+  suites:
+  - name: proxy_protocol
+  - name: config_schema
+- name: rabbitmq_web_mqtt_examples
+  suites: []
+- name: rabbitmq_web_stomp
+  skip_dialyzer: true
+  suites:
+  - name: proxy_protocol
+  - name: raw_websocket
+  - name: cowboy_websocket
+  - name: unit
+  - name: config_schema
+  - name: amqp_stomp
+- name: rabbitmq_web_stomp_examples
+  suites: []
diff --git a/workflow_sources/test/ct.lib.yml b/workflow_sources/test/ct.lib.yml
new file mode 100644
index 0000000000..5a3666776d
--- /dev/null
+++ b/workflow_sources/test/ct.lib.yml
@@ -0,0 +1,133 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:assert", "assert")
+#@ load("util.star", "is_unique", "group_by_one", "to_build_args")
+#@ load("helpers.star", "ci_image_tag", "ci_dep_image", "skip_ci_condition")
+
+#@ def checks_job(dep):
+name: #@ dep.name + "-checks"
+needs: [prepare]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+outputs:
+  step_start: ${{ steps.buildevents.outputs.step_start }}
+#@yaml/text-templated-strings
+steps:
+  - name: RECORD STEP START
+    id: buildevents
+    run: |
+      echo "::set-output name=step_start::$(date +%s)"
+  - name: CHECKOUT REPOSITORY
+    uses: actions/checkout@v2
+  - name: VALIDATE KNOWN CT SUITES
+    env:
+      project: #@ dep.name
+    #@ suite_names = [suite.name for suite in dep.suites]
+    #@ None if is_unique(suite_names) else assert.fail('{} suite names are not unique'.format(dep.name))
+    run: |
+      ci/scripts/validate-workflow.sh (@= " ".join(suite_names) @)
+  - name: RUN CHECKS
+    uses: docker/build-push-action@v1
+    with:
+      username: _json_key
+      password: ${{ secrets.GCR_JSON_KEY }}
+      registry: eu.gcr.io
+      repository: cf-rabbitmq-core/ci-(@= dep.name @)
+      dockerfile: ci/dockerfiles/ci-dep
+      #@ build_args = {}
+      #@ build_args["IMAGE_TAG"] = ci_image_tag()
+      #@ build_args["BUILDEVENT_APIKEY"] = '${{ secrets.HONEYCOMB_API_KEY }}'
+      #@ build_args["project"] = dep.name
+      build_args: #@ to_build_args(build_args)
+      tags: (@= ci_image_tag() @)
+#@ end
+
+#@ def ct_suites_job(dep, group):
+name: #@ dep.name + "-ct-" + group["name"]
+needs:
+  - prepare
+  - #@ dep.name + "-checks"
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: PULL IMAGE
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_dep_image(dep.name) @)
+  #@ for suite in group["suites"]:
+  - name: #@ "RUN ct-" + suite
+    run: |
+      mkdir ct-(@= suite @)-logs && chmod 777 ct-(@= suite @)-logs
+      docker run \
+        --env project=(@= dep.name @) \
+        --env CT_SUITE=(@= suite @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --volume ${PWD}/ct-(@= suite @)-logs:/workspace/ct-logs \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/ct-suite.sh
+  - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+    #! so just run always
+    if: always()
+    with:
+      name: (@= dep.name @)-ct-(@= suite @)-logs
+      path: "ct-(@= suite @)-logs/*-ct-logs-*.tar.xz"
+  #@ if data.values.erlang_version == data.values.erlang_versions[0]:
+  #@ for version in data.values.secondary_umbrella_gitrefs:
+  #@   logs_dir = 'ct-{}-logs-{}'.format(suite, version)
+  - name: #@ "RUN ct-{} [mixed {}]".format(suite, version)
+    run: |
+      mkdir (@= logs_dir @) && chmod 777 (@= logs_dir @)
+      docker run \
+        --env project=(@= dep.name @) \
+        --env CT_SUITE=(@= suite @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env SECONDARY_UMBRELLA_VERSION=(@= version @) \
+        --volume ${PWD}/(@= logs_dir @):/workspace/ct-logs \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/ct-suite.sh
+  - name: #@ 'ON FAILURE UPLOAD TESTS LOGS ARTIFACT [mixed {}]'.format(version)
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+    #! so just run always
+    if: always()
+    with:
+      name: (@= dep.name @)-ct-(@= suite @)-logs-mixed-(@= version @)
+      path: "(@= logs_dir @)/*-ct-logs-*.tar.xz"
+  #@ end
+  #@ end
+  #@ end
+#@ end
+
+#@ def collect_job(dep):
+name: #@ dep.name
+needs: #@ [dep.name + "-checks"] + [dep.name + "-ct-" + group["name"] for group in group_by_one(dep.suites)]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: RECORD STEP FINISH
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_dep_image(dep.name) @)
+      docker run \
+        --env project=(@= dep.name @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env STEP_START=${{ needs.(@= dep.name @)-checks.outputs.step_start }} \
+        (@= ci_dep_image(dep.name) @) \
+        /workspace/rabbitmq/ci/scripts/collect.sh
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/dep.star b/workflow_sources/test/dep.star
new file mode 100644
index 0000000000..d56e2fe2f6
--- /dev/null
+++ b/workflow_sources/test/dep.star
@@ -0,0 +1,22 @@
+load("util.star", "group_by_one")
+load("rabbitmq_cli.lib.yml", "rabbitmq_cli_job")
+load("ct.lib.yml", "checks_job", "ct_suites_job", "collect_job")
+load("tests.lib.yml", "tests_job")
+
+def dep_jobs(dep):
+  jobs = {}
+  if not getattr(dep, "skip_tests", False):
+    if dep.name == "rabbitmq_cli":
+      jobs[dep.name] = rabbitmq_cli_job(dep)
+    elif getattr(dep, "test_suites_in_parallel", False):
+      jobs[dep.name + "-checks"] = checks_job(dep)
+      for group in group_by_one(dep.suites):
+        jobs[dep.name + "-ct-" + group["name"]] = ct_suites_job(dep, group)
+      end
+      jobs[dep.name] = collect_job(dep)
+    else:
+      jobs[dep.name] = tests_job(dep)
+    end
+  end
+  return jobs
+end
diff --git a/workflow_sources/test/finish.lib.yml b/workflow_sources/test/finish.lib.yml
new file mode 100644
index 0000000000..3d249a7f12
--- /dev/null
+++ b/workflow_sources/test/finish.lib.yml
@@ -0,0 +1,83 @@
+#@ load("@ytt:data", "data")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def gcs_path():
+#@   c = ['monorepo_github_actions_conclusions']
+#@   c.append('${{ github.sha }}')
+#@   c.append('${{ github.workflow }}')
+#@   return '/'.join(c)
+#@ end
+
+#@ def finish_jobs(prepare_jobs_names):
+package-generic-unix:
+  name: package-generic-unix
+  needs: [prepare]
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD STEP START
+      id: buildevents
+      run: |
+        echo "::set-output name=step_start::$(date +%s)"
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: PACKAGE GENERIC UNIX
+      run: |
+        gcloud auth configure-docker
+        docker pull (@= ci_image() @)
+        mkdir PACKAGES && chmod 777 PACKAGES
+        docker run \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          --volume ${PWD}/PACKAGES:/workspace/rabbitmq/PACKAGES \
+          (@= ci_image() @) \
+          ci/scripts/package_generic_unix.sh
+    - name: UPLOAD GENERIC UNIX PACKAGE ARTIFACT
+      #! https://github.com/marketplace/actions/upload-artifact
+      uses: actions/upload-artifact@v2-preview
+      with:
+        name: rabbitmq-server-generic-unix.tar.xz
+        path: "PACKAGES/rabbitmq-server-generic-unix-*.tar.xz"
+    - name: RECORD STEP FINISH
+      if: always()
+      run: |
+        docker run \
+          --env project=packaging \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          (@= ci_image() @) \
+          ci/scripts/collect.sh
+finish:
+  name: finish
+  needs: #@ prepare_jobs_names + [dep.name for dep in data.values.deps if not getattr(dep, "skip_tests", False)] + ['package-generic-unix']
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - uses: technote-space/workflow-conclusion-action@v1
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: RECORD BUILD FINISH
+      run: |
+        echo -n "${{ env.WORKFLOW_CONCLUSION }}" > conclusion
+
+        gsutil cp conclusion \
+          'gs://(@= gcs_path() @)'
+
+        gcloud auth configure-docker
+        docker pull (@= ci_image() @)
+        docker run \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env BUILD_START=${{ needs.prepare.outputs.build_start }} \
+          --env BUILD_RESULT=${{ env.WORKFLOW_CONCLUSION }} \
+          (@= ci_image() @) \
+          ci/scripts/finish.sh
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/helpers.star b/workflow_sources/test/helpers.star
new file mode 100644
index 0000000000..4db571163b
--- /dev/null
+++ b/workflow_sources/test/helpers.star
@@ -0,0 +1,17 @@
+load("@ytt:data", "data")
+
+def ci_image_tag():
+  return "erlang-" + data.values.erlang_version + "-rabbitmq-${{ github.sha }}"
+end
+
+def ci_image():
+  return "eu.gcr.io/cf-rabbitmq-core/ci:" + ci_image_tag()
+end
+
+def ci_dep_image(dep_name):
+  return "eu.gcr.io/cf-rabbitmq-core/ci-" + dep_name + ":" + ci_image_tag()
+end
+
+def skip_ci_condition():
+  return "!contains(github.event.head_commit.message, '[ci skip]')"
+end
diff --git a/workflow_sources/test/prepare.lib.yml b/workflow_sources/test/prepare.lib.yml
new file mode 100644
index 0000000000..da074a0ec6
--- /dev/null
+++ b/workflow_sources/test/prepare.lib.yml
@@ -0,0 +1,146 @@
+#@ load("@ytt:data", "data")
+#@ load("util.star", "to_build_args")
+#@ load("helpers.star", "ci_image", "ci_image_tag", "skip_ci_condition")
+
+#@ def prepare_jobs():
+prepare:
+  name: prepare
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  outputs:
+    build_start: ${{ steps.buildevents.outputs.build_start }}
+    branch_or_tag_name: ${{ steps.buildevents.outputs.branch_or_tag_name }}
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD BUILD START
+      id: buildevents
+      run: |
+        echo "::set-output name=build_start::$(date +%s)"
+        branch_or_tag_name=${GITHUB_REF#refs/*/}
+        echo "::set-output name=branch_or_tag_name::$branch_or_tag_name"
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2
+    - name: PREPARE BUILD IMAGE
+      uses: docker/build-push-action@v1
+      with:
+        username: _json_key
+        password: ${{ secrets.GCR_JSON_KEY }}
+        registry: eu.gcr.io
+        repository: cf-rabbitmq-core/ci
+        dockerfile: ci/dockerfiles/ci
+        #@ build_args = {}
+        #@ build_args["ERLANG_VERSION"] = data.values.erlang_version
+        #@ build_args["GITHUB_RUN_ID"] = '${{ github.run_id }}'
+        #@ build_args["BUILDEVENT_APIKEY"] = '${{ secrets.HONEYCOMB_API_KEY }}'
+        #@ build_args["GITHUB_SHA"] = '${{ github.sha }}'
+        #@ build_args["base_rmq_ref"] = data.values.base_rmq_ref
+        #@ build_args["current_rmq_ref"] = '${{ steps.buildevents.outputs.branch_or_tag_name }}'
+        #@ build_args["RABBITMQ_VERSION"] = data.values.base_rmq_ref.replace('master', '3.9.x').replace('.x', '.0')
+        build_args: #@ to_build_args(build_args)
+        tags: (@= ci_image_tag() @)
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: RECORD STEP FINISH
+      run: |
+        docker run \
+          --env project=prepare \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.build_start }} \
+          (@= ci_image() @) \
+          ci/scripts/collect.sh
+xref:
+  name: xref
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD XREF START
+      id: buildevents
+      run: |
+        echo "::set-output name=step_start::$(date +%s)"
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: PULL IMAGE
+      #@ base_image = "eu.gcr.io/cf-rabbitmq-core/ci-base:" + data.values.erlang_version
+      run: |
+        gcloud auth configure-docker
+        docker pull (@= base_image @)
+    #@ for dep in [d for d in data.values.deps if not getattr(d, "skip_xref", False)]:
+    - name: RUN XREF (@= dep.name @)
+      run: |
+        docker run \
+          --env project=(@= dep.name @) \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --volume ${PWD}:/workspace/rabbitmq \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/xref.sh
+    #@ end
+    - name: RECORD STEP FINISH
+      if: always()
+      run: |
+        docker run \
+          --env project=xref \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          --volume ${PWD}/ci/scripts:/workspace/rabbitmq/ci/scripts \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/collect.sh
+
+#@ if/end data.values.erlang_version == data.values.erlang_versions[-1]:
+dialyze:
+  name: dialyze
+  runs-on: ubuntu-18.04
+  if: #@ skip_ci_condition()
+  #@yaml/text-templated-strings
+  steps:
+    - name: RECORD DIALYZE START
+      id: buildevents
+      run: |
+        echo "::set-output name=step_start::$(date +%s)"
+    - name: CHECKOUT REPOSITORY
+      uses: actions/checkout@v2
+    - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      with:
+        service_account_key: ${{ secrets.GCR_JSON_KEY }}
+        export_default_credentials: true
+    - name: PULL IMAGE
+      #@ base_image = "eu.gcr.io/cf-rabbitmq-core/ci-base:" + data.values.erlang_version
+      run: |
+        gcloud auth configure-docker
+        docker pull (@= base_image @)
+    #@ for dep in [d for d in data.values.deps if not getattr(d, "skip_dialyzer", False)]:
+    - name: RUN DIALYZE (@= dep.name @)
+      run: |
+        docker run \
+          --env project=(@= dep.name @) \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --volume ${PWD}:/workspace/rabbitmq \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/dialyze.sh
+    #@ end
+    - name: RECORD STEP FINISH
+      if: always()
+      run: |
+        docker run \
+          --env project=dialyze \
+          --env GITHUB_RUN_ID=${{ github.run_id }} \
+          --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+          --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+          --volume ${PWD}/ci/scripts:/workspace/rabbitmq/ci/scripts \
+          --workdir /workspace/rabbitmq \
+          (@= base_image @) \
+          ci/scripts/collect.sh
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/rabbitmq_cli.lib.yml b/workflow_sources/test/rabbitmq_cli.lib.yml
new file mode 100644
index 0000000000..4281547814
--- /dev/null
+++ b/workflow_sources/test/rabbitmq_cli.lib.yml
@@ -0,0 +1,40 @@
+#@ load("@ytt:data", "data")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def rabbitmq_cli_job(dep):
+name: #@ dep.name
+needs:
+  - prepare
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - name: RECORD STEP START
+    id: buildevents
+    run: |
+      echo "::set-output name=step_start::$(date +%s)"
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: RUN TESTS
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_image() @)
+      mkdir broker-logs && chmod 777 broker-logs
+      docker run \
+        --env project=(@= dep.name @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+        --volume ${PWD}/broker-logs:/workspace/broker-logs \
+        (@= ci_image() @) \
+        ci/scripts/rabbitmq_cli.sh
+  - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    if: failure()
+    with:
+      name: (@= dep.name @)-broker-logs
+      path: "broker-logs/broker-logs.tar.xz"
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/tests.lib.yml b/workflow_sources/test/tests.lib.yml
new file mode 100644
index 0000000000..f6c7c05912
--- /dev/null
+++ b/workflow_sources/test/tests.lib.yml
@@ -0,0 +1,41 @@
+#@ load("@ytt:data", "data")
+#@ load("util.star", "to_build_args")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def tests_job(dep):
+name: #@ dep.name
+needs: [prepare]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+  - name: RECORD STEP START
+    id: buildevents
+    run: |
+      echo "::set-output name=step_start::$(date +%s)"
+  - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+    with:
+      service_account_key: ${{ secrets.GCR_JSON_KEY }}
+      export_default_credentials: true
+  - name: RUN TESTS
+    run: |
+      gcloud auth configure-docker
+      docker pull (@= ci_image() @)
+      mkdir ct-logs && chmod 777 ct-logs
+      docker run \
+        --env project=(@= dep.name @) \
+        --env GITHUB_RUN_ID=${{ github.run_id }} \
+        --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+        --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+        --volume ${PWD}/ct-logs:/workspace/ct-logs \
+        --oom-kill-disable \
+        (@= ci_image() @) \
+        ci/scripts/tests.sh
+  - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+    #! https://github.com/marketplace/actions/upload-artifact
+    uses: actions/upload-artifact@v2-preview
+    if: failure()
+    with:
+      name: (@= dep.name @)-ct-logs
+      path: "ct-logs/*-ct-logs-*.tar.xz"
+#@ end
\ No newline at end of file
diff --git a/workflow_sources/test/util.star b/workflow_sources/test/util.star
new file mode 100644
index 0000000000..2c6d348564
--- /dev/null
+++ b/workflow_sources/test/util.star
@@ -0,0 +1,58 @@
+def is_unique(l):
+  return len(l) == len(set(l))
+end
+
+def merge(dicts):
+  r = {}
+  for d in dicts:
+    r.update(**d)
+  end
+  return r
+end
+
+def name(suites):
+  if len(suites) == 1:
+    return suites[0].name
+  else:
+    return suites[0].name + "-plus-" + str(len(suites) - 1) + "-more"
+  end
+end
+
+def sum(ints):
+  s = 0
+  for i in ints:
+    s += i
+  end
+  return s
+end
+
+def partition(target, groups, suites):
+  if len(suites) == 0:
+    return groups
+  end
+  group = []
+  rest = []
+  for suite in sorted(suites, key=lambda suite: suite.time):
+    if sum([suite2.time for suite2 in group]) + suite.time <= target:
+      group.append(suite)
+    else:
+      rest.append(suite)
+    end
+  end
+  return partition(target, groups + [group], rest)
+end
+
+def group_by_time(suites):
+  longest = max([suite.time for suite in suites])
+  groups = partition(longest, [], suites)
+  return [{"name": name(suites), "suites": [suite.name for suite in suites]} for suites in groups]
+end
+
+# Used when we don't actually want multiple ct-suites per job
+def group_by_one(suites):
+  return [{"name": suite.name, "suites": [suite.name]} for suite in suites]
+end
+
+def to_build_args(d):
+  return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
+end
diff --git a/workflow_sources/test/workflow.yml b/workflow_sources/test/workflow.yml
new file mode 100644
index 0000000000..c3746b1944
--- /dev/null
+++ b/workflow_sources/test/workflow.yml
@@ -0,0 +1,16 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:template", "template")
+#@ load("util.star", "merge")
+#@ load("prepare.lib.yml", "prepare_jobs")
+#@ load("dep.star", "dep_jobs")
+#@ load("finish.lib.yml", "finish_jobs")
+
+#@ prepare = prepare_jobs()
+#@ deps = [dep_jobs(dep) for dep in data.values.deps]
+#@ finish = finish_jobs([prepare[k]['name'] for k in prepare])
+
+---
+name: #@ "Test - Erlang " + data.values.erlang_version
+#! https://github.com/k14s/ytt/issues/189
+a_magic_string_that_we_will_sed_to_on: push
+jobs: #@ merge([prepare] + deps + [finish])