summaryrefslogtreecommitdiff
path: root/workflow_sources
diff options
context:
space:
mode:
Diffstat (limited to 'workflow_sources')
-rw-r--r--workflow_sources/base_image/workflow.yml49
-rw-r--r--workflow_sources/base_values.yml10
-rw-r--r--workflow_sources/deps.yml431
-rw-r--r--workflow_sources/test/ct.lib.yml133
-rw-r--r--workflow_sources/test/dep.star22
-rw-r--r--workflow_sources/test/finish.lib.yml83
-rw-r--r--workflow_sources/test/helpers.star17
-rw-r--r--workflow_sources/test/prepare.lib.yml146
-rw-r--r--workflow_sources/test/rabbitmq_cli.lib.yml40
-rw-r--r--workflow_sources/test/tests.lib.yml41
-rw-r--r--workflow_sources/test/util.star58
-rw-r--r--workflow_sources/test/workflow.yml16
12 files changed, 1046 insertions, 0 deletions
diff --git a/workflow_sources/base_image/workflow.yml b/workflow_sources/base_image/workflow.yml
new file mode 100644
index 0000000000..e1b32c708b
--- /dev/null
+++ b/workflow_sources/base_image/workflow.yml
@@ -0,0 +1,49 @@
+#@ load("@ytt:data", "data")
+
+#@ def jobs_map_from_list(list_of_jobs):
+#@ jobs = {}
+#@ for job in list_of_jobs:
+#@ name = job['name']
+#@ jobs[name] = job
+#@ end
+#@ return jobs
+#@ end
+
+#@ def to_build_args(d):
+#@ return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
+#@ end
+
+#@ def base_image_job(erlang_version):
+name: #@ ("ci-base-" + erlang_version).replace(".", "_")
+runs-on: ubuntu-18.04
+steps:
+ - name: CHECKOUT REPOSITORY
+ uses: actions/checkout@v2
+ - name: #@ 'CREATE ERLANG+ELIXIR IMAGE ({})'.format(erlang_version)
+ uses: docker/build-push-action@v1
+ with:
+ username: _json_key
+ password: ${{ secrets.GCR_JSON_KEY }}
+ registry: eu.gcr.io
+ repository: cf-rabbitmq-core/erlang_elixir
+ dockerfile: #@ 'ci/dockerfiles/{}/erlang_elixir'.format(erlang_version)
+ tags: #@ erlang_version
+ - name: #@ 'CREATE BASE CI IMAGE ({})'.format(erlang_version)
+ uses: docker/build-push-action@v1
+ with:
+ username: _json_key
+ password: ${{ secrets.GCR_JSON_KEY }}
+ registry: eu.gcr.io
+ repository: cf-rabbitmq-core/ci-base
+ dockerfile: ci/dockerfiles/ci-base
+ #@ build_args = {}
+ #@ build_args["ERLANG_VERSION"] = erlang_version
+ #@ build_args["SECONDARY_UMBRELLA_GITREFS"] = ' '.join(data.values.secondary_umbrella_gitrefs)
+ build_args: #@ to_build_args(build_args)
+ tags: #@ erlang_version
+#@ end
+---
+name: Workflow Base Images
+#! https://github.com/k14s/ytt/issues/189
+a_magic_string_that_we_will_sed_to_on: [workflow_dispatch]
+jobs: #@ jobs_map_from_list([base_image_job(v) for v in data.values.erlang_versions])
diff --git a/workflow_sources/base_values.yml b/workflow_sources/base_values.yml
new file mode 100644
index 0000000000..e40c75bb16
--- /dev/null
+++ b/workflow_sources/base_values.yml
@@ -0,0 +1,10 @@
+#@data/values
+---
+base_rmq_ref: master
+secondary_umbrella_gitrefs:
+- v3.7.28
+- v3.8.9
+#! these values are injected at template time from the cli,
+#! but must be declared first here
+erlang_versions: []
+erlang_version: null
diff --git a/workflow_sources/deps.yml b/workflow_sources/deps.yml
new file mode 100644
index 0000000000..12f0699332
--- /dev/null
+++ b/workflow_sources/deps.yml
@@ -0,0 +1,431 @@
+#@data/values
+---
+#@overlay/match missing_ok=True
+deps:
+- name: rabbit_common
+ suites:
+ - name: unit_priority_queue
+ - name: unit
+ - name: worker_pool
+ - name: rabbit_env
+- name: rabbit
+ test_suites_in_parallel: true
+ suites:
+ - name: amqqueue_backward_compatibility
+ time: 63
+ - name: backing_queue
+ time: 320
+ - name: channel_interceptor
+ time: 103
+ - name: channel_operation_timeout
+ time: 104
+ - name: cluster
+ time: 85
+ - name: cluster_rename
+ time: 284
+ - name: clustering_management
+ time: 708
+ - name: config_schema
+ time: 81
+ - name: confirms_rejects
+ time: 138
+ - name: consumer_timeout
+ time: 123
+ - name: crashing_queues
+ time: 124
+ - name: dead_lettering
+ time: 173
+ - name: definition_import
+ time: 84
+ - name: disconnect_detected_during_alarm
+ time: 88
+ - name: dynamic_ha
+ time: 900
+ - name: dynamic_qq
+ time: 238
+ - name: eager_sync
+ time: 452
+ - name: feature_flags
+ time: 102
+ - name: lazy_queue
+ time: 125
+ - name: list_consumers_sanity_check
+ time: 86
+ - name: list_queues_online_and_offline
+ time: 85
+ - name: maintenance_mode
+ time: 185
+ - name: many_node_ha
+ time: 112
+ - name: message_size_limit
+ time: 78
+ - name: metrics
+ time: 160
+ - name: mirrored_supervisor
+ time: 82
+ - name: msg_store
+ time: 65
+ - name: peer_discovery_classic_config
+ time: 136
+ - name: peer_discovery_dns
+ time: 76
+ - name: per_user_connection_channel_limit
+ time: 69
+ - name: per_user_connection_channel_limit_partitions
+ time: 56
+ - name: per_user_connection_channel_tracking
+ time: 89
+ - name: per_user_connection_tracking
+ time: 132
+ - name: per_vhost_connection_limit
+ time: 252
+ - name: per_vhost_connection_limit_partitions
+ time: 123
+ - name: per_vhost_msg_store
+ time: 66
+ - name: per_vhost_queue_limit
+ time: 137
+ - name: policy
+ time: 86
+ - name: priority_queue
+ time: 243
+ - name: priority_queue_recovery
+ time: 90
+ - name: product_info
+ time: 93
+ - name: proxy_protocol
+ time: 78
+ - name: publisher_confirms_parallel
+ time: 126
+ - name: queue_length_limits
+ time: 84
+ - name: queue_master_location
+ time: 401
+ - name: queue_parallel
+ time: 260
+ - name: queue_type
+ time: 56
+ - name: quorum_queue
+ time: 687
+ - name: rabbit_confirms
+ time: 4
+ - name: rabbit_core_metrics_gc
+ time: 221
+ - name: rabbit_fifo
+ time: 79
+ - name: rabbit_fifo_int
+ time: 86
+ - name: rabbit_fifo_prop
+ time: 169
+ - name: rabbit_fifo_v0
+ time: 6
+ - name: rabbit_msg_record
+ time: 4
+ - name: rabbit_stream_queue
+ time: 592
+ - name: rabbitmq_queues_cli_integration
+ time: 85
+ - name: rabbitmqctl_integration
+ time: 100
+ - name: rabbitmqctl_shutdown
+ time: 67
+ - name: signal_handling
+ time: 164
+ - name: simple_ha
+ time: 426
+ - name: single_active_consumer
+ time: 78
+ - name: sync_detection
+ time: 125
+ - name: term_to_binary_compat_prop
+ time: 66
+ - name: topic_permission
+ time: 77
+ - name: unit_access_control
+ time: 90
+ - name: unit_access_control_authn_authz_context_propagation
+ time: 87
+ - name: unit_access_control_credential_validation
+ time: 74
+ - name: unit_amqp091_content_framing
+ time: 67
+ - name: unit_amqp091_server_properties
+ time: 95
+ - name: unit_app_management
+ time: 79
+ - name: unit_cluster_formation_locking_mocks
+ time: 56
+ - name: unit_collections
+ time: 61
+ - name: unit_config_value_encryption
+ time: 99
+ - name: unit_connection_tracking
+ time: 16
+ - name: unit_credit_flow
+ time: 81
+ - name: unit_disk_monitor
+ time: 80
+ - name: unit_disk_monitor_mocks
+ time: 76
+ - name: unit_file_handle_cache
+ time: 93
+ - name: unit_gen_server2
+ time: 80
+ - name: unit_gm
+ time: 65
+ - name: unit_log_config
+ time: 75
+ - name: unit_log_management
+ time: 104
+ - name: unit_operator_policy
+ time: 66
+ - name: unit_pg_local
+ time: 68
+ - name: unit_plugin_directories
+ time: 52
+ - name: unit_plugin_versioning
+ time: 69
+ - name: unit_policy_validators
+ time: 68
+ - name: unit_priority_queue
+ time: 70
+ - name: unit_queue_consumers
+ time: 73
+ - name: unit_stats_and_metrics
+ time: 88
+ - name: unit_supervisor2
+ time: 71
+ - name: unit_vm_memory_monitor
+ time: 81
+ - name: upgrade_preparation
+ time: 117
+ - name: vhost
+ time: 211
+- name: amqp_client
+ skip_dialyzer: true
+ suites:
+ - name: unit
+ - name: system
+- name: amqp10_client
+ skip_tests: true
+ suites:
+ - name: system
+ - name: msg
+- name: amqp10_common
+ suites:
+ - name: binary_generator
+- name: rabbitmq_amqp1_0
+ skip_dialyzer: true
+ skip_tests: true
+ suites:
+ - name: amqp10_client
+ - name: proxy_protocol
+ - name: command
+ - name: unit
+ - name: system
+- name: rabbitmq_auth_backend_cache
+ skip_dialyzer: true
+ suites:
+ - name: rabbit_auth_backend_cache
+ - name: rabbit_auth_cache
+ - name: config_schema
+- name: rabbitmq_auth_backend_http
+ skip_dialyzer: true
+ suites:
+ - name: unit
+ - name: auth
+ - name: config_schema
+- name: rabbitmq_auth_backend_ldap
+ skip_dialyzer: true
+ skip_tests: true
+ suites:
+ - name: unit
+ - name: system
+ - name: config_schema
+- name: rabbitmq_auth_backend_oauth2
+ skip_dialyzer: true
+ suites:
+ - name: wildcard_match
+ - name: unit
+ - name: scope
+ - name: system
+ - name: add_uaa_key_command
+- name: rabbitmq_auth_mechanism_ssl
+ suites: []
+- name: rabbitmq_aws
+ skip_dialyzer: true
+ suites: []
+- name: rabbitmq_cli
+ skip_dialyzer: true
+ suites: []
+- name: rabbitmq_codegen
+ skip_xref: true
+ skip_dialyzer: true
+ skip_tests: true
+ suites: []
+- name: rabbitmq_consistent_hash_exchange
+ skip_dialyzer: true
+ suites:
+ - name: rabbit_exchange_type_consistent_hash
+- name: rabbitmq_event_exchange
+ suites:
+ - name: unit
+ - name: system
+ - name: config_schema
+- name: rabbitmq_federation
+ skip_dialyzer: true
+ suites:
+ - name: unit_inbroker
+ - name: restart_federation_link_command
+ - name: rabbit_federation_status
+ - name: federation_status_command
+ - name: unit
+ - name: queue
+ - name: exchange
+- name: rabbitmq_federation_management
+ suites:
+ - name: federation_mgmt
+- name: rabbitmq_jms_topic_exchange
+ skip_dialyzer: true
+ suites:
+ - name: sjx_evaluation
+ - name: rjms_topic_selector
+ - name: rjms_topic_selector_unit
+- name: rabbitmq_management
+ skip_dialyzer: true
+ suites:
+ - name: cache
+ - name: stats
+ - name: rabbit_mgmt_rabbitmqadmin
+ - name: listener_config
+ - name: rabbit_mgmt_test_db
+ - name: rabbit_mgmt_only_http
+ - name: rabbit_mgmt_stats
+ - name: rabbit_mgmt_test_unit
+ - name: rabbit_mgmt_http
+ - name: clustering_prop
+ - name: clustering
+ - name: config_schema
+- name: rabbitmq_management_agent
+ skip_dialyzer: true
+ suites:
+ - name: metrics
+ - name: exometer_slide
+ - name: rabbit_mgmt_gc
+ - name: rabbit_mgmt_slide
+- name: rabbitmq_mqtt
+ skip_dialyzer: true
+ skip_tests: true
+ suites:
+ - name: util
+ - name: retainer
+ - name: proxy_protocol
+ - name: command
+ - name: reader
+ - name: mqtt_machine
+ - name: processor
+ - name: java
+ - name: auth
+ - name: cluster
+ - name: config_schema
+- name: rabbitmq_peer_discovery_common
+ suites:
+ - name: config_schema
+- name: rabbitmq_peer_discovery_aws
+ skip_dialyzer: true
+ skip_tests: true
+ suites:
+ - name: rabbitmq_peer_discovery_aws
+ - name: integration
+ - name: config_schema
+- name: rabbitmq_peer_discovery_k8s
+ skip_dialyzer: true
+ suites:
+ - name: rabbitmq_peer_discovery_k8s
+ - name: config_schema
+- name: rabbitmq_peer_discovery_consul
+ skip_dialyzer: true
+ suites:
+ - name: rabbitmq_peer_discovery_consul
+ - name: config_schema
+- name: rabbitmq_peer_discovery_etcd
+ skip_dialyzer: true
+ skip_tests: true
+ suites:
+ - name: unit
+ - name: system
+ - name: config_schema
+- name: rabbitmq_prometheus
+ skip_dialyzer: true
+ suites:
+ - name: rabbit_prometheus_http
+ - name: config_schema
+- name: rabbitmq_random_exchange
+ suites: []
+- name: rabbitmq_recent_history_exchange
+ skip_dialyzer: true
+ suites:
+ - name: system
+- name: rabbitmq_sharding
+ skip_dialyzer: true
+ suites: []
+- name: rabbitmq_shovel
+ skip_dialyzer: true
+ suites:
+ - name: amqp10_shovel
+ - name: configuration
+ - name: dynamic
+ - name: parameters
+ - name: config
+ - name: shovel_status_command
+ - name: delete_shovel_command
+ - name: amqp10_dynamic
+ - name: amqp10
+- name: rabbitmq_shovel_management
+ suites:
+ - name: http
+- name: rabbitmq_stomp
+ skip_dialyzer: true
+ suites:
+ - name: util
+ - name: proxy_protocol
+ - name: amqqueue
+ - name: command
+ - name: topic
+ - name: frame
+ - name: python
+ - name: connections
+ - name: config_schema
+- name: rabbitmq_top
+ suites: []
+- name: rabbitmq_tracing
+ suites:
+ - name: rabbit_tracing
+- name: rabbitmq_trust_store
+ skip_dialyzer: true
+ suites:
+ - name: system
+ - name: config_schema
+- name: rabbitmq_web_dispatch
+ skip_dialyzer: true
+ suites:
+ - name: rabbit_web_dispatch
+ - name: rabbit_web_dispatch_unit
+- name: rabbitmq_web_mqtt
+ skip_dialyzer: true
+ suites:
+ - name: proxy_protocol
+ - name: config_schema
+- name: rabbitmq_web_mqtt_examples
+ suites: []
+- name: rabbitmq_web_stomp
+ skip_dialyzer: true
+ suites:
+ - name: proxy_protocol
+ - name: raw_websocket
+ - name: cowboy_websocket
+ - name: unit
+ - name: config_schema
+ - name: amqp_stomp
+- name: rabbitmq_web_stomp_examples
+ suites: []
diff --git a/workflow_sources/test/ct.lib.yml b/workflow_sources/test/ct.lib.yml
new file mode 100644
index 0000000000..5a3666776d
--- /dev/null
+++ b/workflow_sources/test/ct.lib.yml
@@ -0,0 +1,133 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:assert", "assert")
+#@ load("util.star", "is_unique", "group_by_one", "to_build_args")
+#@ load("helpers.star", "ci_image_tag", "ci_dep_image", "skip_ci_condition")
+
+#@ def checks_job(dep):
+name: #@ dep.name + "-checks"
+needs: [prepare]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+outputs:
+ step_start: ${{ steps.buildevents.outputs.step_start }}
+#@yaml/text-templated-strings
+steps:
+ - name: RECORD STEP START
+ id: buildevents
+ run: |
+ echo "::set-output name=step_start::$(date +%s)"
+ - name: CHECKOUT REPOSITORY
+ uses: actions/checkout@v2
+ - name: VALIDATE KNOWN CT SUITES
+ env:
+ project: #@ dep.name
+ #@ suite_names = [suite.name for suite in dep.suites]
+ #@ None if is_unique(suite_names) else assert.fail('{} suite names are not unique'.format(dep.name))
+ run: |
+ ci/scripts/validate-workflow.sh (@= " ".join(suite_names) @)
+ - name: RUN CHECKS
+ uses: docker/build-push-action@v1
+ with:
+ username: _json_key
+ password: ${{ secrets.GCR_JSON_KEY }}
+ registry: eu.gcr.io
+ repository: cf-rabbitmq-core/ci-(@= dep.name @)
+ dockerfile: ci/dockerfiles/ci-dep
+ #@ build_args = {}
+ #@ build_args["IMAGE_TAG"] = ci_image_tag()
+ #@ build_args["BUILDEVENT_APIKEY"] = '${{ secrets.HONEYCOMB_API_KEY }}'
+ #@ build_args["project"] = dep.name
+ build_args: #@ to_build_args(build_args)
+ tags: (@= ci_image_tag() @)
+#@ end
+
+#@ def ct_suites_job(dep, group):
+name: #@ dep.name + "-ct-" + group["name"]
+needs:
+ - prepare
+ - #@ dep.name + "-checks"
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: PULL IMAGE
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= ci_dep_image(dep.name) @)
+ #@ for suite in group["suites"]:
+ - name: #@ "RUN ct-" + suite
+ run: |
+ mkdir ct-(@= suite @)-logs && chmod 777 ct-(@= suite @)-logs
+ docker run \
+ --env project=(@= dep.name @) \
+ --env CT_SUITE=(@= suite @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --volume ${PWD}/ct-(@= suite @)-logs:/workspace/ct-logs \
+ (@= ci_dep_image(dep.name) @) \
+ /workspace/rabbitmq/ci/scripts/ct-suite.sh
+ - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+ #! https://github.com/marketplace/actions/upload-artifact
+ uses: actions/upload-artifact@v2-preview
+ #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+ #! so just run always
+ if: always()
+ with:
+ name: (@= dep.name @)-ct-(@= suite @)-logs
+ path: "ct-(@= suite @)-logs/*-ct-logs-*.tar.xz"
+ #@ if data.values.erlang_version == data.values.erlang_versions[0]:
+ #@ for version in data.values.secondary_umbrella_gitrefs:
+ #@ logs_dir = 'ct-{}-logs-{}'.format(suite, version)
+ - name: #@ "RUN ct-{} [mixed {}]".format(suite, version)
+ run: |
+ mkdir (@= logs_dir @) && chmod 777 (@= logs_dir @)
+ docker run \
+ --env project=(@= dep.name @) \
+ --env CT_SUITE=(@= suite @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env SECONDARY_UMBRELLA_VERSION=(@= version @) \
+ --volume ${PWD}/(@= logs_dir @):/workspace/ct-logs \
+ (@= ci_dep_image(dep.name) @) \
+ /workspace/rabbitmq/ci/scripts/ct-suite.sh
+ - name: #@ 'ON FAILURE UPLOAD TESTS LOGS ARTIFACT [mixed {}]'.format(version)
+ #! https://github.com/marketplace/actions/upload-artifact
+ uses: actions/upload-artifact@v2-preview
+ #! For whatever reason, "if: steps.(@= suite @).outcome == 'failure'" never executes,
+ #! so just run always
+ if: always()
+ with:
+ name: (@= dep.name @)-ct-(@= suite @)-logs-mixed-(@= version @)
+ path: "(@= logs_dir @)/*-ct-logs-*.tar.xz"
+ #@ end
+ #@ end
+ #@ end
+#@ end
+
+#@ def collect_job(dep):
+name: #@ dep.name
+needs: #@ [dep.name + "-checks"] + [dep.name + "-ct-" + group["name"] for group in group_by_one(dep.suites)]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: RECORD STEP FINISH
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= ci_dep_image(dep.name) @)
+ docker run \
+ --env project=(@= dep.name @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ needs.(@= dep.name @)-checks.outputs.step_start }} \
+ (@= ci_dep_image(dep.name) @) \
+ /workspace/rabbitmq/ci/scripts/collect.sh
+#@ end \ No newline at end of file
diff --git a/workflow_sources/test/dep.star b/workflow_sources/test/dep.star
new file mode 100644
index 0000000000..d56e2fe2f6
--- /dev/null
+++ b/workflow_sources/test/dep.star
@@ -0,0 +1,22 @@
+load("util.star", "group_by_one")
+load("rabbitmq_cli.lib.yml", "rabbitmq_cli_job")
+load("ct.lib.yml", "checks_job", "ct_suites_job", "collect_job")
+load("tests.lib.yml", "tests_job")
+
+def dep_jobs(dep):
+ jobs = {}
+ if not getattr(dep, "skip_tests", False):
+ if dep.name == "rabbitmq_cli":
+ jobs[dep.name] = rabbitmq_cli_job(dep)
+ elif getattr(dep, "test_suites_in_parallel", False):
+ jobs[dep.name + "-checks"] = checks_job(dep)
+ for group in group_by_one(dep.suites):
+ jobs[dep.name + "-ct-" + group["name"]] = ct_suites_job(dep, group)
+ end
+ jobs[dep.name] = collect_job(dep)
+ else:
+ jobs[dep.name] = tests_job(dep)
+ end
+ end
+ return jobs
+end
diff --git a/workflow_sources/test/finish.lib.yml b/workflow_sources/test/finish.lib.yml
new file mode 100644
index 0000000000..3d249a7f12
--- /dev/null
+++ b/workflow_sources/test/finish.lib.yml
@@ -0,0 +1,83 @@
+#@ load("@ytt:data", "data")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def gcs_path():
+#@ c = ['monorepo_github_actions_conclusions']
+#@ c.append('${{ github.sha }}')
+#@ c.append('${{ github.workflow }}')
+#@ return '/'.join(c)
+#@ end
+
+#@ def finish_jobs(prepare_jobs_names):
+package-generic-unix:
+ name: package-generic-unix
+ needs: [prepare]
+ runs-on: ubuntu-18.04
+ if: #@ skip_ci_condition()
+ #@yaml/text-templated-strings
+ steps:
+ - name: RECORD STEP START
+ id: buildevents
+ run: |
+ echo "::set-output name=step_start::$(date +%s)"
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: PACKAGE GENERIC UNIX
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= ci_image() @)
+ mkdir PACKAGES && chmod 777 PACKAGES
+ docker run \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+ --volume ${PWD}/PACKAGES:/workspace/rabbitmq/PACKAGES \
+ (@= ci_image() @) \
+ ci/scripts/package_generic_unix.sh
+ - name: UPLOAD GENERIC UNIX PACKAGE ARTIFACT
+ #! https://github.com/marketplace/actions/upload-artifact
+ uses: actions/upload-artifact@v2-preview
+ with:
+ name: rabbitmq-server-generic-unix.tar.xz
+ path: "PACKAGES/rabbitmq-server-generic-unix-*.tar.xz"
+ - name: RECORD STEP FINISH
+ if: always()
+ run: |
+ docker run \
+ --env project=packaging \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+ (@= ci_image() @) \
+ ci/scripts/collect.sh
+finish:
+ name: finish
+ needs: #@ prepare_jobs_names + [dep.name for dep in data.values.deps if not getattr(dep, "skip_tests", False)] + ['package-generic-unix']
+ runs-on: ubuntu-18.04
+ if: #@ skip_ci_condition()
+ #@yaml/text-templated-strings
+ steps:
+ - uses: technote-space/workflow-conclusion-action@v1
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: RECORD BUILD FINISH
+ run: |
+ echo -n "${{ env.WORKFLOW_CONCLUSION }}" > conclusion
+
+ gsutil cp conclusion \
+ 'gs://(@= gcs_path() @)'
+
+ gcloud auth configure-docker
+ docker pull (@= ci_image() @)
+ docker run \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env BUILD_START=${{ needs.prepare.outputs.build_start }} \
+ --env BUILD_RESULT=${{ env.WORKFLOW_CONCLUSION }} \
+ (@= ci_image() @) \
+ ci/scripts/finish.sh
+#@ end \ No newline at end of file
diff --git a/workflow_sources/test/helpers.star b/workflow_sources/test/helpers.star
new file mode 100644
index 0000000000..4db571163b
--- /dev/null
+++ b/workflow_sources/test/helpers.star
@@ -0,0 +1,17 @@
+load("@ytt:data", "data")
+
+def ci_image_tag():
+ return "erlang-" + data.values.erlang_version + "-rabbitmq-${{ github.sha }}"
+end
+
+def ci_image():
+ return "eu.gcr.io/cf-rabbitmq-core/ci:" + ci_image_tag()
+end
+
+def ci_dep_image(dep_name):
+ return "eu.gcr.io/cf-rabbitmq-core/ci-" + dep_name + ":" + ci_image_tag()
+end
+
+def skip_ci_condition():
+ return "!contains(github.event.head_commit.message, '[ci skip]')"
+end
diff --git a/workflow_sources/test/prepare.lib.yml b/workflow_sources/test/prepare.lib.yml
new file mode 100644
index 0000000000..da074a0ec6
--- /dev/null
+++ b/workflow_sources/test/prepare.lib.yml
@@ -0,0 +1,146 @@
+#@ load("@ytt:data", "data")
+#@ load("util.star", "to_build_args")
+#@ load("helpers.star", "ci_image", "ci_image_tag", "skip_ci_condition")
+
+#@ def prepare_jobs():
+prepare:
+ name: prepare
+ runs-on: ubuntu-18.04
+ if: #@ skip_ci_condition()
+ outputs:
+ build_start: ${{ steps.buildevents.outputs.build_start }}
+ branch_or_tag_name: ${{ steps.buildevents.outputs.branch_or_tag_name }}
+ #@yaml/text-templated-strings
+ steps:
+ - name: RECORD BUILD START
+ id: buildevents
+ run: |
+ echo "::set-output name=build_start::$(date +%s)"
+ branch_or_tag_name=${GITHUB_REF#refs/*/}
+ echo "::set-output name=branch_or_tag_name::$branch_or_tag_name"
+ - name: CHECKOUT REPOSITORY
+ uses: actions/checkout@v2
+ - name: PREPARE BUILD IMAGE
+ uses: docker/build-push-action@v1
+ with:
+ username: _json_key
+ password: ${{ secrets.GCR_JSON_KEY }}
+ registry: eu.gcr.io
+ repository: cf-rabbitmq-core/ci
+ dockerfile: ci/dockerfiles/ci
+ #@ build_args = {}
+ #@ build_args["ERLANG_VERSION"] = data.values.erlang_version
+ #@ build_args["GITHUB_RUN_ID"] = '${{ github.run_id }}'
+ #@ build_args["BUILDEVENT_APIKEY"] = '${{ secrets.HONEYCOMB_API_KEY }}'
+ #@ build_args["GITHUB_SHA"] = '${{ github.sha }}'
+ #@ build_args["base_rmq_ref"] = data.values.base_rmq_ref
+ #@ build_args["current_rmq_ref"] = '${{ steps.buildevents.outputs.branch_or_tag_name }}'
+ #@ build_args["RABBITMQ_VERSION"] = data.values.base_rmq_ref.replace('master', '3.9.x').replace('.x', '.0')
+ build_args: #@ to_build_args(build_args)
+ tags: (@= ci_image_tag() @)
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: RECORD STEP FINISH
+ run: |
+ docker run \
+ --env project=prepare \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.build_start }} \
+ (@= ci_image() @) \
+ ci/scripts/collect.sh
+xref:
+ name: xref
+ runs-on: ubuntu-18.04
+ if: #@ skip_ci_condition()
+ #@yaml/text-templated-strings
+ steps:
+ - name: RECORD XREF START
+ id: buildevents
+ run: |
+ echo "::set-output name=step_start::$(date +%s)"
+ - name: CHECKOUT REPOSITORY
+ uses: actions/checkout@v2
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: PULL IMAGE
+ #@ base_image = "eu.gcr.io/cf-rabbitmq-core/ci-base:" + data.values.erlang_version
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= base_image @)
+ #@ for dep in [d for d in data.values.deps if not getattr(d, "skip_xref", False)]:
+ - name: RUN XREF (@= dep.name @)
+ run: |
+ docker run \
+ --env project=(@= dep.name @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --volume ${PWD}:/workspace/rabbitmq \
+ --workdir /workspace/rabbitmq \
+ (@= base_image @) \
+ ci/scripts/xref.sh
+ #@ end
+ - name: RECORD STEP FINISH
+ if: always()
+ run: |
+ docker run \
+ --env project=xref \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+ --volume ${PWD}/ci/scripts:/workspace/rabbitmq/ci/scripts \
+ --workdir /workspace/rabbitmq \
+ (@= base_image @) \
+ ci/scripts/collect.sh
+
+#@ if/end data.values.erlang_version == data.values.erlang_versions[-1]:
+dialyze:
+ name: dialyze
+ runs-on: ubuntu-18.04
+ if: #@ skip_ci_condition()
+ #@yaml/text-templated-strings
+ steps:
+ - name: RECORD DIALYZE START
+ id: buildevents
+ run: |
+ echo "::set-output name=step_start::$(date +%s)"
+ - name: CHECKOUT REPOSITORY
+ uses: actions/checkout@v2
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: PULL IMAGE
+ #@ base_image = "eu.gcr.io/cf-rabbitmq-core/ci-base:" + data.values.erlang_version
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= base_image @)
+ #@ for dep in [d for d in data.values.deps if not getattr(d, "skip_dialyzer", False)]:
+ - name: RUN DIALYZE (@= dep.name @)
+ run: |
+ docker run \
+ --env project=(@= dep.name @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --volume ${PWD}:/workspace/rabbitmq \
+ --workdir /workspace/rabbitmq \
+ (@= base_image @) \
+ ci/scripts/dialyze.sh
+ #@ end
+ - name: RECORD STEP FINISH
+ if: always()
+ run: |
+ docker run \
+ --env project=dialyze \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+ --volume ${PWD}/ci/scripts:/workspace/rabbitmq/ci/scripts \
+ --workdir /workspace/rabbitmq \
+ (@= base_image @) \
+ ci/scripts/collect.sh
+#@ end \ No newline at end of file
diff --git a/workflow_sources/test/rabbitmq_cli.lib.yml b/workflow_sources/test/rabbitmq_cli.lib.yml
new file mode 100644
index 0000000000..4281547814
--- /dev/null
+++ b/workflow_sources/test/rabbitmq_cli.lib.yml
@@ -0,0 +1,40 @@
+#@ load("@ytt:data", "data")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def rabbitmq_cli_job(dep):
+name: #@ dep.name
+needs:
+ - prepare
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+ - name: RECORD STEP START
+ id: buildevents
+ run: |
+ echo "::set-output name=step_start::$(date +%s)"
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: RUN TESTS
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= ci_image() @)
+ mkdir broker-logs && chmod 777 broker-logs
+ docker run \
+ --env project=(@= dep.name @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+ --volume ${PWD}/broker-logs:/workspace/broker-logs \
+ (@= ci_image() @) \
+ ci/scripts/rabbitmq_cli.sh
+ - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+ #! https://github.com/marketplace/actions/upload-artifact
+ uses: actions/upload-artifact@v2-preview
+ if: failure()
+ with:
+ name: (@= dep.name @)-broker-logs
+ path: "broker-logs/broker-logs.tar.xz"
+#@ end \ No newline at end of file
diff --git a/workflow_sources/test/tests.lib.yml b/workflow_sources/test/tests.lib.yml
new file mode 100644
index 0000000000..f6c7c05912
--- /dev/null
+++ b/workflow_sources/test/tests.lib.yml
@@ -0,0 +1,41 @@
+#@ load("@ytt:data", "data")
+#@ load("util.star", "to_build_args")
+#@ load("helpers.star", "ci_image", "skip_ci_condition")
+
+#@ def tests_job(dep):
+name: #@ dep.name
+needs: [prepare]
+runs-on: ubuntu-18.04
+if: #@ skip_ci_condition()
+#@yaml/text-templated-strings
+steps:
+ - name: RECORD STEP START
+ id: buildevents
+ run: |
+ echo "::set-output name=step_start::$(date +%s)"
+ - uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+ with:
+ service_account_key: ${{ secrets.GCR_JSON_KEY }}
+ export_default_credentials: true
+ - name: RUN TESTS
+ run: |
+ gcloud auth configure-docker
+ docker pull (@= ci_image() @)
+ mkdir ct-logs && chmod 777 ct-logs
+ docker run \
+ --env project=(@= dep.name @) \
+ --env GITHUB_RUN_ID=${{ github.run_id }} \
+ --env BUILDEVENT_APIKEY=${{ secrets.HONEYCOMB_API_KEY }} \
+ --env STEP_START=${{ steps.buildevents.outputs.step_start }} \
+ --volume ${PWD}/ct-logs:/workspace/ct-logs \
+ --oom-kill-disable \
+ (@= ci_image() @) \
+ ci/scripts/tests.sh
+ - name: ON FAILURE UPLOAD TESTS LOGS ARTIFACT
+ #! https://github.com/marketplace/actions/upload-artifact
+ uses: actions/upload-artifact@v2-preview
+ if: failure()
+ with:
+ name: (@= dep.name @)-ct-logs
+ path: "ct-logs/*-ct-logs-*.tar.xz"
+#@ end \ No newline at end of file
diff --git a/workflow_sources/test/util.star b/workflow_sources/test/util.star
new file mode 100644
index 0000000000..2c6d348564
--- /dev/null
+++ b/workflow_sources/test/util.star
@@ -0,0 +1,58 @@
+def is_unique(l):
+ return len(l) == len(set(l))
+end
+
+def merge(dicts):
+ r = {}
+ for d in dicts:
+ r.update(**d)
+ end
+ return r
+end
+
+def name(suites):
+ if len(suites) == 1:
+ return suites[0].name
+ else:
+ return suites[0].name + "-plus-" + str(len(suites) - 1) + "-more"
+ end
+end
+
+def sum(ints):
+ s = 0
+ for i in ints:
+ s += i
+ end
+ return s
+end
+
+def partition(target, groups, suites):
+ if len(suites) == 0:
+ return groups
+ end
+ group = []
+ rest = []
+ for suite in sorted(suites, key=lambda suite: suite.time):
+ if sum([suite2.time for suite2 in group]) + suite.time <= target:
+ group.append(suite)
+ else:
+ rest.append(suite)
+ end
+ end
+ return partition(target, groups + [group], rest)
+end
+
+def group_by_time(suites):
+ longest = max([suite.time for suite in suites])
+ groups = partition(longest, [], suites)
+ return [{"name": name(suites), "suites": [suite.name for suite in suites]} for suites in groups]
+end
+
+# Used when we don't actually want multiple ct-suites per job
+def group_by_one(suites):
+ return [{"name": suite.name, "suites": [suite.name]} for suite in suites]
+end
+
+def to_build_args(d):
+ return ",".join(['{0}={1}'.format(k,d[k]) for k in d.keys()])
+end
diff --git a/workflow_sources/test/workflow.yml b/workflow_sources/test/workflow.yml
new file mode 100644
index 0000000000..c3746b1944
--- /dev/null
+++ b/workflow_sources/test/workflow.yml
@@ -0,0 +1,16 @@
+#@ load("@ytt:data", "data")
+#@ load("@ytt:template", "template")
+#@ load("util.star", "merge")
+#@ load("prepare.lib.yml", "prepare_jobs")
+#@ load("dep.star", "dep_jobs")
+#@ load("finish.lib.yml", "finish_jobs")
+
+#@ prepare = prepare_jobs()
+#@ deps = [dep_jobs(dep) for dep in data.values.deps]
+#@ finish = finish_jobs([prepare[k]['name'] for k in prepare])
+
+---
+name: #@ "Test - Erlang " + data.values.erlang_version
+#! https://github.com/k14s/ytt/issues/189
+a_magic_string_that_we_will_sed_to_on: push
+jobs: #@ merge([prepare] + deps + [finish])
View Lorry Controller Status