diff options
Diffstat (limited to 'mk')
| -rw-r--r-- | mk/github-actions.mk | 158 | ||||
| -rw-r--r-- | mk/rabbitmq-mix.mk | 21 | ||||
| -rw-r--r-- | mk/stats.mk | 17 | ||||
| -rw-r--r-- | mk/topic-branches.mk | 48 |
4 files changed, 244 insertions, 0 deletions
diff --git a/mk/github-actions.mk b/mk/github-actions.mk new file mode 100644 index 0000000000..6a9989808b --- /dev/null +++ b/mk/github-actions.mk @@ -0,0 +1,158 @@ +YTT ?= /usr/local/bin/ytt + +$(YTT): + $(error Please install ytt from https://get-ytt.io/) + +VENDORED_COMPONENTS = rabbit_common \ + rabbit \ + amqp_client \ + amqp10_client \ + amqp10_common \ + rabbitmq_amqp1_0 \ + rabbitmq_auth_backend_cache \ + rabbitmq_auth_backend_http \ + rabbitmq_auth_backend_ldap \ + rabbitmq_auth_backend_oauth2 \ + rabbitmq_auth_mechanism_ssl \ + rabbitmq_aws \ + rabbitmq_cli \ + rabbitmq_codegen \ + rabbitmq_consistent_hash_exchange \ + rabbitmq_event_exchange \ + rabbitmq_federation \ + rabbitmq_federation_management \ + rabbitmq_jms_topic_exchange \ + rabbitmq_management \ + rabbitmq_management_agent \ + rabbitmq_mqtt \ + rabbitmq_peer_discovery_common \ + rabbitmq_peer_discovery_aws \ + rabbitmq_peer_discovery_k8s \ + rabbitmq_peer_discovery_consul \ + rabbitmq_peer_discovery_etcd \ + rabbitmq_prometheus \ + rabbitmq_random_exchange \ + rabbitmq_recent_history_exchange \ + rabbitmq_sharding \ + rabbitmq_shovel \ + rabbitmq_shovel_management \ + rabbitmq_stomp \ + rabbitmq_stream \ + rabbitmq_top \ + rabbitmq_tracing \ + rabbitmq_trust_store \ + rabbitmq_web_dispatch \ + rabbitmq_web_mqtt \ + rabbitmq_web_mqtt_examples \ + rabbitmq_web_stomp \ + rabbitmq_web_stomp_examples + +DEPS_YAML_FILE = workflow_sources/deps.yml + +define dep_yaml_chunk +$(eval SUITES := $(sort $(subst _SUITE.erl,,$(notdir $(wildcard deps/$(1)/test/*_SUITE.erl))))) +echo -n "\n- name: $(1)\n suites:$(if $(SUITES),$(foreach suite,$(SUITES),\n - name: $(suite)\n time: 0), [])" >> $(DEPS_YAML_FILE); +endef + +$(DEPS_YAML_FILE): + @echo -n "#@data/values\n---\n#@overlay/match missing_ok=True\ndeps:" > $@ + @$(foreach dep,$(VENDORED_COMPONENTS),$(call dep_yaml_chunk,$(dep))) + @cat $@ | git stripspace > $@.fixed && mv $@.fixed $@ + +ERLANG_VERSIONS = 22.3 23.1 +ERLANG_VERSIONS_YAML1 = [$(foreach v,$(ERLANG_VERSIONS),,"$(v)")] +UNFIXED := [," +ERLANG_VERSIONS_YAML2 = $(subst $(UNFIXED),[",$(ERLANG_VERSIONS_YAML1)) + +.github/workflows/base-images.yaml: $(YTT) $(wildcard workflow_sources/base_image/*) + ytt -f workflow_sources/base_image \ + -f workflow_sources/base_values.yml \ + --data-value-yaml erlang_versions='$(ERLANG_VERSIONS_YAML2)' \ + --output-files /tmp + cat /tmp/workflow.yml | sed s/a_magic_string_that_we_will_sed_to_on/on/ \ + > $@ + +.github/workflows/test-erlang-otp-%.yaml: $(YTT) $(DEPS_YAML_FILE) $(wildcard workflow_sources/test/*) + ytt -f workflow_sources/test \ + -f workflow_sources/base_values.yml \ + -f $(DEPS_YAML_FILE) \ + --data-value-yaml erlang_versions='$(ERLANG_VERSIONS_YAML2)' \ + --data-value erlang_version=$* \ + --output-files /tmp + cat /tmp/workflow.yml | sed s/a_magic_string_that_we_will_sed_to_on/on/ \ + > $@ + +monorepo-actions: \ + .github/workflows/base-images.yaml \ + $(foreach v,$(ERLANG_VERSIONS), .github/workflows/test-erlang-otp-$(v).yaml) + +DOCKER_REPO ?= eu.gcr.io/cf-rabbitmq-core + +CI_BASE_IMAGES = $(foreach v,$(ERLANG_VERSIONS),ci-base-image-$(v)) +.PHONY: $(CI_BASE_IMAGES) +$(CI_BASE_IMAGES): + docker build . \ + -f ci/dockerfiles/$(subst ci-base-image-,,$@)/base \ + -t $(DOCKER_REPO)/ci-base:$(subst ci-base-image-,,$@) + +.PHONY: ci-base-images +ci-base-images: $(CI_BASE_IMAGES) + +PUSHES = $(foreach v,$(ERLANG_VERSIONS),push-base-image-$(v)) +.PHONY: $(PUSHES) +$(PUSHES): + docker push $(DOCKER_REPO)/ci-base:$(subst push-base-image-,,$@) + +.PHONY: push-base-images +push-base-images: $(PUSHES) + +LOCAL_CI_GOALS = $(foreach dep,$(filter-out rabbitmq_cli,$(VENDORED_COMPONENTS)),ci-$(dep)) +ERLANG_VERSION ?= 23.1 +SKIP_DIALYZE ?= False + +TAG = erlang-$(ERLANG_VERSION)-rabbitmq-$(shell git rev-parse HEAD)$(shell git diff-index --quiet HEAD -- || echo -dirty) +LOCAL_IMAGE = $(DOCKER_REPO)/ci:$(TAG) + +.PHONY: local-ci-image +local-ci-image: + docker build . \ + -f ci/dockerfiles/ci \ + -t $(LOCAL_IMAGE) \ + --build-arg ERLANG_VERSION=$(ERLANG_VERSION) \ + --build-arg GITHUB_RUN_ID=none \ + --build-arg BUILDEVENT_APIKEY=$(BUILDEVENT_APIKEY) \ + --build-arg GITHUB_SHA=$$(git rev-parse HEAD) \ + --build-arg base_rmq_ref=master \ + --build-arg current_rmq_ref=$$(git rev-parse --abbrev-ref HEAD) \ + --build-arg RABBITMQ_VERSION=3.9.0 + +.PHONY: $(LOCAL_CI_GOALS) +$(LOCAL_CI_GOALS): local-ci-image + docker run --rm \ + --env project=$(subst ci-,,$@) \ + --env SKIP_DIALYZE=$(SKIP_DIALYZE) \ + --env GITHUB_RUN_ID=none \ + --env BUILDEVENT_APIKEY=$(BUILDEVENT_APIKEY) \ + --env STEP_START=$$(date +%s) \ + --volume /tmp/ct-logs:/ct-logs \ + --oom-score-adj -500 \ + $(LOCAL_IMAGE) \ + /workspace/rabbitmq/ci/scripts/tests.sh + +ci-rabbitmq_cli: local-ci-image + docker run --rm \ + --env project=$(subst ci-,,$@) \ + --env SKIP_DIALYZE=$(SKIP_DIALYZE) \ + --env GITHUB_RUN_ID=none \ + --env BUILDEVENT_APIKEY=$(BUILDEVENT_APIKEY) \ + --env STEP_START=$$(date +%s) \ + --volume /tmp/broker-logs:/broker-logs \ + $(LOCAL_IMAGE) \ + /workspace/rabbitmq/ci/scripts/rabbitmq_cli.sh + +.PHONY: docker +docker: local-ci-image + docker run --rm -it \ + --oom-score-adj -500 \ + $(LOCAL_IMAGE) \ + /bin/bash diff --git a/mk/rabbitmq-mix.mk b/mk/rabbitmq-mix.mk new file mode 100644 index 0000000000..d946a819b6 --- /dev/null +++ b/mk/rabbitmq-mix.mk @@ -0,0 +1,21 @@ +# This file is copied to rabbitmq_cli (and possibly other Elixir-based +# components) when the RabbitMQ source archive is created, to allow +# those Elixir applications to build even with no access to Hex.pm, +# using the bundled sources only. + +HEX_OFFLINE := 1 + +# mix(1) centralizes its data in `$MIX_HOME`. When unset, it defaults +# to something under `$XDG_DATA_HOME`/`$XDG_CONFIG_HOME` or `$HOME` +# depending on the Elixir version. +# +# We store those data for offline build in `$(DEPS_DIR)`. + +override MIX_HOME := $(DEPS_DIR)/.mix + +# In addition to `$MIX_HOME`, we still have to set `$HOME` which is used +# to find `~/.hex` where the Hex.pm cache and packages are stored. + +override HOME := $(DEPS_DIR) + +export HEX_OFFLINE MIX_HOME HOME diff --git a/mk/stats.mk b/mk/stats.mk new file mode 100644 index 0000000000..1f4b59214f --- /dev/null +++ b/mk/stats.mk @@ -0,0 +1,17 @@ +# Used & tested on macOS + +DOCKER := /usr/local/bin/docker + +define STATS_CONTAINER +$(DOCKER) run \ + --interactive --tty --rm \ + --volume rabbitmq_releases:/home/node \ + node bash +endef +.PHONY: stats +stats: + $(verbose) $(STATS_CONTAINER) -c "cd /home/node && npm install rels && node_modules/.bin/rels --repo rabbitmq/rabbitmq-server --all" + +.PHONY: stats_interactive +stats_interactive: + $(verbose) $(STATS_CONTAINER) diff --git a/mk/topic-branches.mk b/mk/topic-branches.mk new file mode 100644 index 0000000000..fb279eb3c2 --- /dev/null +++ b/mk/topic-branches.mk @@ -0,0 +1,48 @@ +SCRATCH := $(CURDIR)/topic-branch-scratch + +GIT_FILTER_REPO := $(CURDIR)/bin/git-filter-repo + +$(GIT_FILTER_REPO): + mkdir -p $(TMPDIR) \ + && cd $(TMPDIR) \ + && curl -LO https://github.com/newren/git-filter-repo/releases/download/v2.28.0/git-filter-repo-2.28.0.tar.xz \ + && tar -xJf git-filter-repo-*.tar.xz \ + && mkdir -p $(CURDIR)/bin \ + && cp git-filter-repo-*/git-filter-repo $(GIT_FILTER_REPO) \ + && chmod +x $(GIT_FILTER_REPO) + +.PHONY: clean-state +clean-state: + @git diff-index --quiet HEAD -- \ + || (echo "Cannot proceed with uncommitted changes"; exit 1) + +PARENT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) + +define fetch_topic_branch +echo "Collecting commits from $(1)/$(2)..." \ +&& git clone --quiet git@github.com:rabbitmq/$(call rmq_cmp_repo_name,$(1)).git $(SCRATCH)/$(2)/repo-$(1) \ +&& cd $(SCRATCH)/$(2)/repo-$(1) \ +&& $(GIT_FILTER_REPO) --quiet --to-subdirectory-filter deps/$(1) \ +&& git checkout $(2) \ +&& git format-patch $(PARENT_BRANCH) \ +&& mkdir -p $(SCRATCH)/$(2)/$(1) \ +&& cp *.patch $(SCRATCH)/$(2)/$(1) \ +|| printf "Topic branch $(2) does not appear to exist in $(1).\n\n"; +endef + +define rebase_topic_branch +git am --reject $(sort $(wildcard $(1)/*.patch)); +endef + +fetch-topic-branch-%: $(GIT_FILTER_REPO) + $(eval TOPIC_BRANCH := $(subst fetch-topic-branch-,,$@)) + mkdir -p $(SCRATCH)/$(TOPIC_BRANCH) + @$(foreach dep,$(VENDORED_COMPONENTS),$(call fetch_topic_branch,$(dep),$(TOPIC_BRANCH))) + rm -rf $(SCRATCH)/$(TOPIC_BRANCH)/repo-* + +topic-branch-%: $(GIT_FILTER_REPO) clean-state + $(eval TOPIC_BRANCH := $(subst topic-branch-,,$@)) + ls $(SCRATCH)/$(TOPIC_BRANCH) \ + || (echo "Fetch the branch first with 'make fetch-$@')"; exit 1) + git checkout -b $(TOPIC_BRANCH) + @$(foreach dir,$(wildcard $(SCRATCH)/$(TOPIC_BRANCH)/*),$(call rebase_topic_branch,$(dir))) |