Fixing bug in retrieving archived workflows from Google Cloud Store #20552

Workflow file for this run

.github/workflows/run-tests.yml at 122e619

	name: All Tests

	on:
	pull_request:
	push:
	branches:
	- main
	- cloud/*
	- feature/*
	- release/*

	permissions:
	contents: read

	concurrency: # Auto-cancel existing runs in the PR when a new commit is pushed
	group: run-tests-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	env:
	# For pull_request: use the head of the PR branch (not the merge branch which is the default!)
	# For push: use the pushed commit.
	COMMIT: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	PR_BASE_COMMIT: ${{ github.event.pull_request.base.sha }}
	DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml
	TEMPORAL_VERSION_CHECK_DISABLED: 1
	MAX_TEST_ATTEMPTS: 3
	SHARD_COUNT: 3 # NOTE: must match shard count in optimize-test-sharding.yml

	jobs:
	test-setup:
	name: Test setup
	runs-on: ubuntu-latest
	outputs:
	job_matrix: ${{ steps.build_matrix.outputs.job_matrix }}
	full_test_reason: ${{ steps.determine_scope.outputs.full_test_reason }}
	runner_arm: ${{ steps.configure_runners.outputs.runner_arm }}
	steps:
	- name: Checkout Code
	uses: actions/checkout@v6
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ env.COMMIT }}
	fetch-depth: 0

	- name: Fetch base branch
	if: ${{ github.event_name == 'pull_request' }}
	run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}

	- name: Compute merge base
	if: ${{ github.event_name == 'pull_request' }}
	run: \|
	MERGE_BASE="$(git merge-base "${{ env.COMMIT }}" "${{ github.event.pull_request.base.ref }}")"
	echo "MERGE_BASE=${MERGE_BASE}" >> "$GITHUB_ENV"

	# If possible, run the "abridged" jobs on PRs to save time.
	#
	# The goal is to still cover all persistence code paths:
	# 1. SQL
	# 2. NoSQL (ie Cassandra)
	# 3. Elasticsearch/OpenSearch
	# All other jobs for the remaining databases only run a small set of smoke tests.
	#
	# Exceptions to this optimization:
	# 1. commit is not a PR
	# 2. `test-all-dbs` label set on PR
	# 3. changes to persistence packages were made
	- name: Determine test scope
	id: determine_scope
	run: \|
	FULL_TEST_REASON=""

	event_name="${{ github.event_name }}"
	labels='${{ toJson(github.event.pull_request.labels.*.name) }}'
	merge_base="${MERGE_BASE:-}"

	echo "event_name=$event_name"
	echo "labels=$labels"
	echo "merge_base=$merge_base"

	# Push events (main, release branches) run all tests on all DBs
	if [[ "$event_name" == "push" ]]; then
	FULL_TEST_REASON="Running full tests on all DBs (push event)."
	# Check for test-all-dbs label
	elif echo "$labels" \| jq -e 'any(. == "test-all-dbs")' > /dev/null 2>&1; then
	FULL_TEST_REASON="Running full tests on all DBs (test-all-dbs label)."
	# Check for persistence code changes
	elif [[ -n "$merge_base" ]]; then
	echo "Changed persistence files:"
	git diff --name-only "$merge_base" "$COMMIT" \| grep -E "^(common/persistence/\|schema/)" \|\| echo "(none)"
	if git diff --name-only "$merge_base" "$COMMIT" \| grep -qE "^(common/persistence/\|schema/)"; then
	FULL_TEST_REASON="Running full tests on all DBs (persistence code changes)."
	fi
	fi

	echo "full_test_reason=$FULL_TEST_REASON" >> "$GITHUB_OUTPUT"

	- name: ${{ steps.determine_scope.outputs.full_test_reason && 'ℹ️ Full tests' \|\| 'ℹ️ Smoke tests' }}
	run: echo "::notice::${{ steps.determine_scope.outputs.full_test_reason \|\| 'Running smoke tests on extended DBs. Add the test-all-dbs label to run all tests on all DBs.' }}"

	- name: Configure runners
	id: configure_runners
	run: \|
	# Use 8-core runners for temporalio org, standard runners for forks
	if [[ "${{ github.repository_owner }}" == "temporalio" ]]; then
	runner_arm="ubuntu-24.04-arm64-8-cores"
	else
	runner_arm="ubuntu-24.04-arm"
	fi
	echo "runner_arm=$runner_arm" >> "$GITHUB_OUTPUT"

	# Primary DBs always get full tests, extended DBs get smoke tests (1 job) in abridged PRs.
	- name: Build job matrix
	id: build_matrix
	env:
	FULL_TEST_REASON: ${{ steps.determine_scope.outputs.full_test_reason }}
	DB_CONFIGS: \|
	cass_es:
	persistence_type: nosql
	persistence_driver: cassandra
	containers: [cassandra, elasticsearch]
	required: true
	cass_es8:
	persistence_type: nosql
	persistence_driver: cassandra
	containers: [cassandra, elasticsearch8]
	cass_os2:
	persistence_type: nosql
	persistence_driver: cassandra
	containers: [cassandra, opensearch2]
	cass_os3:
	persistence_type: nosql
	persistence_driver: cassandra
	containers: [cassandra, opensearch3]
	sqlite:
	persistence_type: sql
	persistence_driver: sqlite
	containers: []
	mysql8:
	persistence_type: sql
	persistence_driver: mysql8
	containers: [mysql]
	postgres12:
	persistence_type: sql
	persistence_driver: postgres12
	containers: [postgresql]
	required: true
	postgres12_pgx:
	persistence_type: sql
	persistence_driver: postgres12_pgx
	containers: [postgresql]
	JOB_TYPES: \|
	functest:
	cmd: make functional-test-coverage
	test_timeout: 35m
	github_timeout: 40
	sharded: true
	smoke:
	cmd: make functional-test-coverage
	test_timeout: 5m
	github_timeout: 10
	test_args: '"-run=TestActivityTestSuite\|TestSignalWorkflowTestSuite\|TestWorkflowTestSuite"'
	ndc:
	cmd: make functional-test-ndc-coverage
	test_timeout: 10m
	github_timeout: 15
	xdc:
	cmd: make functional-test-xdc-coverage
	test_timeout: 30m
	github_timeout: 35
	run: \|
	# Convert YAML inputs to JSON for jq.
	DBS_JSON=$(yq -o=json <<< "$DB_CONFIGS")
	JOBS_JSON=$(yq -o=json <<< "$JOB_TYPES")

	# Build the job matrix as a JSON array.
	MATRIX=$(
	jq -c -n \
	--argjson dbs "$DBS_JSON" \
	--argjson jobs "$JOBS_JSON" \
	--argjson shard_count "$SHARD_COUNT" \
	--arg full_test_reason "$FULL_TEST_REASON" \
	'
	[
	$dbs \| to_entries[] as $db \|

	# Full test suite for required DBs (or all DBs when full_test_reason is set).
	(if $db.value.required or ($full_test_reason != "") then
	$jobs \| to_entries[] \| select(.key != "smoke")
	else
	$jobs \| to_entries[] \| select(.key == "smoke")
	end) as $job \|

	# Base entry: merge db config + job config + name
	($db.value + ($job.value \| del(.sharded)) + {name: $db.key}) as $base \|

	# If sharded, emit one entry per shard; otherwise emit one entry
	if $job.value.sharded then
	range($shard_count) as $i \|
	$base + {
	display_name: "shard\($i)",
	shard_index: $i,
	total_shards: $shard_count
	}
	else
	$base + {display_name: $job.key}
	end
	]
	'
	)

	echo "job_matrix=$MATRIX" >> "$GITHUB_OUTPUT"
	echo "Generated $(jq length <<< "$MATRIX") jobs"

	pre-build:
	name: Pre-build for cache
	needs: test-setup
	runs-on: ${{ needs.test-setup.outputs.runner_arm }}
	steps:
	- uses: actions/checkout@v6
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ env.COMMIT }}

	- uses: actions/setup-go@v6
	with:
	go-version-file: "go.mod"
	cache: false # do our own caching

	- name: Restore dependencies
	id: restore-deps
	uses: actions/cache/restore@v4
	with:
	path: ~/go/pkg/mod
	key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }}

	- run: make pre-build-functional-test-coverage

	- name: Save dependencies
	uses: actions/cache/save@v4
	if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }}
	with:
	path: ~/go/pkg/mod
	key: ${{ steps.restore-deps.outputs.cache-primary-key }}

	- name: Save build outputs
	uses: actions/cache/save@v4
	with:
	path: ~/.cache/go-build
	key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}

	misc-checks:
	name: Misc checks
	needs: [pre-build, test-setup]
	runs-on: ${{ needs.test-setup.outputs.runner_arm }}
	steps:
	- uses: actions/checkout@v6
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ env.COMMIT }}
	# buf-breaking tries to compare HEAD against merge base so we need to be able to find it
	fetch-depth: 100

	- uses: actions/setup-go@v6
	with:
	go-version-file: "go.mod"
	cache: false # do our own caching

	- name: Restore dependencies
	uses: actions/cache/restore@v4
	with:
	path: ~/go/pkg/mod
	key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }}

	- name: Restore build outputs
	uses: actions/cache/restore@v4
	with:
	path: ~/.cache/go-build
	key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}

	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}

	- run: GOOS=windows GOARCH=amd64 make clean-bins bins

	- run: GOOS=darwin GOARCH=arm64 make clean-bins bins

	- run: make clean-bins ci-build-misc

	unit-test:
	name: Unit test
	needs: [pre-build, test-setup]
	runs-on: ${{ needs.test-setup.outputs.runner_arm }}
	steps:
	- uses: actions/checkout@v6
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ env.COMMIT }}

	- uses: actions/setup-go@v6
	with:
	go-version-file: "go.mod"
	cache: false # do our own caching

	- name: Restore dependencies
	uses: actions/cache/restore@v4
	with:
	path: ~/go/pkg/mod
	key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }}

	- name: Restore build outputs
	uses: actions/cache/restore@v4
	with:
	path: ~/.cache/go-build
	key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}

	- name: Run unit tests
	timeout-minutes: 20
	run: TEST_TIMEOUT=15m ./develop/github/monitor_test.sh make unit-test-coverage

	- name: Print memory snapshot
	if: always()
	run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi

	- name: Generate crash report
	if: failure() # if the tests failed, we would expect one JUnit XML report per attempt; otherwise it must have crashed
	run: \|
	[ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' \| wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
	CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash

	- name: Generate test summary
	uses: mikepenz/action-junit-report@v5.0.0-rc01
	if: failure()
	with:
	report_paths: ./.testoutput/junit.*.xml
	detailed_summary: true
	check_annotations: false
	annotate_only: true
	skip_annotations: true

	- name: Upload code coverage to Codecov
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./.testoutput
	flags: unit-test

	- name: Upload test results to Codecov
	if: ${{ !cancelled() }}
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./.testoutput
	flags: unit-test
	report_type: test_results

	- name: Upload test results to GitHub
	# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
	uses: actions/upload-artifact@v4.4.3
	if: ${{ !cancelled() }}
	with:
	name: junit-xml--${{github.run_id}}--${{github.run_attempt}}--unit-test
	path: ./.testoutput/junit.*.xml
	include-hidden-files: true
	retention-days: 28

	integration-test:
	name: Integration test
	needs: [pre-build, test-setup]
	runs-on: ${{ needs.test-setup.outputs.runner_arm }}
	steps:
	- uses: actions/checkout@v6
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ env.COMMIT }}

	- name: Start containerized dependencies
	uses: hoverkraft-tech/compose-action@v2.0.1
	with:
	compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
	services: \|
	cassandra
	mysql
	postgresql
	down-flags: -v

	- uses: actions/setup-go@v6
	with:
	go-version-file: "go.mod"
	cache: false # do our own caching

	- name: Restore dependencies
	uses: actions/cache/restore@v4
	with:
	path: ~/go/pkg/mod
	key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }}

	- name: Restore build outputs
	uses: actions/cache/restore@v4
	with:
	path: ~/.cache/go-build
	key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}

	- name: Run integration test
	timeout-minutes: 15
	run: ./develop/github/monitor_test.sh make integration-test-coverage

	- name: Print memory snapshot
	if: always()
	run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi

	- name: Generate crash report
	if: failure() # if the tests failed, we would expect one JUnit XML report per attempt; otherwise it must have crashed
	run: \|
	[ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' \| wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
	CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash

	- name: Generate test summary
	uses: mikepenz/action-junit-report@v5.0.0-rc01
	if: failure()
	with:
	report_paths: ./.testoutput/junit.*.xml
	detailed_summary: true
	check_annotations: false
	annotate_only: true
	skip_annotations: true

	- name: Upload code coverage to Codecov
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./.testoutput
	flags: integration-test

	- name: Upload test results to Codecov
	if: ${{ !cancelled() }}
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./.testoutput
	flags: integration-test
	report_type: test_results

	- name: Upload test results to GitHub
	# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
	uses: actions/upload-artifact@v4.4.3
	if: ${{ !cancelled() }}
	with:
	name: junit-xml--${{github.run_id}}--${{github.run_attempt}}--integration-test
	path: ./.testoutput/junit.*.xml
	include-hidden-files: true
	retention-days: 28

	- name: Tear down docker compose
	if: ${{ always() }}
	run: \|
	docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v

	# Root job name includes matrix details so it is unique per job variant.
	# This MUST stay in sync with the `job_name` passed to the job-id action below.
	functional-test:
	# Display name shown in the UI. The job-id lookup uses this exact value.
	name: Functional test (${{ matrix.name }}, ${{ matrix.display_name }})
	needs: [pre-build, test-setup]
	strategy:
	fail-fast: false
	matrix:
	include: ${{ fromJson(needs.test-setup.outputs.job_matrix) }}
	runs-on: ${{ needs.test-setup.outputs.runner_arm }}
	env:
	PERSISTENCE_TYPE: ${{ matrix.persistence_type }}
	PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }}
	TEST_TIMEOUT: ${{ matrix.test_timeout }}
	steps:
	- uses: ScribeMD/docker-cache@0.3.7
	with:
	key: docker-${{ runner.os }}${{ runner.arch }}-${{ hashFiles(env.DOCKER_COMPOSE_FILE) }}

	- uses: actions/checkout@v6
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	ref: ${{ env.COMMIT }}

	- name: Start containerized dependencies
	if: ${{ toJson(matrix.containers) != '[]' }}
	uses: hoverkraft-tech/compose-action@v2.0.1
	with:
	compose-file: ${{ env.DOCKER_COMPOSE_FILE }}
	services: "${{ join(matrix.containers, '\n') }}"
	down-flags: -v

	- uses: actions/setup-go@v6
	with:
	go-version-file: "go.mod"
	cache: false # do our own caching

	- name: Restore dependencies
	uses: actions/cache/restore@v4
	with:
	path: ~/go/pkg/mod
	key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }}

	- name: Restore build outputs
	uses: actions/cache/restore@v4
	with:
	path: ~/.cache/go-build
	key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }}

	- name: Get job ID
	id: get_job_id
	uses: ./.github/actions/get-job-id
	with:
	job_name: Functional test (${{ matrix.name }}, ${{ matrix.display_name }})
	run_id: ${{ github.run_id }}

	- name: ${{ matrix.display_name == 'smoke' && 'ℹ️ Smoke test' \|\| 'ℹ️ Full test' }}
	run: echo "::notice::${{ matrix.display_name == 'smoke' && 'This is a smoke test. Add the test-all-dbs label to run all tests on all DBs.' \|\| needs.test-setup.outputs.full_test_reason }}"

	- name: Run functional test
	timeout-minutes: ${{ matrix.github_timeout }}
	run: ./develop/github/monitor_test.sh ${{ matrix.cmd }}
	env:
	TEST_TOTAL_SHARDS: ${{ matrix.total_shards }}
	TEST_SHARD_INDEX: ${{ matrix.total_shards && matrix.shard_index }} # guard with total_shards to avoid falsy eval of shard_index=0
	TEST_ARGS: "${{ matrix.test_args }}"

	- name: Print memory snapshot
	if: always()
	run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi

	- name: Generate crash report
	if: failure() # if the tests failed, we would expect one JUnit XML report per attempt; otherwise it must have crashed
	run: \|
	[ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' \| wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] &&
	CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash

	- name: Generate test summary
	uses: mikepenz/action-junit-report@v5.0.0-rc01
	if: failure()
	with:
	report_paths: ./.testoutput/junit.*.xml
	detailed_summary: true
	check_annotations: false
	annotate_only: true
	skip_annotations: true

	- name: Upload code coverage to Codecov
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./.testoutput
	flags: functional-test

	- name: Upload test results to Codecov
	if: ${{ !cancelled() }}
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./.testoutput
	flags: functional-test
	report_type: test_results

	- name: Upload test results to GitHub
	# Can't pin to major because the action linter doesn't recognize the include-hidden-files flag.
	uses: actions/upload-artifact@v4.4.3
	if: ${{ !cancelled() }}
	with:
	name: junit-xml--${{ github.run_id }}--${{ steps.get_job_id.outputs.job_id }}--${{ github.run_attempt }}--${{ matrix.name }}--${{ matrix.display_name }}--functional-test
	path: ./.testoutput/junit.*.xml
	include-hidden-files: true
	retention-days: 28

	test-status:
	if: always()
	name: Test Status
	needs:
	- misc-checks
	- unit-test
	- integration-test
	- functional-test
	runs-on: ubuntu-latest
	env:
	RESULTS: ${{ toJSON(needs.*.result) }}
	steps:
	- name: Check results
	run: \|
	# all statuses must be success
	if [[ -n $(echo "$RESULTS" \| jq '.[] \| select (. != "success")') ]]; then
	exit 1
	fi

	notify-failure:
	name: Notify Slack on Failure
	if: \|
	always() &&
	github.ref == 'refs/heads/main' &&
	needs.test-status.result == 'failure'
	needs: test-status
	runs-on: ubuntu-latest
	permissions:
	contents: read
	actions: read
	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Go
	uses: actions/setup-go@v5
	with:
	go-version-file: "go.mod"
	cache: true

	- name: Send Slack notification
	env:
	SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
	GH_TOKEN: ${{ github.token }}
	run: \|
	go run ./cmd/tools/ci-notify alert \
	--run-id "${{ github.run_id }}" \
	--slack-webhook "$SLACK_WEBHOOK"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Fixing bug in retrieving archived workflows from Google Cloud Store #20552

Workflow file

Fixing bug in retrieving archived workflows from Google Cloud Store #20552

Uh oh!

Workflow file for this run