Skip to content

Daily Full Interoperability Tests #101

Daily Full Interoperability Tests

Daily Full Interoperability Tests #101

name: Daily Full Interoperability Tests
on:
schedule:
- cron: '0 2 * * *' # 2 AM UTC daily
workflow_dispatch:
inputs:
# Transport parameters
transport-test-select:
description: '[Transport] Select specific tests (pipe-separated substrings)'
required: false
default: ''
type: string
transport-test-ignore:
description: '[Transport] Ignore specific tests (pipe-separated substrings)'
required: false
default: '~failing'
type: string
transport-force-matrix-rebuild:
description: '[Transport] Force test matrix regeneration'
required: false
default: false
type: boolean
transport-force-image-rebuild:
description: '[Transport] Force Docker image rebuilds'
required: false
default: false
type: boolean
transport-debug:
description: '[Transport] Enable debug mode'
required: false
default: false
type: boolean
# Hole-punch parameters
hole-punch-test-select:
description: '[Hole-punch] Select specific tests (pipe-separated substrings)'
required: false
default: ''
type: string
hole-punch-test-ignore:
description: '[Hole-punch] Ignore specific tests (pipe-separated substrings)'
required: false
default: '~failing'
type: string
hole-punch-relay-select:
description: '[Hole-punch] Select specific relays (pipe-separated substrings)'
required: false
default: ''
type: string
hole-punch-relay-ignore:
description: '[Hole-punch] Ignore specific relays (pipe-separated substrings)'
required: false
default: '~failing'
type: string
hole-punch-router-select:
description: '[Hole-punch] Select specific routers (pipe-separated substrings)'
required: false
default: ''
type: string
hole-punch-router-ignore:
description: '[Hole-punch] Ignore specific routers (pipe-separated substrings)'
required: false
default: '~failing'
type: string
hole-punch-force-matrix-rebuild:
description: '[Hole-punch] Force test matrix regeneration'
required: false
default: false
type: boolean
hole-punch-force-image-rebuild:
description: '[Hole-punch] Force Docker image rebuilds'
required: false
default: false
type: boolean
hole-punch-debug:
description: '[Hole-punch] Enable debug mode'
required: false
default: false
type: boolean
# Perf parameters
perf-test-select:
description: '[Perf] Select specific implementations to test (pipe-separated substrings)'
required: false
default: ''
type: string
perf-test-ignore:
description: '[Perf] Ignore specific implementations (pipe-separated substrings)'
required: false
default: '~failing'
type: string
perf-baseline-select:
description: '[Perf] Select specific baseline tests (pipe-separated substrings)'
required: false
default: ''
type: string
perf-baseline-ignore:
description: '[Perf] Ignore specific baseline tests (pipe-separated substrings)'
required: false
default: '~failing'
type: string
perf-iterations:
description: '[Perf] Number of iterations per test'
required: false
default: 10
type: number
perf-force-matrix-rebuild:
description: '[Perf] Force test matrix regeneration'
required: false
default: false
type: boolean
perf-force-image-rebuild:
description: '[Perf] Force Docker image rebuilds'
required: false
default: false
type: boolean
perf-debug:
description: '[Perf] Enable debug mode'
required: false
default: false
type: boolean
jobs:
resolve-transport-parameters:
runs-on: [self-hosted, linux, x64, ephemeral]
outputs:
test-select: ${{ steps.resolve.outputs.test-select }}
test-ignore: ${{ steps.resolve.outputs.test-ignore }}
force-matrix-rebuild: ${{ steps.resolve.outputs.force-matrix-rebuild }}
force-image-rebuild: ${{ steps.resolve.outputs.force-image-rebuild }}
debug: ${{ steps.resolve.outputs.debug }}
should-run-tests: ${{ steps.resolve.outputs.should-run-tests }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Resolve transport parameters
id: resolve
shell: bash
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
# Manual trigger - use workflow inputs
echo "test-select=${{ github.event.inputs.transport-test-select }}" >> $GITHUB_OUTPUT
echo "test-ignore=${{ github.event.inputs.transport-test-ignore }}" >> $GITHUB_OUTPUT
echo "force-matrix-rebuild=${{ github.event.inputs.transport-force-matrix-rebuild }}" >> $GITHUB_OUTPUT
echo "force-image-rebuild=${{ github.event.inputs.transport-force-image-rebuild }}" >> $GITHUB_OUTPUT
echo "debug=${{ github.event.inputs.transport-debug }}" >> $GITHUB_OUTPUT
echo "should-run-tests=true" >> $GITHUB_OUTPUT
echo "→ Manual trigger: using workflow inputs for transport"
else
# Automatic trigger - check for changes in last 24 hours
CHANGES=$(git log --oneline --since="24 hours ago" -- "transport/**" | wc -l)
IMPLS_CHANGES=$(git log --oneline --since="24 hours ago" -- "transport/images.yaml" | wc -l)
echo "test-select=" >> $GITHUB_OUTPUT
echo "test-ignore=~failing" >> $GITHUB_OUTPUT
echo "force-image-rebuild=false" >> $GITHUB_OUTPUT
echo "debug=false" >> $GITHUB_OUTPUT
if [ "$CHANGES" -gt 0 ]; then
echo "should-run-tests=true" >> $GITHUB_OUTPUT
if [ "$IMPLS_CHANGES" -gt 0 ]; then
echo "force-matrix-rebuild=true" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: transport changes detected, images.yaml changed"
else
echo "force-matrix-rebuild=false" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: transport changes detected"
fi
else
echo "should-run-tests=false" >> $GITHUB_OUTPUT
echo "force-matrix-rebuild=false" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: no transport changes in last 24h"
fi
fi
- name: Debug outputs
shell: bash
run: |
echo "DEBUG: should-run-tests output = '${{ steps.resolve.outputs.should-run-tests }}'"
echo "DEBUG: github.event_name = '${{ github.event_name }}'"
resolve-hole-punch-parameters:
runs-on: [self-hosted, linux, x64, ephemeral]
outputs:
test-select: ${{ steps.resolve.outputs.test-select }}
test-ignore: ${{ steps.resolve.outputs.test-ignore }}
relay-select: ${{ steps.resolve.outputs.relay-select }}
relay-ignore: ${{ steps.resolve.outputs.relay-ignore }}
router-select: ${{ steps.resolve.outputs.router-select }}
router-ignore: ${{ steps.resolve.outputs.router-ignore }}
force-matrix-rebuild: ${{ steps.resolve.outputs.force-matrix-rebuild }}
force-image-rebuild: ${{ steps.resolve.outputs.force-image-rebuild }}
debug: ${{ steps.resolve.outputs.debug }}
should-run-tests: ${{ steps.resolve.outputs.should-run-tests }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Resolve hole-punch parameters
id: resolve
shell: bash
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
# Manual trigger - use workflow inputs
echo "test-select=${{ github.event.inputs.hole-punch-test-select }}" >> $GITHUB_OUTPUT
echo "test-ignore=${{ github.event.inputs.hole-punch-test-ignore }}" >> $GITHUB_OUTPUT
echo "relay-select=${{ github.event.inputs.hole-punch-relay-select }}" >> $GITHUB_OUTPUT
echo "relay-ignore=${{ github.event.inputs.hole-punch-relay-ignore }}" >> $GITHUB_OUTPUT
echo "router-select=${{ github.event.inputs.hole-punch-router-select }}" >> $GITHUB_OUTPUT
echo "router-ignore=${{ github.event.inputs.hole-punch-router-ignore }}" >> $GITHUB_OUTPUT
echo "force-matrix-rebuild=${{ github.event.inputs.hole-punch-force-matrix-rebuild }}" >> $GITHUB_OUTPUT
echo "force-image-rebuild=${{ github.event.inputs.hole-punch-force-image-rebuild }}" >> $GITHUB_OUTPUT
echo "debug=${{ github.event.inputs.hole-punch-debug }}" >> $GITHUB_OUTPUT
echo "should-run-tests=true" >> $GITHUB_OUTPUT
echo "→ Manual trigger: using workflow inputs for hole-punch"
else
# Automatic trigger - check for changes in last 24 hours
CHANGES=$(git log --oneline --since="24 hours ago" -- "hole-punch/**" | wc -l)
IMPLS_CHANGES=$(git log --oneline --since="24 hours ago" -- "hole-punch/images.yaml" | wc -l)
echo "test-select=" >> $GITHUB_OUTPUT
echo "test-ignore=~failing" >> $GITHUB_OUTPUT
echo "relay-select=" >> $GITHUB_OUTPUT
echo "relay-ignore=" >> $GITHUB_OUTPUT
echo "router-select=" >> $GITHUB_OUTPUT
echo "router-ignore=" >> $GITHUB_OUTPUT
echo "force-image-rebuild=false" >> $GITHUB_OUTPUT
echo "debug=false" >> $GITHUB_OUTPUT
if [ "$CHANGES" -gt 0 ]; then
echo "should-run-tests=true" >> $GITHUB_OUTPUT
if [ "$IMPLS_CHANGES" -gt 0 ]; then
echo "force-matrix-rebuild=true" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: hole-punch changes detected, images.yaml changed"
else
echo "force-matrix-rebuild=false" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: hole-punch changes detected"
fi
else
echo "should-run-tests=false" >> $GITHUB_OUTPUT
echo "force-matrix-rebuild=false" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: no hole-punch changes in last 24h"
fi
fi
resolve-perf-parameters:
runs-on: [self-hosted, linux, x64, ephemeral]
outputs:
test-select: ${{ steps.resolve.outputs.test-select }}
test-ignore: ${{ steps.resolve.outputs.test-ignore }}
baseline-select: ${{ steps.resolve.outputs.baseline-select }}
baseline-ignore: ${{ steps.resolve.outputs.baseline-ignore }}
iterations: ${{ steps.resolve.outputs.iterations }}
force-matrix-rebuild: ${{ steps.resolve.outputs.force-matrix-rebuild }}
force-image-rebuild: ${{ steps.resolve.outputs.force-image-rebuild }}
debug: ${{ steps.resolve.outputs.debug }}
should-run-tests: ${{ steps.resolve.outputs.should-run-tests }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Resolve perf parameters
id: resolve
shell: bash
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
# Manual trigger - use workflow inputs
echo "test-select=${{ github.event.inputs.perf-test-select }}" >> $GITHUB_OUTPUT
echo "test-ignore=${{ github.event.inputs.perf-test-ignore }}" >> $GITHUB_OUTPUT
echo "baseline-select=${{ github.event.inputs.perf-baseline-select }}" >> $GITHUB_OUTPUT
echo "baseline-ignore=${{ github.event.inputs.perf-baseline-ignore }}" >> $GITHUB_OUTPUT
echo "iterations=${{ github.event.inputs.perf-iterations }}" >> $GITHUB_OUTPUT
echo "force-matrix-rebuild=${{ github.event.inputs.perf-force-matrix-rebuild }}" >> $GITHUB_OUTPUT
echo "force-image-rebuild=${{ github.event.inputs.perf-force-image-rebuild }}" >> $GITHUB_OUTPUT
echo "debug=${{ github.event.inputs.perf-debug }}" >> $GITHUB_OUTPUT
echo "should-run-tests=true" >> $GITHUB_OUTPUT
echo "→ Manual trigger: using workflow inputs for perf"
else
# Automatic trigger - check for changes in last 24 hours
CHANGES=$(git log --oneline --since="24 hours ago" -- "perf/**" | wc -l)
IMPLS_CHANGES=$(git log --oneline --since="24 hours ago" -- "perf/images.yaml" | wc -l)
echo "test-select=" >> $GITHUB_OUTPUT
echo "test-ignore=~failing" >> $GITHUB_OUTPUT
echo "baseline-select=" >> $GITHUB_OUTPUT
echo "baseline-ignore=~failing" >> $GITHUB_OUTPUT
echo "iterations=10" >> $GITHUB_OUTPUT
echo "force-image-rebuild=false" >> $GITHUB_OUTPUT
echo "debug=false" >> $GITHUB_OUTPUT
if [ "$CHANGES" -gt 0 ]; then
echo "should-run-tests=true" >> $GITHUB_OUTPUT
if [ "$IMPLS_CHANGES" -gt 0 ]; then
echo "force-matrix-rebuild=true" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: perf changes detected, images.yaml changed"
else
echo "force-matrix-rebuild=false" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: perf changes detected"
fi
else
echo "should-run-tests=false" >> $GITHUB_OUTPUT
echo "force-matrix-rebuild=false" >> $GITHUB_OUTPUT
echo "→ Automatic trigger: no perf changes in last 24h"
fi
fi
run-transport-full:
needs: resolve-transport-parameters
if: ${{ !cancelled() && (github.event_name == 'workflow_dispatch' || needs.resolve-transport-parameters.outputs.should-run-tests) }}
runs-on: [self-hosted, linux, x64, ephemeral]
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Run full transport interop tests
uses: ./.github/actions/run-bash-transport-test
with:
test-select: '${{ needs.resolve-transport-parameters.outputs.test-select }}'
test-ignore: '${{ needs.resolve-transport-parameters.outputs.test-ignore }}'
cache-dir: /srv/cache
snapshot: true
force-matrix-rebuild: ${{ needs.resolve-transport-parameters.outputs.force-matrix-rebuild }}
force-image-rebuild: ${{ needs.resolve-transport-parameters.outputs.force-image-rebuild }}
debug: ${{ needs.resolve-transport-parameters.outputs.debug }}
run-hole-punch-full:
needs: resolve-hole-punch-parameters
if: ${{ !cancelled() && (github.event_name == 'workflow_dispatch' || needs.resolve-hole-punch-parameters.outputs.should-run-tests) }}
runs-on: [self-hosted, linux, x64, ephemeral]
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Run full hole punch interop tests
uses: ./.github/actions/run-bash-hole-punch-test
with:
test-select: '${{ needs.resolve-hole-punch-parameters.outputs.test-select }}'
test-ignore: '${{ needs.resolve-hole-punch-parameters.outputs.test-ignore }}'
relay-select: '${{ needs.resolve-hole-punch-parameters.outputs.relay-select }}'
relay-ignore: '${{ needs.resolve-hole-punch-parameters.outputs.relay-ignore }}'
router-select: '${{ needs.resolve-hole-punch-parameters.outputs.router-select }}'
router-ignore: '${{ needs.resolve-hole-punch-parameters.outputs.router-ignore }}'
cache-dir: /srv/cache
snapshot: true
force-matrix-rebuild: ${{ needs.resolve-hole-punch-parameters.outputs.force-matrix-rebuild }}
force-image-rebuild: ${{ needs.resolve-hole-punch-parameters.outputs.force-image-rebuild }}
debug: ${{ needs.resolve-hole-punch-parameters.outputs.debug }}
run-perf-full:
needs: resolve-perf-parameters
if: ${{ !cancelled() && (github.event_name == 'workflow_dispatch' || needs.resolve-perf-parameters.outputs.should-run-tests) }}
runs-on: [self-hosted, linux, x64, ephemeral]
continue-on-error: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Run full perf benchmarks
uses: ./.github/actions/run-bash-perf-test
with:
test-select: '${{ needs.resolve-perf-parameters.outputs.test-select }}'
test-ignore: '${{ needs.resolve-perf-parameters.outputs.test-ignore }}'
baseline-select: '${{ needs.resolve-perf-parameters.outputs.baseline-select }}'
baseline-ignore: '${{ needs.resolve-perf-parameters.outputs.baseline-ignore }}'
iterations: '${{ needs.resolve-perf-parameters.outputs.iterations }}'
cache-dir: /srv/cache
snapshot: true
force-matrix-rebuild: ${{ needs.resolve-perf-parameters.outputs.force-matrix-rebuild }}
force-image-rebuild: ${{ needs.resolve-perf-parameters.outputs.force-image-rebuild }}
debug: ${{ needs.resolve-perf-parameters.outputs.debug }}
merge-and-commit-results:
needs: [run-transport-full, run-hole-punch-full, run-perf-full]
if: |
always() &&
(needs.run-transport-full.result != 'skipped' ||
needs.run-hole-punch-full.result != 'skipped' ||
needs.run-perf-full.result != 'skipped')
runs-on: [self-hosted, linux, x64, ephemeral]
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0
- name: Download transport test results
if: needs.run-transport-full.result != 'skipped'
uses: actions/download-artifact@v4
with:
pattern: transport-test-results-*
path: /tmp/transport-results
merge-multiple: true
- name: Download hole-punch test results
if: needs.run-hole-punch-full.result != 'skipped'
uses: actions/download-artifact@v4
with:
pattern: hole-punch-test-results-*
path: /tmp/hole-punch-results
merge-multiple: true
- name: Download perf test results
if: needs.run-perf-full.result != 'skipped'
uses: actions/download-artifact@v4
with:
pattern: perf-test-results-*
path: /tmp/perf-results
merge-multiple: true
- name: Download perf box plots
if: needs.run-perf-full.result != 'skipped'
uses: actions/download-artifact@v4
with:
pattern: perf-test-box-plot-*
path: /tmp/perf-boxplots
merge-multiple: true
- name: Debug artifact structures
shell: bash
run: |
echo "=== Transport artifact ==="
find /tmp/transport-results -type f 2>/dev/null | head -20 || echo "No transport results found"
TRANSPORT_YAML=$(find /tmp/transport-results -name "results.yaml" 2>/dev/null | head -1)
if [ -n "$TRANSPORT_YAML" ] && [ -f "$TRANSPORT_YAML" ]; then
echo "--- First test entry ---"
yq eval '.tests[0]' "$TRANSPORT_YAML" 2>/dev/null || echo "No tests found"
fi
echo ""
echo "=== Hole-punch artifact ==="
find /tmp/hole-punch-results -type f 2>/dev/null | head -20 || echo "No hole-punch results found"
HP_YAML=$(find /tmp/hole-punch-results -name "results.yaml" 2>/dev/null | head -1)
if [ -n "$HP_YAML" ] && [ -f "$HP_YAML" ]; then
echo "--- First test entry ---"
yq eval '.tests[0]' "$HP_YAML" 2>/dev/null || echo "No tests found"
fi
echo ""
echo "=== Perf artifact ==="
find /tmp/perf-results -type f 2>/dev/null | head -20 || echo "No perf results found"
PERF_YAML=$(find /tmp/perf-results -name "results.yaml" 2>/dev/null | head -1)
if [ -n "$PERF_YAML" ] && [ -f "$PERF_YAML" ]; then
echo "--- First baseline entry ---"
yq eval '.baselineResults[0]' "$PERF_YAML" 2>/dev/null || echo "No baseline results found"
echo "--- First test entry ---"
yq eval '.testResults[0]' "$PERF_YAML" 2>/dev/null || echo "No test results found"
fi
echo ""
echo "=== Boxplot artifact ==="
find /tmp/perf-boxplots -name "*.png" -type f 2>/dev/null | head -10 || echo "No boxplots found"
- name: Find result directories and prepare merge
id: find-results
shell: bash
run: |
# Find transport results directory
TRANSPORT_DIR=""
if [ -d /tmp/transport-results ]; then
if [ -f /tmp/transport-results/results.yaml ]; then
TRANSPORT_DIR="/tmp/transport-results"
else
# Results might be in a subdirectory
TRANSPORT_DIR=$(find /tmp/transport-results -name "results.yaml" -exec dirname {} \; | head -1)
fi
fi
echo "transport-dir=$TRANSPORT_DIR" >> $GITHUB_OUTPUT
echo "→ Transport results: ${TRANSPORT_DIR:-not found}"
# Find hole-punch results directory
HOLE_PUNCH_DIR=""
if [ -d /tmp/hole-punch-results ]; then
if [ -f /tmp/hole-punch-results/results.yaml ]; then
HOLE_PUNCH_DIR="/tmp/hole-punch-results"
else
HOLE_PUNCH_DIR=$(find /tmp/hole-punch-results -name "results.yaml" -exec dirname {} \; | head -1)
fi
fi
echo "hole-punch-dir=$HOLE_PUNCH_DIR" >> $GITHUB_OUTPUT
echo "→ Hole-punch results: ${HOLE_PUNCH_DIR:-not found}"
# Find perf results directory (merge boxplots into it)
PERF_DIR=""
if [ -d /tmp/perf-results ]; then
if [ -f /tmp/perf-results/results.yaml ]; then
PERF_DIR="/tmp/perf-results"
else
PERF_DIR=$(find /tmp/perf-results -name "results.yaml" -exec dirname {} \; | head -1)
fi
# Copy boxplots to perf results directory (use find to handle nested directories)
if [ -n "$PERF_DIR" ] && [ -d /tmp/perf-boxplots ]; then
echo "Copying boxplots to $PERF_DIR"
find /tmp/perf-boxplots -name "*.png" -type f -exec cp -v {} "$PERF_DIR/" \; 2>/dev/null || echo "No boxplots found"
fi
fi
echo "perf-dir=$PERF_DIR" >> $GITHUB_OUTPUT
echo "→ Perf results: ${PERF_DIR:-not found}"
- name: Verify merge script version
shell: bash
run: |
echo "=== Checking merge script uses correct field names ==="
if grep -q '\.dialer\.id' lib/merge-interop-results.sh; then
echo "ERROR: Script still uses old .dialer.id syntax"
exit 1
fi
if grep -q '\.listener\.id' lib/merge-interop-results.sh; then
echo "ERROR: Script still uses old .listener.id syntax"
exit 1
fi
if grep -q '"name": \.id' lib/merge-interop-results.sh; then
echo "ERROR: Script still uses old .id syntax for name"
exit 1
fi
if grep -q '"status": \.outcome' lib/merge-interop-results.sh; then
echo "ERROR: Script still uses old .outcome syntax for status"
exit 1
fi
echo "Script version verified - using correct field names"
- name: Run merge-interop-results script
shell: bash
run: |
WORKFLOW_RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
bash lib/merge-interop-results.sh \
"${{ steps.find-results.outputs.transport-dir }}" \
"${{ steps.find-results.outputs.hole-punch-dir }}" \
"${{ steps.find-results.outputs.perf-dir }}" \
"results/daily-full-interop.yml" \
"$WORKFLOW_RUN_URL"
- name: Verify files to commit
shell: bash
run: |
echo "=== Files in results/ directory ==="
ls -la results/
echo ""
echo "=== Box plot files ==="
ls -la results/*.png 2>/dev/null || echo "No PNG files found"
echo ""
echo "=== Generated YAML content preview ==="
head -100 results/daily-full-interop.yml
- name: Commit and push combined results
shell: bash
run: |
set -euo pipefail
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Check if there are changes to commit
git add results/
if git diff --cached --quiet 2>/dev/null; then
echo "→ No changes to commit"
exit 0
fi
git commit -m "chore: update combined interop results [skip ci]" \
-m "Combined results from daily full interoperability test run." \
-m "Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
-m "Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>"
echo "→ Commit created successfully"
# Push with retry logic (handles race conditions)
MAX_ATTEMPTS=5
ATTEMPT=1
while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
echo "→ Push attempt $ATTEMPT/$MAX_ATTEMPTS"
if git push origin master; then
echo "✓ Combined results committed and pushed successfully"
exit 0
else
echo "✗ Push failed (attempt $ATTEMPT/$MAX_ATTEMPTS)"
if [ $ATTEMPT -eq $MAX_ATTEMPTS ]; then
echo "✗ Failed to push after $MAX_ATTEMPTS attempts"
exit 1
fi
echo "→ Pulling latest changes and rebasing..."
git fetch origin master
if git rebase origin/master; then
echo "✓ Rebase successful"
else
echo "✗ Rebase conflict detected, resolving..."
git checkout --ours results/ 2>/dev/null || true
git add results/
git rebase --continue || {
echo "✗ Failed to resolve rebase conflict"
git rebase --abort
exit 1
}
echo "✓ Rebase conflict resolved"
fi
SLEEP_TIME=$((5 * (2 ** ($ATTEMPT - 1))))
echo "→ Waiting ${SLEEP_TIME}s before retry..."
sleep $SLEEP_TIME
ATTEMPT=$((ATTEMPT + 1))
fi
done