diff --git a/.github/actions/get-image-tag/action.yml b/.github/actions/get-image-tag/action.yml new file mode 100644 index 0000000000..b9fc3e565b --- /dev/null +++ b/.github/actions/get-image-tag/action.yml @@ -0,0 +1,84 @@ +name: "#๏ธโƒฃ Get image tag (action)" + +description: This action gets the image tag from the commit ref or input (if provided) + +outputs: + tag: + description: The image tag + value: ${{ steps.get_tag.outputs.tag }} + is_semver: + description: Whether the tag is a semantic version + value: ${{ steps.check_semver.outputs.is_semver }} + +inputs: + tag: + description: The image tag. If this is set it will return the tag as is. + required: false + default: "" + +runs: + using: "composite" + steps: + - name: "#๏ธโƒฃ Get image tag (step)" + id: get_tag + shell: bash + run: | + if [[ -n "${{ inputs.tag }}" ]]; then + tag="${{ inputs.tag }}" + elif [[ "${{ github.ref_type }}" == "tag" ]]; then + if [[ "${{ github.ref_name }}" == infra-*-* ]]; then + env=$(echo ${{ github.ref_name }} | cut -d- -f2) + sha=$(echo ${{ github.sha }} | head -c7) + ts=$(date +%s) + tag=${env}-${sha}-${ts} + elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then + version="${GITHUB_REF_NAME#v.docker.}" + tag="v${version}" + elif [[ "${{ github.ref_name }}" == build-* ]]; then + tag="${GITHUB_REF_NAME#build-}" + else + echo "Invalid git tag: ${{ github.ref_name }}" + exit 1 + fi + elif [[ "${{ github.ref_name }}" == "main" ]]; then + tag="main" + else + echo "Invalid git ref: ${{ github.ref }}" + exit 1 + fi + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + + - name: ๐Ÿ” Check for validity + id: check_validity + shell: bash + env: + tag: ${{ steps.get_tag.outputs.tag }} + run: | + if [[ "${tag}" =~ ^[a-z0-9]+([._-][a-z0-9]+)*$ ]]; then + echo "Tag is valid: ${tag}" + else + echo "Tag is not valid: ${tag}" + exit 1 + fi + + - name: ๐Ÿ†š Check for semver + id: check_semver + shell: bash + env: + tag: ${{ steps.get_tag.outputs.tag }} + # Will match most semver formats except build metadata, i.e. v1.2.3+build.1 + # Valid matches: + # v1.2.3 + # v1.2.3-alpha + # v1.2.3-alpha.1 + # v1.2.3-rc.1 + # v1.2.3-beta-1 + run: | + if [[ "${tag}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$ ]]; then + echo "Tag is a semantic version: ${tag}" + is_semver=true + else + echo "Tag is not a semantic version: ${tag}" + is_semver=false + fi + echo "is_semver=${is_semver}" >> "$GITHUB_OUTPUT" diff --git a/.github/test/README.md b/.github/test/README.md new file mode 100644 index 0000000000..1e8383fda5 --- /dev/null +++ b/.github/test/README.md @@ -0,0 +1,70 @@ +# GitHub Action Tests + +This directory contains necessary files to allow local testing of GitHub Actions workflows, composite actions, etc. You will need to install [act](https://github.com/nektos/act) to perform tests. + +## Workflow tests + +Trigger specific workflow files by specifying their full path: + +``` +act -W .github/workflow/release.yml +``` + +You will likely need to override any custom runners we use, e.g. buildjet. For example: + +``` +override=catthehacker/ubuntu:act-latest + +act -W .github/workflow/release.yml \ + -P buildjet-8vcpu-ubuntu-2204=$override + +# override multiple images at the same time +act -W .github/workflow/release.yml \ + -P buildjet-8vcpu-ubuntu-2204=$override \ + -P buildjet-16vcpu-ubuntu-2204=$override +``` + +Trigger with specific event payloads to test pushing to branches or tags: + +``` +override=catthehacker/ubuntu:act-latest + +# simulate push to main +act -W .github/workflow/publish.yml \ + -P buildjet-8vcpu-ubuntu-2204=$override \ + -P buildjet-16vcpu-ubuntu-2204=$override \ + -e .github/events/push-tag-main.json + +# simulate a `build-` prefixed tag +act -W .github/workflow/publish.yml \ + -P buildjet-8vcpu-ubuntu-2204=$override \ + -P buildjet-16vcpu-ubuntu-2204=$override \ + -e .github/events/push-tag-buld.json +``` + +By default, `act` will send a push event. To trigger a different event: + +``` +# basic syntax +act ... + +# simulate a pull request +act pull_request + +# only trigger a specific workflow +act pull_request -W .github/workflow/pr_checks.yml +``` + +## Composite action tests + +The composite (custom) action tests can be run by triggering the `test-actions` workflow: + +``` +act -W .github/test/test-actions.yml +``` + +## Helpful flags + +- `--pull=false` - perform fully offline tests if all images are already present +- `-j ` - run the specified job only +- `-l push` - list all workflows with push triggers diff --git a/.github/test/events/push-main.json b/.github/test/events/push-main.json new file mode 100644 index 0000000000..ccb4cb1c17 --- /dev/null +++ b/.github/test/events/push-main.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/heads/main" +} diff --git a/.github/test/events/push-tag-build.json b/.github/test/events/push-tag-build.json new file mode 100644 index 0000000000..9490c181ab --- /dev/null +++ b/.github/test/events/push-tag-build.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/build-buildtag" +} diff --git a/.github/test/events/push-tag-docker-nonsemver.json b/.github/test/events/push-tag-docker-nonsemver.json new file mode 100644 index 0000000000..5ce2d8dcf3 --- /dev/null +++ b/.github/test/events/push-tag-docker-nonsemver.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/v.docker.nonsemver" +} diff --git a/.github/test/events/push-tag-docker.json b/.github/test/events/push-tag-docker.json new file mode 100644 index 0000000000..7b55610ca2 --- /dev/null +++ b/.github/test/events/push-tag-docker.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/v.docker.1.2.3" +} diff --git a/.github/test/events/push-tag-infra-prod.json b/.github/test/events/push-tag-infra-prod.json new file mode 100644 index 0000000000..7d4bb3a0bb --- /dev/null +++ b/.github/test/events/push-tag-infra-prod.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/infra-prod-anything" +} diff --git a/.github/test/events/push-tag-infra-test.json b/.github/test/events/push-tag-infra-test.json new file mode 100644 index 0000000000..78eeefbe41 --- /dev/null +++ b/.github/test/events/push-tag-infra-test.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/infra-test-anything" +} diff --git a/.github/test/events/push-tag-semver.json b/.github/test/events/push-tag-semver.json new file mode 100644 index 0000000000..3fb65c9207 --- /dev/null +++ b/.github/test/events/push-tag-semver.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/1.2.3" +} diff --git a/.github/test/events/push-tag.json b/.github/test/events/push-tag.json new file mode 100644 index 0000000000..26496f8087 --- /dev/null +++ b/.github/test/events/push-tag.json @@ -0,0 +1,3 @@ +{ + "ref": "refs/tags/standard-tag" +} diff --git a/.github/test/test-actions.yml b/.github/test/test-actions.yml new file mode 100644 index 0000000000..0d913ebc0e --- /dev/null +++ b/.github/test/test-actions.yml @@ -0,0 +1,152 @@ +name: Test Actions + +on: + workflow_dispatch: + +jobs: + get-image-tag-none: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log current ref + run: | + echo "ref: ${{ github.ref }}" + echo "ref_type: ${{ github.ref_type }}" + echo "ref_name: ${{ github.ref_name }}" + + - name: Run without input tag + id: get_tag + # this step may fail depending on the current ref + continue-on-error: true + uses: ./.github/actions/get-image-tag + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" + + get-image-tag-null: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log current ref + run: | + echo "ref: ${{ github.ref }}" + echo "ref_type: ${{ github.ref_type }}" + echo "ref_name: ${{ github.ref_name }}" + + - name: Run without input tag + id: get_tag + uses: ./.github/actions/get-image-tag + # this step may fail depending on the current ref + continue-on-error: true + with: + # this should behave exactly as when no tag is provided + tag: null + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" + + get-image-tag-override: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run with tag override + id: get_tag + uses: ./.github/actions/get-image-tag + with: + tag: "abc-123" + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" + + get-image-tag-invalid-string: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run with invalid string + id: get_tag + uses: ./.github/actions/get-image-tag + # this step is expected to fail + continue-on-error: true + with: + # does not end with alphanumeric character + tag: "abc-123-" + + - name: Fail job if previous step did not fail + if: steps.get_tag.outcome != 'failure' + run: exit 1 + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" + + get-image-tag-prerelease: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run with prerelease semver + id: get_tag + uses: ./.github/actions/get-image-tag + with: + tag: "v1.2.3-beta.4" + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" + + get-image-tag-semver: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run with basic semver + id: get_tag + uses: ./.github/actions/get-image-tag + with: + tag: "v1.2.3" + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" + + get-image-tag-invalid-semver: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run with invalid semver + id: get_tag + uses: ./.github/actions/get-image-tag + # this step is expected to fail + continue-on-error: true + with: + tag: "v1.2.3-" + + - name: Fail job if previous step did not fail + if: steps.get_tag.outcome != 'failure' + run: exit 1 + + - name: Verify output + run: | + echo "${{ toJson(steps.get_tag) }}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..3abe5cb4fe --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,42 @@ +name: ๐Ÿ“š Docs Checks + +on: + push: + branches: + - main + paths: + - "docs/**" + pull_request: + types: [opened, synchronize, reopened] + paths: + - "docs/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + check-broken-links: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./docs + steps: + - name: ๐Ÿ“ฅ Checkout repository + uses: actions/checkout@v4 + + - name: ๐Ÿ“ฆ Cache npm + uses: actions/cache@v4 + with: + path: | + ~/.npm + key: | + ${{ runner.os }}-mintlify + restore-keys: | + ${{ runner.os }}-mintlify + + - name: ๐Ÿ”— Check for broken links + run: npx mintlify@4.0.222 broken-links diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 535f94f9b5..968f509aa5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,4 +1,5 @@ name: "E2E" + on: workflow_call: inputs: @@ -7,6 +8,7 @@ on: default: webapp required: false type: string + jobs: cli-v3: name: "๐Ÿงช CLI v3 tests (${{ matrix.os }} - ${{ matrix.package-manager }})" @@ -19,7 +21,7 @@ jobs: package-manager: ["npm", "pnpm"] steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -49,5 +51,3 @@ jobs: shell: bash run: | LOG=debug PM=${{ matrix.package-manager }} pnpm --filter trigger.dev run test:e2e - - diff --git a/.github/workflows/pr_checks.yml b/.github/workflows/pr_checks.yml index e579843270..298c0766ff 100644 --- a/.github/workflows/pr_checks.yml +++ b/.github/workflows/pr_checks.yml @@ -30,12 +30,11 @@ jobs: preview-release: name: Preview Release needs: [typecheck, units, e2e] - if: | - github.repository == 'triggerdotdev/trigger.dev' + if: github.repository == 'triggerdotdev/trigger.dev' runs-on: buildjet-8vcpu-ubuntu-2204 steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -56,5 +55,5 @@ jobs: - name: ๐Ÿ—๏ธ Build run: pnpm run build --filter "@trigger.dev/*" --filter "trigger.dev" - - name: โšก publish preview release + - name: โšก Publish preview release run: npx pkg-pr-new publish --no-template $(ls -d ./packages/*) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml deleted file mode 100644 index 688b1d25db..0000000000 --- a/.github/workflows/publish-docker.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: "๐Ÿณ Publish Docker" -on: - workflow_call: -jobs: - publish: - runs-on: ubuntu-latest - env: - PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1 - outputs: - version: ${{ steps.get_version.outputs.version }} - short_sha: ${{ steps.get_commit.outputs.sha_short }} - steps: - - name: Setup Depot CLI - uses: depot/setup-action@v1 - - - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: ๐Ÿ†š Get the version - id: get_version - run: | - IMAGE_TAG="${GITHUB_REF#refs/tags/}" - if [[ $GITHUB_REF == refs/tags/* ]]; then - if [[ $IMAGE_TAG == v.docker.* ]]; then - ORIGINAL_VERSION="${IMAGE_TAG#v.docker.}" - IMAGE_TAG="v${ORIGINAL_VERSION}" - elif [[ $IMAGE_TAG == build-* ]]; then - IMAGE_TAG="${IMAGE_TAG#build-}" - fi - echo "IMAGE_TAG=${IMAGE_TAG}" - elif [[ $GITHUB_REF == refs/heads/main ]]; then - # Handle main branch specifically - IMAGE_TAG="main" - echo "IMAGE_TAG=${IMAGE_TAG}" - else - echo "Invalid reference: ${GITHUB_REF}" - exit 1 - fi - echo "::set-output name=version::${IMAGE_TAG}" - - - name: ๐Ÿ”ข Get the commit hash - id: get_commit - run: | - echo ::set-output name=sha_short::$(echo ${{ github.sha }} | cut -c1-7) - - - name: ๐Ÿ“› Set the tags - id: set_tags - run: | - ref_without_tag=ghcr.io/triggerdotdev/trigger.dev - image_tags=$ref_without_tag:${{ steps.get_version.outputs.version }} - - # if it's a versioned tag, also tag it as v3 - if [[ "${{ github.ref_name }}" == v.docker.* ]]; then - image_tags=$image_tags,$ref_without_tag:v3 - fi - - echo "IMAGE_TAGS=${image_tags}" >> "$GITHUB_OUTPUT" - - - name: ๐Ÿ™ Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: ๐Ÿณ Build image and push to GitHub Container Registry - uses: depot/build-push-action@v1 - with: - file: ./docker/Dockerfile - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.set_tags.outputs.IMAGE_TAGS }} - push: true diff --git a/.github/workflows/publish-infra.yml b/.github/workflows/publish-infra.yml deleted file mode 100644 index 1a3486b018..0000000000 --- a/.github/workflows/publish-infra.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: "๐Ÿšข Publish Infra Images" - -on: - workflow_call: - push: - tags: - - "infra-dev-*" - - "infra-test-*" - - "infra-prod-*" - paths: - - ".github/workflows/publish.yml" - - "packages/**" - - "!packages/**/*.md" - - "!packages/**/*.eslintrc" - - "apps/**" - - "!apps/**/*.md" - - "!apps/**/*.eslintrc" - - "integrations/**" - - "!integrations/**/*.md" - - "!integrations/**/*.eslintrc" - - "pnpm-lock.yaml" - - "pnpm-workspace.yaml" - - "turbo.json" - - "docker/Dockerfile" - - "docker/scripts/**" - - "tests/**" - -permissions: - id-token: write - packages: write - contents: read - -env: - AWS_REGION: us-east-1 - -jobs: - build: - strategy: - matrix: - package: [coordinator, docker-provider, kubernetes-provider] - runs-on: buildjet-16vcpu-ubuntu-2204 - env: - DOCKER_BUILDKIT: "1" - steps: - - uses: actions/checkout@v4 - - - name: Generate image reference - id: prep - run: | - # set image repo - if [[ "${{ matrix.package }}" == *-provider ]]; then - provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1) - repository=provider/${provider_type} - else - repository="${{ matrix.package }}" - fi - echo "REPOSITORY=${repository}" >> "$GITHUB_OUTPUT" - - # set image tag - if [[ "${{ github.ref_type }}" == "tag" ]]; then - if [[ "${{ github.ref_name }}" == infra-*-* ]]; then - env=$(echo ${{ github.ref_name }} | cut -d- -f2) - sha=$(echo ${{ github.sha }} | head -c7) - ts=$(date +%s) - image_tag=${env}-${sha}-${ts} - elif [[ "${{ github.ref_name }}" == v.docker.* ]]; then - version="${GITHUB_REF_NAME#v.docker.}" - image_tag="v${version}" - elif [[ "${{ github.ref_name }}" == build-* ]]; then - image_tag="${GITHUB_REF_NAME#build-}" - else - echo "Invalid tag: ${{ github.ref_name }}" - exit 1 - fi - elif [[ "${{ github.ref_name }}" == "main" ]]; then - image_tag="main" - else - echo "Invalid reference: ${{ github.ref }}" - exit 1 - fi - echo "IMAGE_TAG=${image_tag}" >> "$GITHUB_OUTPUT" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - # ..to avoid rate limits when pulling images - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: ๐Ÿšข Build Container Image - run: | - docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile . - - # ..to push image - - name: ๐Ÿ™ Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: ๐Ÿ™ Push to GitHub Container Registry - run: | - docker tag infra_image $REGISTRY/$REPOSITORY:$IMAGE_TAG - docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG - env: - REGISTRY: ghcr.io/triggerdotdev - REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }} - IMAGE_TAG: ${{ steps.prep.outputs.IMAGE_TAG }} - - - name: ๐Ÿ™ Push 'v3' tag to GitHub Container Registry - if: startsWith(github.ref_name, 'v.docker.') - run: | - docker tag infra_image $REGISTRY/$REPOSITORY:v3 - docker push $REGISTRY/$REPOSITORY:v3 - env: - REGISTRY: ghcr.io/triggerdotdev - REPOSITORY: ${{ steps.prep.outputs.REPOSITORY }} diff --git a/.github/workflows/publish-webapp.yml b/.github/workflows/publish-webapp.yml new file mode 100644 index 0000000000..b977ef0a19 --- /dev/null +++ b/.github/workflows/publish-webapp.yml @@ -0,0 +1,66 @@ +name: "๐Ÿณ Publish Webapp" + +on: + workflow_call: + inputs: + image_tag: + description: The image tag to publish + type: string + required: false + default: "" + +jobs: + publish: + runs-on: ubuntu-latest + env: + PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING: 1 + outputs: + version: ${{ steps.get_tag.outputs.tag }} + short_sha: ${{ steps.get_commit.outputs.sha_short }} + steps: + - name: ๐Ÿญ Setup Depot CLI + uses: depot/setup-action@v1 + + - name: โฌ‡๏ธ Checkout repo + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: "#๏ธโƒฃ Get the image tag" + id: get_tag + uses: ./.github/actions/get-image-tag + with: + tag: ${{ inputs.image_tag }} + + - name: ๐Ÿ”ข Get the commit hash + id: get_commit + run: | + echo "sha_short=$(echo ${{ github.sha }} | cut -c1-7)" >> "$GITHUB_OUTPUT" + + - name: ๐Ÿ“› Set the tags + id: set_tags + run: | + ref_without_tag=ghcr.io/triggerdotdev/trigger.dev + image_tags=$ref_without_tag:${{ steps.get_tag.outputs.tag }} + + # if tag is a semver, also tag it as v3 + if [[ "${{ steps.get_tag.outputs.is_semver }}" == true ]]; then + image_tags=$image_tags,$ref_without_tag:v3 + fi + + echo "image_tags=${image_tags}" >> "$GITHUB_OUTPUT" + + - name: ๐Ÿ™ Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: ๐Ÿณ Build image and push to GitHub Container Registry + uses: depot/build-push-action@v1 + with: + file: ./docker/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.set_tags.outputs.image_tags }} + push: true diff --git a/.github/workflows/publish-worker.yml b/.github/workflows/publish-worker.yml new file mode 100644 index 0000000000..4593205d52 --- /dev/null +++ b/.github/workflows/publish-worker.yml @@ -0,0 +1,87 @@ +name: "โš’๏ธ Publish Worker" + +on: + workflow_call: + inputs: + image_tag: + description: The image tag to publish + type: string + required: false + default: "" + push: + tags: + - "infra-dev-*" + - "infra-test-*" + - "infra-prod-*" + +permissions: + packages: write + contents: read + +jobs: + build: + strategy: + matrix: + package: [coordinator, docker-provider, kubernetes-provider] + runs-on: buildjet-16vcpu-ubuntu-2204 + env: + DOCKER_BUILDKIT: "1" + steps: + - name: โฌ‡๏ธ Checkout git repo + uses: actions/checkout@v4 + + - name: ๐Ÿ“ฆ Get image repo + id: get_repository + run: | + if [[ "${{ matrix.package }}" == *-provider ]]; then + provider_type=$(echo "${{ matrix.package }}" | cut -d- -f1) + repo=provider/${provider_type} + else + repo="${{ matrix.package }}" + fi + echo "repo=${repo}" >> "$GITHUB_OUTPUT" + + - id: get_tag + uses: ./.github/actions/get-image-tag + with: + tag: ${{ inputs.image_tag }} + + - name: ๐Ÿ‹ Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # ..to avoid rate limits when pulling images + - name: ๐Ÿณ Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: ๐Ÿšข Build Container Image + run: | + docker build -t infra_image -f ./apps/${{ matrix.package }}/Containerfile . + + # ..to push image + - name: ๐Ÿ™ Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: ๐Ÿ™ Push to GitHub Container Registry + run: | + docker tag infra_image "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + docker push "$REGISTRY/$REPOSITORY:$IMAGE_TAG" + env: + REGISTRY: ghcr.io/triggerdotdev + REPOSITORY: ${{ steps.get_repository.outputs.repo }} + IMAGE_TAG: ${{ steps.get_tag.outputs.tag }} + + - name: ๐Ÿ™ Push 'v3' tag to GitHub Container Registry + if: steps.get_tag.outputs.is_semver == 'true' + run: | + docker tag infra_image "$REGISTRY/$REPOSITORY:v3" + docker push "$REGISTRY/$REPOSITORY:v3" + env: + REGISTRY: ghcr.io/triggerdotdev + REPOSITORY: ${{ steps.get_repository.outputs.repo }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index fa48190727..b11cb18622 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,12 @@ name: ๐Ÿš€ Publish Trigger.dev Docker on: + workflow_call: + inputs: + image_tag: + description: The image tag to publish + required: true + type: string push: branches: - main @@ -8,11 +14,13 @@ on: - "v.docker.*" - "build-*" paths: + - ".github/actions/**/*.yml" - ".github/workflows/publish.yml" - ".github/workflows/typecheck.yml" - ".github/workflows/unit-tests.yml" - ".github/workflows/e2e.yml" - - ".github/workflows/publish-docker.yml" + - ".github/workflows/publish-webapp.yml" + - ".github/workflows/publish-worker.yml" - "packages/**" - "!packages/**/*.md" - "!packages/**/*.eslintrc" @@ -49,12 +57,16 @@ jobs: uses: ./.github/workflows/unit-tests.yml secrets: inherit - publish: + publish-webapp: needs: [typecheck, units] - uses: ./.github/workflows/publish-docker.yml + uses: ./.github/workflows/publish-webapp.yml secrets: inherit + with: + image_tag: ${{ inputs.image_tag }} - publish-infra: + publish-worker: needs: [typecheck, units] - uses: ./.github/workflows/publish-infra.yml + uses: ./.github/workflows/publish-worker.yml secrets: inherit + with: + image_tag: ${{ inputs.image_tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1c74b0bc8..9926a2ede7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,26 +5,27 @@ on: branches: - main paths-ignore: + - "docs/**" - "**.md" - - "**.mdx" - ".github/CODEOWNERS" - ".github/ISSUE_TEMPLATE/**" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: release: name: ๐Ÿฆ‹ Changesets Release runs-on: buildjet-8vcpu-ubuntu-2204 - if: | - github.repository == 'triggerdotdev/trigger.dev' + if: github.repository == 'triggerdotdev/trigger.dev' outputs: - published_packages: ${{ steps.changesets.outputs.publishedPackages }} published: ${{ steps.changesets.outputs.published }} + published_packages: ${{ steps.changesets.outputs.publishedPackages }} + published_package_version: ${{ steps.get_version.outputs.package_version }} steps: - - name: ๐Ÿ›‘ Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.11.0 - - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -62,6 +63,7 @@ jobs: # PR is merged, the workflow will run again and this action will build + # publish to npm. - name: ๐Ÿš€ PR / Publish + if: ${{ !env.ACT }} id: changesets uses: changesets/action@v1 with: @@ -73,3 +75,27 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: ๐Ÿš€ PR / Publish (mock) + if: ${{ env.ACT }} + id: changesets + run: | + echo "published=true" >> "$GITHUB_OUTPUT" + echo "publishedPackages=[{\"name\": \"@xx/xx\", \"version\": \"1.2.0\"}, {\"name\": \"@xx/xy\", \"version\": \"0.8.9\"}]" >> "$GITHUB_OUTPUT" + + - name: ๐Ÿ“ฆ Get package version + if: steps.changesets.outputs.published == 'true' + id: get_version + run: | + package_version=$(echo '${{ steps.changesets.outputs.publishedPackages }}' | jq -r '.[0].version') + echo "package_version=${package_version}" >> "$GITHUB_OUTPUT" + + publish: + needs: release + uses: ./.github/workflows/publish.yml + secrets: inherit + # if: needs.release.outputs.published == 'true' + # disable automatic publishing for now + if: false + with: + image_tag: v${{ needs.release.outputs.published_package_version }} diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index afc5cf03a9..921e9bd342 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -1,13 +1,15 @@ name: "สฆ TypeScript" + on: workflow_call: + jobs: typecheck: runs-on: buildjet-8vcpu-ubuntu-2204 steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -30,6 +32,6 @@ jobs: - name: ๐Ÿ”Ž Type check run: pnpm run typecheck - + - name: ๐Ÿ”Ž Check exports run: pnpm run check-exports diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 111c3b0378..863c5fcde5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,13 +1,15 @@ name: "๐Ÿงช Unit Tests" + on: workflow_call: + jobs: unitTests: name: "๐Ÿงช Unit Tests" runs-on: buildjet-8vcpu-ubuntu-2204 steps: - name: โฌ‡๏ธ Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -25,6 +27,5 @@ jobs: - name: ๐Ÿ“ฅ Download deps run: pnpm install --frozen-lockfile - - name: Run Unit Tests - run: | - pnpm run test + - name: ๐Ÿงช Run Unit Tests + run: pnpm run test diff --git a/docs/config/config-file.mdx b/docs/config/config-file.mdx index 6613314104..44866a8083 100644 --- a/docs/config/config-file.mdx +++ b/docs/config/config-file.mdx @@ -70,7 +70,7 @@ export default defineConfig({ }); ``` -Read more about task lifecycle functions in the [tasks overview](/tasks-overview). +Read more about task lifecycle functions in the [tasks overview](/tasks/overview). ## Instrumentations @@ -116,7 +116,7 @@ export default defineConfig({ }); ``` -See our [Bun guide](/guides/bun) for more information. +See our [Bun guide](/guides/frameworks/bun) for more information. ## Default machine diff --git a/docs/errors-retrying.mdx b/docs/errors-retrying.mdx index ea40d7bd5d..2a9fd3863b 100644 --- a/docs/errors-retrying.mdx +++ b/docs/errors-retrying.mdx @@ -9,12 +9,12 @@ When an uncaught error is thrown inside your task, that task attempt will fail. You can configure retrying in two ways: -1. In your [trigger.config file](/trigger-config) you can set the default retrying behavior for all tasks. +1. In your [trigger.config file](/config/config-file) you can set the default retrying behavior for all tasks. 2. On each task you can set the retrying behavior. By default when you create your project using the CLI init command we disabled retrying in the DEV - environment. You can enable it in your [trigger.config file](/trigger-config). + environment. You can enable it in your [trigger.config file](/config/config-file). ## A simple example with OpenAI diff --git a/docs/examples/ffmpeg-video-processing.mdx b/docs/examples/ffmpeg-video-processing.mdx index a5a33a0100..31ca6a8943 100644 --- a/docs/examples/ffmpeg-video-processing.mdx +++ b/docs/examples/ffmpeg-video-processing.mdx @@ -22,7 +22,7 @@ export default defineConfig({ ``` - [Build extensions](../guides/build-extensions) allow you to hook into the build system and + [Build extensions](/config/config-file#extensions) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/github-actions.mdx b/docs/github-actions.mdx index cb163aebf3..fb090e1ef0 100644 --- a/docs/github-actions.mdx +++ b/docs/github-actions.mdx @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Use Node.js 20.x uses: actions/setup-node@v4 @@ -56,7 +56,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Use Node.js 20.x uses: actions/setup-node@v4 diff --git a/docs/guides/dashboard/creating-a-project.mdx b/docs/guides/dashboard/creating-a-project.mdx index 73f04f63ee..2854ccbd32 100644 --- a/docs/guides/dashboard/creating-a-project.mdx +++ b/docs/guides/dashboard/creating-a-project.mdx @@ -30,7 +30,7 @@ description: "This guide will show you how to create a new Trigger.dev project." Setup Trigger.dev in 3 minutes - + Learn what tasks are and how to write them diff --git a/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx b/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx index 9ca50a50e3..4cbeb8961e 100644 --- a/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx +++ b/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx @@ -140,7 +140,7 @@ Add a new column called `name` with the type `text`. -By default, Supabase edge functions require a JSON Web Token ([JWT](<(https://supabase.com/docs/guides/auth/jwts)>)) in the authorization header. This is to ensure that only authorized users can access your edge functions. +By default, Supabase edge functions require a JSON Web Token [JWT](https://supabase.com/docs/guides/auth/jwts) in the authorization header. This is to ensure that only authorized users can access your edge functions. In your Supabase project dashboard, click 'Project settings' , then the 'API' tab , and copy the `anon` `public` API key from the table . diff --git a/docs/how-it-works.mdx b/docs/how-it-works.mdx index 45de378b4c..a1eafc53bc 100644 --- a/docs/how-it-works.mdx +++ b/docs/how-it-works.mdx @@ -424,7 +424,7 @@ When you run `npx trigger.dev@latest dev`, we run your task code locally on your Trigger.dev currently does not support "offline" dev mode, where you can run tasks without an - internet connection. [Please let us know](feedback.trigger.dev) if this is a feature you + internet connection. [Please let us know](https://feedback.trigger.dev/) if this is a feature you want/need. diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 1d1325ffbd..90d5d3732a 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -9,8 +9,8 @@ Trigger.dev v3 makes it easy to write reliable long-running tasks without timeou - We run your tasks with no timeouts. You don't have to manage any infrastructure (unless you [self-host](/open-source-self-hosting)). Workers are automatically scaled and managed for you. - We provide a multi-tenant queue that is used when triggering tasks. -- We provide an SDK and CLI for writing tasks in your existing codebase, inside [/trigger folders](/trigger-folder). -- We provide different types of tasks: [regular](/tasks-regular) and [scheduled](/tasks-scheduled). +- We provide an SDK and CLI for writing tasks in your existing codebase, inside [/trigger folders](/config/config-file). +- We provide different types of tasks: [regular](/tasks-regular) and [scheduled](/tasks/scheduled). - We provide a dashboard for monitoring, debugging, and managing your tasks. We're [open source](https://github.com/triggerdotdev/trigger.dev) and you can choose to use the [Trigger.dev Cloud](https://cloud.trigger.dev) or [Self-host Trigger.dev](/open-source-self-hosting) on your own infrastructure. @@ -21,7 +21,7 @@ We're [open source](https://github.com/triggerdotdev/trigger.dev) and you can ch Get started in 3 minutes. - + Tasks are the core of Trigger.dev. Learn what they are and how to write them. diff --git a/docs/migration-defer.mdx b/docs/migration-defer.mdx index 3ccc16587f..b40a29624f 100644 --- a/docs/migration-defer.mdx +++ b/docs/migration-defer.mdx @@ -242,7 +242,7 @@ export async function runLongRunningTask() { #### Example 2: A cron task -We call these [scheduled tasks](/tasks-scheduled) in Trigger.dev. +We call these [scheduled tasks](/tasks/scheduled) in Trigger.dev. In Defer you might have a function like this: @@ -272,6 +272,6 @@ export const sendMondayNewletter = schedules.task({ Then you need to attach a schedule to the task, either using the dashboard or in your code. You can attach unlimited schedules to a task. - + How to attach a schedule to a task diff --git a/docs/mint.json b/docs/mint.json index 1c7c35cb53..be73ca4cdc 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -83,6 +83,10 @@ { "source": "/trigger-folder", "destination": "/config/config-file" + }, + { + "source": "/trigger-config", + "destination": "/config/config-file" } ], "anchors": [ diff --git a/docs/quick-start.mdx b/docs/quick-start.mdx index 75f3d83a51..d6253bc269 100644 --- a/docs/quick-start.mdx +++ b/docs/quick-start.mdx @@ -47,7 +47,7 @@ Once you've created an account, follow the steps in the app to: Learn how to trigger tasks from your code. - + Tasks are the core of Trigger.dev. Learn what they are and how to write them. diff --git a/docs/snippets/useful-next-steps.mdx b/docs/snippets/useful-next-steps.mdx index 3ad1c44ac0..8fdb53af19 100644 --- a/docs/snippets/useful-next-steps.mdx +++ b/docs/snippets/useful-next-steps.mdx @@ -1,7 +1,7 @@ ## Useful next steps - + Learn what tasks are and their options diff --git a/docs/tasks-regular.mdx b/docs/tasks-regular.mdx index 91d3740c80..acc1be9021 100644 --- a/docs/tasks-regular.mdx +++ b/docs/tasks-regular.mdx @@ -7,7 +7,7 @@ import OpenaiRetry from "/snippets/code/openai-retry.mdx" They are defined using the `task()` function and can be [triggered](/triggering) from your backend or inside another task. -Like all tasks they don't have timeouts, they should be placed inside a [/trigger folder](/trigger-folder), and you [can configure them](/tasks-overview#defining-a-task). +Like all tasks they don't have timeouts, they should be placed inside a [/trigger folder](/config/config-file), and you [can configure them](/tasks/overview#defining-a-task). ## Example tasks diff --git a/docs/tasks/scheduled.mdx b/docs/tasks/scheduled.mdx index c9f7221735..da0d6dcb7d 100644 --- a/docs/tasks/scheduled.mdx +++ b/docs/tasks/scheduled.mdx @@ -60,7 +60,7 @@ You can see from the comments that the payload has several useful properties: to do that. -Like all tasks they don't have timeouts, they should be placed inside a [/trigger folder](/trigger-folder), and you [can configure them](/tasks-overview#defining-a-task). +Like all tasks they don't have timeouts, they should be placed inside a [/trigger folder](/config/config-file), and you [can configure them](/tasks/overview#defining-a-task). ## How to attach a schedule diff --git a/docs/triggering.mdx b/docs/triggering.mdx index d69bfd5df5..17c3dc32a4 100644 --- a/docs/triggering.mdx +++ b/docs/triggering.mdx @@ -20,11 +20,11 @@ Trigger tasks **from inside a run**: | `yourTask.triggerAndWait()` | Inside task | Triggers a task and then waits until it's complete. You get the result data to continue with. [Read more](#task-triggerandwait) | | `yourTask.batchTriggerAndWait()` | Inside task | Triggers a task multiple times in parallel and then waits until they're all complete. You get the resulting data to continue with. [Read more](#task-batchtriggerandwait) | -Additionally, [scheduled tasks](/tasks-scheduled) get **automatically** triggered on their schedule and webhooks when receiving a webhook. +Additionally, [scheduled tasks](/tasks/scheduled) get **automatically** triggered on their schedule and webhooks when receiving a webhook. ## Scheduled tasks -You should attach one or more schedules to your `schedules.task()` to trigger it on a recurring schedule. [Read the scheduled tasks docs](/tasks-scheduled). +You should attach one or more schedules to your `schedules.task()` to trigger it on a recurring schedule. [Read the scheduled tasks docs](/tasks/scheduled). ## Authentication diff --git a/docs/writing-tasks-introduction.mdx b/docs/writing-tasks-introduction.mdx index 03cbd5fdd6..f9a5d48b42 100644 --- a/docs/writing-tasks-introduction.mdx +++ b/docs/writing-tasks-introduction.mdx @@ -4,7 +4,7 @@ sidebarTitle: "Introduction" description: "Tasks are the core of Trigger.dev. They are long-running processes that are triggered by events." --- -Before digging deeper into the details of writing tasks, you should read the [fundamentals of tasks](/tasks-overview) to understand what tasks are and how they work. +Before digging deeper into the details of writing tasks, you should read the [fundamentals of tasks](/tasks/overview) to understand what tasks are and how they work. ## Writing tasks