From aa032c2da613d62324a887efa2901df6f54b655d Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 09:21:17 +0200 Subject: [PATCH 01/18] Split CI build --- .github/actions/cleanup-disk/action.yml | 28 ++++++ .github/actions/cloudflare-purge/action.yml | 22 +++++ .github/actions/sync-s3/action.yml | 30 +++++++ .github/workflows/deploy-staging.yml | 98 ++++++++++++++------- 4 files changed, 144 insertions(+), 34 deletions(-) create mode 100644 .github/actions/cleanup-disk/action.yml create mode 100644 .github/actions/cloudflare-purge/action.yml create mode 100644 .github/actions/sync-s3/action.yml diff --git a/.github/actions/cleanup-disk/action.yml b/.github/actions/cleanup-disk/action.yml new file mode 100644 index 0000000000..deb112f960 --- /dev/null +++ b/.github/actions/cleanup-disk/action.yml @@ -0,0 +1,28 @@ +name: Disk cleanup +description: "Cleanup disk space" +runs: + using: composite + steps: + - name: Free up disk space + shell: bash + run: | + echo "Disk space before cleanup..." + df -h / + echo "Removing unnecessary files to free up disk space..." + # https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832 + sudo rm -rf \ + /opt/hostedtoolcache \ + /opt/google/chrome \ + /opt/microsoft/msedge \ + /opt/microsoft/powershell \ + /opt/pipx \ + /usr/lib/mono \ + /usr/local/julia* \ + /usr/local/lib/android \ + /usr/local/lib/node_modules \ + /usr/local/share/chromium \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift + echo "Disk space after cleanup..." + df -h / diff --git a/.github/actions/cloudflare-purge/action.yml b/.github/actions/cloudflare-purge/action.yml new file mode 100644 index 0000000000..8e406a46ad --- /dev/null +++ b/.github/actions/cloudflare-purge/action.yml @@ -0,0 +1,22 @@ +name: Purge Cloudflare Cache +description: "Purge Cloudflare Cache" +inputs: + zone: + description: "The Cloudflare Zone ID" + required: true + api-token: + description: "The Cloudflare API Token with purge cache permission" + required: true + prefix: + description: "The prefix to purge cache" + required: true +runs: + using: composite + steps: + - name: Purge Cloudflare Cache + shell: bash + run: | + curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ inputs.zone }}/purge_cache" \ + -H "Authorization: Bearer ${{ inputs.api-token }}" \ + -H "Content-Type: application/json" \ + --data '{"prefixes":["${{ inputs.prefix }}"]}' diff --git a/.github/actions/sync-s3/action.yml b/.github/actions/sync-s3/action.yml new file mode 100644 index 0000000000..7f2a7524ac --- /dev/null +++ b/.github/actions/sync-s3/action.yml @@ -0,0 +1,30 @@ +name: Sync assets on S3 +description: "Sync Docs assets on S3" +inputs: + role-to-assume: + description: "The IAM role to assume" + required: true + bucket-name: + description: "The name of the S3 bucket to sync assets to" + required: true +runs: + using: composite + steps: + - name: Configure AWS credentials from Staging account + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: us-east-1 + + - name: Sync all cacheable assets + shell: bash + run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ inputs.bucket-name }}/ + + - name: Sync all non-cacheable assets + shell: bash + # Don't cache any HTML or JSON file: they should always be up-to-dates + run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ inputs.bucket-name }}/ + + - name: Sync PDF + shell: bash + run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ inputs.bucket-name }}/ diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0c67404381..0d144a5698 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -10,12 +10,50 @@ concurrency: group: deploy-staging cancel-in-progress: true +# Allow installation of dependencies permissions: id-token: write contents: read jobs: + #ย This job is used to render datasheets, but only if they have changed. + # It's a separate job so we don't have to cleanup the machine afterwards. + render-datasheets: + name: Render Datasheets + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: "npm" + cache-dependency-path: "package-lock.json" + + - uses: actions/cache@v4 + id: cache + with: + path: static/resources/datasheets + key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} + + - name: Render Datasheets + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh + find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \; + + - name: Export artifact + uses: actions/upload-artifact@v4 + with: + name: datasheets + path: static/resources/datasheets + retention-days: 1 # Only needed to pass it to the next job + build: + name: Build and Deploy + needs: render-datasheets runs-on: ubuntu-latest environment: staging env: @@ -26,20 +64,23 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + + - name: Cleanup runner disk + uses: ./.github/actions/cleanup-disk # Cleanup machine before starting the build + - uses: actions/setup-node@v4 with: node-version: 18 cache: "npm" - cache-dependency-path: "**/package-lock.json" + cache-dependency-path: "package-lock.json" - - name: Render Datasheets - run: cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh + - uses: actions/download-artifact@v4 + with: + name: datasheets - name: Copy Static Files run: | - mkdir -p static/resources/datasheets static/resources/schematics static/resources/pinouts static/resources/models - find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \; - find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \; + mkdir -p static/resources/schematics static/resources/pinouts static/resources/models find ./content/hardware -type f -name "*-full-pinout.pdf" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-pinout.png" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-step.zip" -exec cp {} ./static/resources/models/ \; @@ -63,38 +104,27 @@ jobs: ${{ runner.os }}-public-gatsby-main - run: npm install + - run: npm run build - - name: Clean up node_modules + - name: Clean up node_modules # Just to save space run: rm -rf node_modules - - name: Configure AWS credentials from Staging account - uses: aws-actions/configure-aws-credentials@v4 + - name: Deploy to S3 + uses: ./.github/actions/sync-s3 with: role-to-assume: ${{ secrets.STAGING_IAM_ROLE }} - aws-region: us-east-1 - - # - name: Sync all cacheable assets - # run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - # - name: Sync all non-cacheable assets - # # Don't cache any HTML or JSON file: they should always be up-to-dates - # run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - # - name: Sync PDF - # run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - # - name: Purge cache on CloudFlare - # run: | - # curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ - # -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ - # -H "Content-Type: application/json" \ - # --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' - - - name: Sync all cacheable assets - run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - - name: Sync all non-cacheable assets - # Don't cache any HTML or JSON file: they should always be up-to-dates - run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ + bucket-name: ${{ secrets.STAGING_BUCKET_NAME }} + purge-datasheets: + name: Purge Datasheets cache + needs: build + runs-on: ubuntu-latest + environment: staging + steps: + - name: Invalidate datasheets cache + uses: ./.github/actions/cloudflare-purge + with: + api-token: ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }} + zone: ${{ secrets.CLOUDFLARE_ZONE }} + prefix: ${{ vars.DATASHEETS_BASE_URL }} From 30f285be907d69b9e709fbecd5264b42e3937c91 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 09:32:05 +0200 Subject: [PATCH 02/18] Skip everything on cache hit --- .github/workflows/deploy-staging.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0d144a5698..aecdc0b075 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -22,22 +22,24 @@ jobs: name: Render Datasheets runs-on: ubuntu-latest steps: + - uses: actions/cache@v4 + id: cache + with: + path: static/resources/datasheets + key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} + - uses: actions/checkout@v4 + if: steps.cache.outputs.cache-hit != 'true' with: fetch-depth: 1 - uses: actions/setup-node@v4 + if: steps.cache.outputs.cache-hit != 'true' with: node-version: 18 cache: "npm" cache-dependency-path: "package-lock.json" - - uses: actions/cache@v4 - id: cache - with: - path: static/resources/datasheets - key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} - - name: Render Datasheets if: steps.cache.outputs.cache-hit != 'true' run: | From 9c38d5f03ff1ed5e4a671cd043a99e3dde8405ee Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 09:38:53 +0200 Subject: [PATCH 03/18] Fix output path --- .github/workflows/deploy-staging.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index aecdc0b075..3d749b1624 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -43,8 +43,11 @@ jobs: - name: Render Datasheets if: steps.cache.outputs.cache-hit != 'true' run: | - cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh - find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \; + cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering + ./render-datasheets.sh + cd $GITHUB_WORKSPACE + mkdir -p static/resources/datasheets + find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \; - name: Export artifact uses: actions/upload-artifact@v4 From 94a7be9efa9e688e988160f950c528dcda9b6ce7 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 09:48:58 +0200 Subject: [PATCH 04/18] Move datasheets rendering to own action file --- .github/actions/cloudflare-upload/action.yml | 3 -- .../actions/generate-datasheets/action.yml | 48 +++++++++++++++++++ .github/workflows/deploy-staging.yml | 37 ++------------ 3 files changed, 52 insertions(+), 36 deletions(-) create mode 100644 .github/actions/generate-datasheets/action.yml diff --git a/.github/actions/cloudflare-upload/action.yml b/.github/actions/cloudflare-upload/action.yml index ef274e577b..ceac755bb3 100644 --- a/.github/actions/cloudflare-upload/action.yml +++ b/.github/actions/cloudflare-upload/action.yml @@ -20,7 +20,6 @@ inputs: runs: using: composite steps: - - name: Find PR Preview Comment if: github.event_name == 'pull_request' uses: peter-evans/find-comment@v1 @@ -67,7 +66,6 @@ runs: ๐Ÿš€ Preview this PR: ${{ steps.deploy-cloudflare.outputs.url }} ๐Ÿ“ Commit SHA: ${{ github.sha }} - - name: Update PR Preview Comment if: github.event_name == 'pull_request' && steps.deploy-preview-comment.outputs.comment-id != 0 uses: peter-evans/create-or-update-comment@v1.4.5 @@ -78,4 +76,3 @@ runs: ### ${{ inputs.project-name }} ๐Ÿš€ Preview this PR: ${{ steps.deploy-cloudflare.outputs.url }} ๐Ÿ“ Commit SHA: ${{ github.sha }} - diff --git a/.github/actions/generate-datasheets/action.yml b/.github/actions/generate-datasheets/action.yml new file mode 100644 index 0000000000..3184170f5e --- /dev/null +++ b/.github/actions/generate-datasheets/action.yml @@ -0,0 +1,48 @@ +name: "Generate Datasheets" +description: "Generate product datasheets from markdown files" +inputs: + datasheets-path: + description: "The datasheets path" + required: true + default: static/resources/datasheets + artifact-name: + description: "The name of the output artifact" + required: true + +runs: + using: composite + steps: + - uses: actions/cache@v4 + id: cache + with: + path: ${{ inputs.datasheets-path }} + key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} + + - uses: actions/checkout@v4 + if: steps.cache.outputs.cache-hit != 'true' + with: + fetch-depth: 1 + + - uses: actions/setup-node@v4 + if: steps.cache.outputs.cache-hit != 'true' + with: + node-version: 18 + cache: "npm" + cache-dependency-path: "package-lock.json" + + - name: Render Datasheets + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering + ./render-datasheets.sh + cd $GITHUB_WORKSPACE + mkdir -p ${{ inputs.datasheets-path }} + find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./${{ inputs.datasheets-path }}/ \; + shell: bash + + - name: Export artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.datasheets-path }} + retention-days: 1 # Only needed to pass it to the next job diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 3d749b1624..38aff47a26 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -22,39 +22,10 @@ jobs: name: Render Datasheets runs-on: ubuntu-latest steps: - - uses: actions/cache@v4 - id: cache + - uses: ./.github/actions/generated-datasheets with: - path: static/resources/datasheets - key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} - - - uses: actions/checkout@v4 - if: steps.cache.outputs.cache-hit != 'true' - with: - fetch-depth: 1 - - - uses: actions/setup-node@v4 - if: steps.cache.outputs.cache-hit != 'true' - with: - node-version: 18 - cache: "npm" - cache-dependency-path: "package-lock.json" - - - name: Render Datasheets - if: steps.cache.outputs.cache-hit != 'true' - run: | - cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering - ./render-datasheets.sh - cd $GITHUB_WORKSPACE - mkdir -p static/resources/datasheets - find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \; - - - name: Export artifact - uses: actions/upload-artifact@v4 - with: - name: datasheets - path: static/resources/datasheets - retention-days: 1 # Only needed to pass it to the next job + artifact-name: datasheets + datasheets-path: static/resources/datasheets build: name: Build and Deploy @@ -79,7 +50,7 @@ jobs: cache: "npm" cache-dependency-path: "package-lock.json" - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job with: name: datasheets From 24ce991fbb048fd38812932450c8eb9b4311f416 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 09:50:20 +0200 Subject: [PATCH 05/18] Set name --- .github/workflows/deploy-staging.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 38aff47a26..5284686b9d 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -50,7 +50,8 @@ jobs: cache: "npm" cache-dependency-path: "package-lock.json" - - uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job + - name: Retrieve Datasheets + uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job with: name: datasheets @@ -70,7 +71,7 @@ jobs: restore-keys: | ${{ runner.os }}-cache-gatsby-main - - name: Gatsby Public Folder + - name: Gatsby Public Folder cache uses: actions/cache@v4 id: gatsby-public-folder with: From 96d1694a47a766da7e47236a71cec05de67b6934 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 09:55:03 +0200 Subject: [PATCH 06/18] Add debug step for datasheets --- .github/workflows/deploy-staging.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 5284686b9d..c3a74599ac 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -55,6 +55,9 @@ jobs: with: name: datasheets + - name: Debug datasheet list + run: ls -lah static/resources/datasheets + - name: Copy Static Files run: | mkdir -p static/resources/schematics static/resources/pinouts static/resources/models From 32815659e887f14de25ef0f8db281028f4d7df50 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:01:36 +0200 Subject: [PATCH 07/18] Update production workflow, too --- .github/workflows/deploy-production.yml | 82 ++++++++++++++----------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index cae7749e33..8ec2f2401c 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -10,37 +10,57 @@ concurrency: group: deploy-production cancel-in-progress: true +# Allow installation of dependencies permissions: id-token: write contents: read jobs: + #ย This job is used to render datasheets, but only if they have changed. + # It's a separate job so we don't have to cleanup the machine afterwards. + render-datasheets: + name: Render Datasheets + runs-on: ubuntu-latest + steps: + - uses: ./.github/actions/generated-datasheets + with: + artifact-name: datasheets + datasheets-path: static/resources/datasheets + build: - if: "github.repository == 'arduino/docs-content'" + name: Build and Deploy + needs: render-datasheets runs-on: ubuntu-latest environment: production env: REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} - APP_ENV: prod + APP_ENV: production steps: - uses: actions/checkout@v4 with: fetch-depth: 1 + + - name: Cleanup runner disk + uses: ./.github/actions/cleanup-disk # Cleanup machine before starting the build + - uses: actions/setup-node@v4 with: node-version: 18 cache: "npm" - cache-dependency-path: "**/package-lock.json" + cache-dependency-path: "package-lock.json" + + - name: Retrieve Datasheets + uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job + with: + name: datasheets - - name: Render Datasheets - run: cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh + - name: Debug datasheet list + run: ls -lah static/resources/datasheets - name: Copy Static Files run: | - mkdir -p static/resources/datasheets static/resources/schematics static/resources/pinouts static/resources/models - find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \; - find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \; + mkdir -p static/resources/schematics static/resources/pinouts static/resources/models find ./content/hardware -type f -name "*-full-pinout.pdf" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-pinout.png" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-step.zip" -exec cp {} ./static/resources/models/ \; @@ -54,7 +74,7 @@ jobs: restore-keys: | ${{ runner.os }}-cache-gatsby-main - - name: Gatsby Public Folder + - name: Gatsby Public Folder cache uses: actions/cache@v4 id: gatsby-public-folder with: @@ -64,37 +84,27 @@ jobs: ${{ runner.os }}-public-gatsby-main - run: npm install + - run: npm run build - - name: Clean up node_modules + - name: Clean up node_modules # Just to save space run: rm -rf node_modules - - name: Configure AWS credentials from Production account - uses: aws-actions/configure-aws-credentials@v4 + - name: Deploy to S3 + uses: ./.github/actions/sync-s3 with: role-to-assume: ${{ secrets.PRODUCTION_IAM_ROLE }} - aws-region: us-east-1 + bucket-name: ${{ secrets.PRODUCTION_BUCKET_NAME }} - - name: Sync all cacheable assets - run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - - name: Sync all non-cacheable assets - # Don't cache any HTML or JSON file: they should always be up-to-dates - run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - # - name: Sync PDF - # run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - # - name: Purge cache on CloudFlare - # run: | - # curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ - # -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ - # -H "Content-Type: application/json" \ - # --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' - - - name: Sync all cacheable assets - run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - - name: Sync all non-cacheable assets - # Don't cache any HTML or JSON file: they should always be up-to-dates - run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ + purge-datasheets: + name: Purge Datasheets cache + needs: build + runs-on: ubuntu-latest + environment: production + steps: + - name: Invalidate datasheets cache + uses: ./.github/actions/cloudflare-purge + with: + api-token: ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }} + zone: ${{ secrets.CLOUDFLARE_ZONE }} + prefix: ${{ vars.DATASHEETS_BASE_URL }} From fd38095b4e2f431f1cd1691c1cbc3d065ee8252a Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:08:11 +0200 Subject: [PATCH 08/18] Fix APP_ENV --- .github/workflows/deploy-production.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 8ec2f2401c..b07839ca75 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -34,7 +34,7 @@ jobs: environment: production env: REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }} - APP_ENV: production + APP_ENV: prod steps: - uses: actions/checkout@v4 From 21d2fbd09ef774780937870d56c28f61fb9a532d Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:15:34 +0200 Subject: [PATCH 09/18] Checkout is inevitable --- .github/actions/generate-datasheets/action.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/actions/generate-datasheets/action.yml b/.github/actions/generate-datasheets/action.yml index 3184170f5e..c08ecc47ef 100644 --- a/.github/actions/generate-datasheets/action.yml +++ b/.github/actions/generate-datasheets/action.yml @@ -12,16 +12,18 @@ inputs: runs: using: composite steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - uses: actions/cache@v4 id: cache with: path: ${{ inputs.datasheets-path }} key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} - - - uses: actions/checkout@v4 - if: steps.cache.outputs.cache-hit != 'true' - with: - fetch-depth: 1 + # FIXME: remove restore-keys when the cache is stable + restore-keys: | + ${{ runner.os }}-datasheets- - uses: actions/setup-node@v4 if: steps.cache.outputs.cache-hit != 'true' From 1bf0a2c488f5cd0b6b96506dcbed0f7a1b36e060 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:17:34 +0200 Subject: [PATCH 10/18] Fix purge --- .github/actions/cloudflare-purge/action.yml | 22 --------------------- .github/workflows/deploy-production.yml | 13 ++++++------ .github/workflows/deploy-staging.yml | 13 ++++++------ 3 files changed, 14 insertions(+), 34 deletions(-) delete mode 100644 .github/actions/cloudflare-purge/action.yml diff --git a/.github/actions/cloudflare-purge/action.yml b/.github/actions/cloudflare-purge/action.yml deleted file mode 100644 index 8e406a46ad..0000000000 --- a/.github/actions/cloudflare-purge/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Purge Cloudflare Cache -description: "Purge Cloudflare Cache" -inputs: - zone: - description: "The Cloudflare Zone ID" - required: true - api-token: - description: "The Cloudflare API Token with purge cache permission" - required: true - prefix: - description: "The prefix to purge cache" - required: true -runs: - using: composite - steps: - - name: Purge Cloudflare Cache - shell: bash - run: | - curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ inputs.zone }}/purge_cache" \ - -H "Authorization: Bearer ${{ inputs.api-token }}" \ - -H "Content-Type: application/json" \ - --data '{"prefixes":["${{ inputs.prefix }}"]}' diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index b07839ca75..61f7378eee 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -102,9 +102,10 @@ jobs: runs-on: ubuntu-latest environment: production steps: - - name: Invalidate datasheets cache - uses: ./.github/actions/cloudflare-purge - with: - api-token: ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }} - zone: ${{ secrets.CLOUDFLARE_ZONE }} - prefix: ${{ vars.DATASHEETS_BASE_URL }} + - name: Purge Cloudflare Cache + shell: bash + run: | + curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ + -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index c3a74599ac..0cf82061b5 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -102,9 +102,10 @@ jobs: runs-on: ubuntu-latest environment: staging steps: - - name: Invalidate datasheets cache - uses: ./.github/actions/cloudflare-purge - with: - api-token: ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }} - zone: ${{ secrets.CLOUDFLARE_ZONE }} - prefix: ${{ vars.DATASHEETS_BASE_URL }} + - name: Purge Cloudflare Cache + shell: bash + run: | + curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ + -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' From ac64742264caedbaf20d856b58be2a50abf3482d Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:19:09 +0200 Subject: [PATCH 11/18] Explicit checkout --- .github/actions/generate-datasheets/action.yml | 4 ---- .github/workflows/deploy-production.yml | 7 ++++++- .github/workflows/deploy-staging.yml | 7 ++++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/actions/generate-datasheets/action.yml b/.github/actions/generate-datasheets/action.yml index c08ecc47ef..64498eb1bd 100644 --- a/.github/actions/generate-datasheets/action.yml +++ b/.github/actions/generate-datasheets/action.yml @@ -12,10 +12,6 @@ inputs: runs: using: composite steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - uses: actions/cache@v4 id: cache with: diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 61f7378eee..6eddde7f2c 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -22,7 +22,12 @@ jobs: name: Render Datasheets runs-on: ubuntu-latest steps: - - uses: ./.github/actions/generated-datasheets + + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: ./.github/actions/generate-datasheets with: artifact-name: datasheets datasheets-path: static/resources/datasheets diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0cf82061b5..ec1b10ce34 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -22,7 +22,12 @@ jobs: name: Render Datasheets runs-on: ubuntu-latest steps: - - uses: ./.github/actions/generated-datasheets + + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: ./.github/actions/generate-datasheets with: artifact-name: datasheets datasheets-path: static/resources/datasheets From c6c1c5832faf65f4fabe35f8bf12c15037c50350 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:20:07 +0200 Subject: [PATCH 12/18] Typo --- .github/workflows/deploy-production.yml | 5 ++--- .github/workflows/deploy-staging.yml | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 6eddde7f2c..d209316b13 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -22,10 +22,9 @@ jobs: name: Render Datasheets runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 + with: + fetch-depth: 1 - uses: ./.github/actions/generate-datasheets with: diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index ec1b10ce34..a4d1b25898 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -22,10 +22,9 @@ jobs: name: Render Datasheets runs-on: ubuntu-latest steps: - - - uses: actions/checkout@v4 - with: - fetch-depth: 1 + - uses: actions/checkout@v4 + with: + fetch-depth: 1 - uses: ./.github/actions/generate-datasheets with: From edc7cb2a7a83d1a46f4239ae2ec0efcdf08f5298 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 10:24:15 +0200 Subject: [PATCH 13/18] Remove restore-key --- .github/actions/generate-datasheets/action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/generate-datasheets/action.yml b/.github/actions/generate-datasheets/action.yml index 64498eb1bd..908c4f6d10 100644 --- a/.github/actions/generate-datasheets/action.yml +++ b/.github/actions/generate-datasheets/action.yml @@ -17,9 +17,6 @@ runs: with: path: ${{ inputs.datasheets-path }} key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} - # FIXME: remove restore-keys when the cache is stable - restore-keys: | - ${{ runner.os }}-datasheets- - uses: actions/setup-node@v4 if: steps.cache.outputs.cache-hit != 'true' From 80aa827eaf7300cf77417972566bf9720dae41ce Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Mon, 9 Sep 2024 11:19:14 +0200 Subject: [PATCH 14/18] Use correct folder --- .github/workflows/deploy-production.yml | 1 + .github/workflows/deploy-staging.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index d209316b13..85fedea597 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -58,6 +58,7 @@ jobs: uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job with: name: datasheets + path: static/resources/datasheets - name: Debug datasheet list run: ls -lah static/resources/datasheets diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index a4d1b25898..a49bc0fc1b 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -58,6 +58,7 @@ jobs: uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job with: name: datasheets + path: static/resources/datasheets - name: Debug datasheet list run: ls -lah static/resources/datasheets From 100910f0fe9824ed937d2db2d6c05a1bf56699de Mon Sep 17 00:00:00 2001 From: Matteo Marsala Date: Mon, 9 Sep 2024 12:09:45 +0200 Subject: [PATCH 15/18] 2.0.18-2-alpha.1 --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 34d271db0d..3d706c58bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { - "@arduino/docs-arduino-cc": "^2.0.18-1", + "@arduino/docs-arduino-cc": "^2.0.18-2-alpha.7", "gatsby": "^5.11.0", "gatsby-background-image": "^1.6.0", "gatsby-image": "^3.11.0", @@ -339,9 +339,9 @@ } }, "node_modules/@arduino/docs-arduino-cc": { - "version": "2.0.18-1", - "resolved": "https://npm.pkg.github.com/download/@arduino/docs-arduino-cc/2.0.18-1/c0d48794d00fcc07359d7bec8a9b598457d8d6e4", - "integrity": "sha512-h78/M4NjezAHK9qHDjW1rwcDQGdGNK9mh+yMQVitVNFu9N2EvkR8Q/j0O5J1X3FthYsHjP9FhPyP6Qs8sseWPw==", + "version": "2.0.18-2-alpha.7", + "resolved": "https://npm.pkg.github.com/download/@arduino/docs-arduino-cc/2.0.18-2-alpha.7/6bd9abd7f466fd84529f1136ae43d19188a87382", + "integrity": "sha512-9Gz8XPTlDJidq45027/Vjh1Qn94aTxlxp4vTShe0g/dJDQ3Z8BT9LFz5PGSVBpcd4W0WA+tdMU/pGaYHCw88/Q==", "dependencies": { "@algolia/autocomplete-core": "^1.10.0", "@algolia/autocomplete-plugin-recent-searches": "^1.17.0", diff --git a/package.json b/package.json index 0119b99d23..95e88b5bd5 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ }, "homepage": "https://github.com/arduino/docs-content#readme", "dependencies": { - "@arduino/docs-arduino-cc": "^2.0.18-1", + "@arduino/docs-arduino-cc": "^2.0.18-2-alpha.7", "gatsby": "^5.11.0", "gatsby-background-image": "^1.6.0", "gatsby-image": "^3.11.0", From 16ba6d281ca85976d90e0c297336f381c5daf7a5 Mon Sep 17 00:00:00 2001 From: Matteo Marsala Date: Tue, 10 Sep 2024 10:06:12 +0200 Subject: [PATCH 16/18] 2.0.18-2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3d706c58bd..46007a2a0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { - "@arduino/docs-arduino-cc": "^2.0.18-2-alpha.7", + "@arduino/docs-arduino-cc": "^2.0.18-2", "gatsby": "^5.11.0", "gatsby-background-image": "^1.6.0", "gatsby-image": "^3.11.0", diff --git a/package.json b/package.json index 95e88b5bd5..2fa2064e9a 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ }, "homepage": "https://github.com/arduino/docs-content#readme", "dependencies": { - "@arduino/docs-arduino-cc": "^2.0.18-2-alpha.7", + "@arduino/docs-arduino-cc": "^2.0.18-2", "gatsby": "^5.11.0", "gatsby-background-image": "^1.6.0", "gatsby-image": "^3.11.0", From 7d7950eafe639f849cc312cb768444b985032842 Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Tue, 10 Sep 2024 11:31:57 +0200 Subject: [PATCH 17/18] Fail on purge fail, debug prefix and zone --- .github/workflows/deploy-production.yml | 3 ++- .github/workflows/deploy-staging.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 85fedea597..5dea79cf88 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -110,7 +110,8 @@ jobs: - name: Purge Cloudflare Cache shell: bash run: | - curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ + echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ secrets.CLOUDFLARE_ZONE }}" + curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \ -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ -H "Content-Type: application/json" \ --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index a49bc0fc1b..986456272a 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -110,7 +110,8 @@ jobs: - name: Purge Cloudflare Cache shell: bash run: | - curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ + echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ secrets.CLOUDFLARE_ZONE }}" + curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \ -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ -H "Content-Type: application/json" \ --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' From c0712cdc0363fc0a983d6b0da65ea2b05c2ff3bd Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Tue, 10 Sep 2024 12:54:11 +0200 Subject: [PATCH 18/18] Remove debug log --- .github/workflows/deploy-production.yml | 2 +- .github/workflows/deploy-staging.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 5dea79cf88..6c920f8340 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -110,7 +110,7 @@ jobs: - name: Purge Cloudflare Cache shell: bash run: | - echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ secrets.CLOUDFLARE_ZONE }}" + echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ vars.CLOUDFLARE_ZONE }}" curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \ -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ -H "Content-Type: application/json" \ diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 986456272a..ba16e2bf08 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -110,7 +110,7 @@ jobs: - name: Purge Cloudflare Cache shell: bash run: | - echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ secrets.CLOUDFLARE_ZONE }}" + echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ vars.CLOUDFLARE_ZONE }}" curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \ -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ -H "Content-Type: application/json" \