Merge pull request #3618 from AayushSabharwal/as/release-ci

feat: add CI for testing against released package versions

Author: ChrisRackauckas

.github/workflows/ReleaseTest.yml

@@ -0,0 +1,62 @@
+name: ReleaseTest
+on:
+  push:
+    branches: [master]
+    tags: [v*]
+  pull_request:
+    paths-ignore:
+        - 'docs/**'
+
+concurrency:
+  # Skip intermediate builds: always, but for the master branch and tags.
+  # Cancel intermediate builds: always, but for the master branch and tags.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' && github.refs != 'refs/tags/*' }}
+
+jobs:
+  test:
+    name: ${{ matrix.package.package }}/${{ matrix.package.group }}/${{ matrix.julia-version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GROUP: ${{ matrix.package.group }}
+    strategy:
+      fail-fast: false
+      matrix:
+        julia-version: [1]
+        os: [ubuntu-latest]
+        package:
+          - {package: Catalyst, group: All}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.julia-version }}
+          arch: x64
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: Create test directory
+        run: mkdir downstream
+      - name: Load this and run the downstream tests
+        shell: julia --color=yes --project=downstream {0}
+        run: |
+          using Pkg
+          try
+            Pkg.activate("downstream")
+            # force it to use this PR's version of the package
+            Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
+            Pkg.add("${{ matrix.package.package }}")
+            Pkg.update()
+            Pkg.test("Catalyst"; coverage=true)  # resolver may fail with test time deps
+          catch err
+            err isa Pkg.Resolve.ResolverError || rethrow()
+            # If we can't resolve that means this is incompatible by SemVer and this is fine
+            # It means we marked this as a breaking change, so we don't need to worry about
+            # Mistakenly introducing a breaking change, as we have intentionally made one
+            @info "Not compatible with this release. No problem." exception=err
+            exit(0)  # Exit immediately, as a success
+          end
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v5
+        with:
+          files: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false