Skip to content

feat: add main dataset sharing on runner or scenario sharing #4666

feat: add main dataset sharing on runner or scenario sharing

feat: add main dataset sharing on runner or scenario sharing #4666

name: Build, Test and Package
on:
workflow_dispatch:
push:
branches:
- main
tags:
- '*'
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
branches:
- 'main'
- 'release/**'
- 'feature/**'
jobs:
build:
runs-on: ubuntu-latest
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Build with Gradle
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: |
build
-x generateClients
-x test
-x integrationTest
-x spotlessCheck
-x openApiValidate
-x detekt
test:
runs-on: ubuntu-latest
needs:
- build
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run unit tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: test -x spotlessCheck -x openApiValidate
connector_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Connector integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-connector-api:integrationTest -x spotlessCheck -x openApiValidate
dataset_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run RBAC Dataset integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-dataset-api:integrationTest --tests "com.cosmotech.dataset.service.DatasetServiceRBACTest" -x spotlessCheck -x openApiValidate
- name: Run Dataset integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-dataset-api:integrationTest --tests "com.cosmotech.dataset.service.DatasetServiceIntegrationTest" -x spotlessCheck -x openApiValidate
organization_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Organization integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-organization-api:integrationTest --tests "com.cosmotech.organization.service.OrganizationServiceIntegrationTest" -x spotlessCheck -x openApiValidate
- name: Run RBAC Organization integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-organization-api:integrationTest --tests "com.cosmotech.organization.service.OrganizationServiceRBACTest" -x spotlessCheck -x openApiValidate
scenario_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run RBAC integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-scenario-api:integrationTest --tests "com.cosmotech.scenario.service.ScenarioServiceRBACTest" -x spotlessCheck -x openApiValidate
- name: Run Scenario integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-scenario-api:integrationTest --tests "com.cosmotech.scenario.service.ScenarioServiceIntegrationTest" -x spotlessCheck -x openApiValidate
scenariorun_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Scenario Run integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-scenariorun-api:integrationTest --tests "com.cosmotech.scenariorun.service.ScenarioRunServiceIntegrationTest" -x spotlessCheck -x openApiValidate
- name: Run RBAC Scenario Run integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-scenariorun-api:integrationTest --tests "com.cosmotech.scenariorun.service.ScenarioRunServiceRBACTest" -x spotlessCheck -x openApiValidate
solution_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Solution integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-solution-api:integrationTest --tests "com.cosmotech.solution.service.SolutionServiceIntegrationTest" -x spotlessCheck -x openApiValidate
- name: Run RBAC Solution integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-solution-api:integrationTest --tests "com.cosmotech.solution.service.SolutionServiceRBACTest" -x spotlessCheck -x openApiValidate
twingraph_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Twin Graph integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-twingraph-api:integrationTest -x spotlessCheck -x openApiValidate
workspace_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Workspace integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-workspace-api:integrationTest --tests "com.cosmotech.worskpace.service.WorkspaceServiceIntegrationTest" -x spotlessCheck -x openApiValidate
- name: Run RBAC Workspace integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-workspace-api:integrationTest --tests "com.cosmotech.worskpace.service.WorkspaceServiceRBACTest" -x spotlessCheck -x openApiValidate
runner_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run Runner integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-runner-api:integrationTest --tests "com.cosmotech.runner.service.RunnerServiceIntegrationTest" -x spotlessCheck -x openApiValidate
- name: Run RBAC Runner integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-runner-api:integrationTest --tests "com.cosmotech.runner.service.RunnerServiceRBACTest" -x spotlessCheck -x openApiValidate
run_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Launch Run integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-run-api:integrationTest -x spotlessCheck -x openApiValidate
homecontroller_it:
runs-on: ubuntu-latest
needs:
- test
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
permissions: read-all
if: github.event_name != 'pull_request_target'
steps:
- uses: actions/checkout@v4
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Run HomeController integration tests
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: :cosmotech-api:integrationTest -x spotlessCheck -x openApiValidate
scan_and_push_container_images_to_registries:
runs-on: ubuntu-latest
needs:
- test
- connector_it
- dataset_it
- organization_it
- scenario_it
- scenariorun_it
- solution_it
- twingraph_it
- workspace_it
- runner_it
- run_it
- homecontroller_it
permissions:
packages: write
contents: read
security-events: write
env:
DOCKER_BUILDKIT: 1
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v4
with:
# Fetch all tags since Gradle project version is built upon SCM
fetch-depth: 0
- name: Set up JDK
uses: actions/setup-java@v3
with:
distribution: 'temurin'
java-version: '21'
- name: Retrieve branch or tag name
id: refvar
run: echo "::set-output name=gitRefName::${GITHUB_REF#refs/*/}"
- name: Build local Container Image for scanning
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: gradle/gradle-build-action@v3
with:
cache-disabled: true
# Cache storage space is limited for GitHub actions
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
arguments: |
:cosmotech-api:jibDockerBuild
-Djib.to.image=com.cosmotech/cosmotech-api:${{ github.sha }}
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
id: scan
# Add TRIVY_DB_REPOSITORY due to ratelimit issue
# https://github.com/aquasecurity/trivy-action/issues/389
env:
DOCKLE_HOST: "unix:///var/run/docker.sock"
TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1
# TODO For now, do not block if the container scan action returns issues
continue-on-error: true
with:
image-ref: com.cosmotech/cosmotech-api:${{ github.sha }}
format: "sarif"
severity: "CRITICAL,HIGH"
output: "trivy-results.sarif"
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: "trivy-results.sarif"
- name: Archive container image scan report
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: container-image-scan-report
path: "trivy-results.sarif"
retention-days: 3
- name: Login to GitHub Container Registry
uses: docker/[email protected]
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }}
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push API Container Image (sha) to GitHub Container Registry
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }}
run: |
docker image tag com.cosmotech/cosmotech-api:${GITHUB_SHA} \
ghcr.io/cosmo-tech/cosmotech-api:${GITHUB_SHA}
docker image push ghcr.io/cosmo-tech/cosmotech-api:${GITHUB_SHA}
- name: Push API Container Image (tag) to GitHub Container Registry
if: startsWith(github.ref, 'refs/tags/')
run: |
docker image tag com.cosmotech/cosmotech-api:${GITHUB_SHA} \
ghcr.io/cosmo-tech/cosmotech-api:${{ steps.refvar.outputs.gitRefName }}
docker image push ghcr.io/cosmo-tech/cosmotech-api:${{ steps.refvar.outputs.gitRefName }}
- name: Push API Container Image (latest) to GitHub Container Registry
if: github.ref == 'refs/heads/main'
run: |
docker image tag com.cosmotech/cosmotech-api:${GITHUB_SHA} \
ghcr.io/cosmo-tech/cosmotech-api:latest
docker image push ghcr.io/cosmo-tech/cosmotech-api:latest
push_helm_charts_to_oci_registries:
runs-on: ubuntu-latest
if: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }}
strategy:
fail-fast: false
matrix:
include:
- path: api/kubernetes/helm-chart
chart_name: cosmotech-api-chart
name: API
needs:
- test
- connector_it
- dataset_it
- organization_it
- scenario_it
- scenariorun_it
- solution_it
- twingraph_it
- workspace_it
- runner_it
- run_it
- homecontroller_it
permissions:
packages: write
contents: read
env:
# OCI Support by Helm is considered experimental
HELM_EXPERIMENTAL_OCI: 1
steps:
- uses: actions/checkout@v4
- name: Set up Helm
uses: azure/[email protected]
with:
version: v3.6.0
- name: Retrieve branch or tag name
id: refvar
run: echo "::set-output name=gitRefName::${GITHUB_REF#refs/*/}"
- name: Login to GitHub Container Registry
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
echo "${GITHUB_TOKEN}" | \
helm registry login ghcr.io \
--username "${{ github.actor }}" \
--password-stdin
- name: Push Helm Chart (sha) for ${{ matrix.name }} to GitHub Container Registry
run: |
helm dependency update ${{ matrix.path }}
helm chart save ${{ matrix.path }} ghcr.io/cosmo-tech/${{ matrix.chart_name }}:${GITHUB_SHA}
helm chart push ghcr.io/cosmo-tech/${{ matrix.chart_name }}:${GITHUB_SHA}
- name: Push Helm Chart (tag) for ${{ matrix.name }} to GitHub Container Registry
if: startsWith(github.ref, 'refs/tags/')
run: |
helm chart save ${{ matrix.path }} ghcr.io/cosmo-tech/${{ matrix.chart_name }}:${{ steps.refvar.outputs.gitRefName }}
helm chart push ghcr.io/cosmo-tech/${{ matrix.chart_name }}:${{ steps.refvar.outputs.gitRefName }}
- name: Push Helm Chart (latest) for ${{ matrix.name }} to GitHub Container Registry
if: github.ref == 'refs/heads/main'
run: |
helm chart save ${{ matrix.path }} ghcr.io/cosmo-tech/${{ matrix.chart_name }}:latest
helm chart push ghcr.io/cosmo-tech/${{ matrix.chart_name }}:latest
- name: Logout from GitHub Container Registry
if: ${{ always() }}
run: |
helm registry logout ghcr.io || true