Skip to content
.github/workflows/build-node-python.yml

Add python version inputs #4

Sign in to view logs

Add python version inputs

Add python version inputs #4

Workflow file for this run

.github/workflows/build-node-python.yml at 81a707c
name: build-node-python

Check failure on line 1 in .github/workflows/build-node-python.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/build-node-python.yml

Invalid workflow file 

(Line: 118, Col: 17): Unexpected symbol: '"3'. Located at position 22 within expression: vars.NODE_VERSION || "3.10", (Line: 121, Col: 19): Unexpected symbol: '"3'. Located at position 24 within expression: vars.PYTHON_VERSION || "3.10"
on:
workflow_call:
inputs:
branch:
type: string
required: false
cypress_enable:
description: "Global enable for cypress"
type: boolean
required: false
default: false
cypress_run_because_flag:
description: "Local flag to run cypress. Will only work if cypress_enable: true."
type: boolean
required: false
default: false
cypress_run_because_branch:
description: "Branch names to run cypress on. Will only work if cypress_enable: true."
# TODO: actionlint is failing as it expects true/false as default value for boolean types
type: string
required: false
default: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' || !!startsWith(github.ref, 'refs/heads/release-') || !!startsWith(github.ref, 'refs/heads/dependabot')}}
cypress_spec_param:
description: "Value for the `--spec` parameter. Example: `cypress/e2e/your_spec.cy.ts`. Will only work if cypress_enable: true."
type: string
required: false
default: ''
playwright_enable:
description: "Global enable for playwright"
type: boolean
required: false
default: false
playwright_run_because_flag:
description: "Local flag to run playwright. Will only work if playwright_enable: true."
type: boolean
required: false
default: false
playwright_run_because_branch:
description: "Branch names to run playwright on. Will only work if playwright_enable: true."
# TODO: actionlint is failing as it expects true/false as default value for boolean types
type: string
required: false
default: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop' || !!startsWith(github.ref, 'refs/heads/release-') || !!startsWith(github.ref, 'refs/heads/dependabot')}}
playwright_spec_param:
type: string
description: "Run specific test(s). Example: `landing-page.spec.ts`"
required: false
node_run_webpack:
description: "Flag to always run the webpack production build."
type: boolean
required: false
default: false
runs_on:
type: string
required: false
default: "ubuntu-22.04"
run_parallel:
description: "Flag to ensure that node and python jobs are run in parallel."
type: boolean
required: false
default: true
cypress_runs_on:
type: string
required: false
default: "ubuntu-22.04-large"
playwright_runs_on:
type: string
required: false
default: "ubuntu-22.04-large"
deduplication_id:
type: string
required: false
description: Unique id per workflow run. Must be set to unique value if dispatched multiple times for a single workflow.
default: ""
chromatic_enable:
description: 'Enable Chromatic tests'
required: false
type: boolean
default: false
rust_enable:
description: 'Enable Rust install'
required: false
type: boolean
default: false
cancel_in_progress:
type: boolean
default: true
timeout:
description: "Timeout for each job in minutes."
type: number
required: false
default: 60
python_version:
type: string
required: false
node_version:
type: string
required: false
secrets:
DATAVISYN_BOT_REPO_TOKEN:
required: false
NODE_VERSION:
required: false
PYTHON_VERSION:
required: false
ENV_PASSWORD:
required: false
CYPRESS_ENV:
required: false
CHROMATIC_PROJECT_TOKEN:
required: false
env:
NPM_REGISTRY: "https://registry.npmjs.org/"
NODE_VERSION: ${{ vars.NODE_VERSION || "3.10" }}
PYPI_REGISTRY: "https://upload.pypi.org/legacy/"
PYPI_USERNAME: "datavisyn"
PYTHON_VERSION: ${{ vars.PYTHON_VERSION || "3.10" }}
WORKFLOW_BRANCH: "main"
POSTGRES_HOSTNAME: postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
permissions:
contents: write
packages: write
actions: write
id-token: write
jobs:
build-node:
name: Node
timeout-minutes: ${{ fromJSON(inputs.timeout) }}
if: ${{ (!inputs.cypress_enable || (!inputs.cypress_run_because_flag && inputs.cypress_run_because_branch != 'true')) && (!inputs.playwright_enable || (!inputs.playwright_run_because_flag && inputs.playwright_run_because_branch != 'true')) }}
concurrency:
group: "node-${{ github.workflow }}-${{ github.ref || github.head_ref }}-${{ inputs.branch }}"
cancel-in-progress: ${{ inputs.cancel_in_progress }}
permissions:
id-token: write
contents: write
runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
steps:
- name: Checkout source repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || github.sha }}
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
fetch-depth: ${{ inputs.chromatic_enable && '0' || '1' }}
- name: Checkout github-workflows
uses: actions/checkout@v4
with:
repository: datavisyn/github-workflows
ref: ${{ env.WORKFLOW_BRANCH }}
path: ./tmp/github-workflows
- name: Build node
uses: ./tmp/github-workflows/.github/actions/build-node-python
with:
enable_node: true
enable_python: false
# We probably won't need Rust on Node builds...
# enable_rust: ${{ inputs.rust_enable }}
run_parallel: ${{ inputs.run_parallel }}
node_version: ${{ inputs.node_version || inputs.node_version || secrets.NODE_VERSION || env.NODE_VERSION }}
npm_registry: ${{ env.NPM_REGISTRY }}
python_version: ${{ inputs.python_version || inputs.python_version || secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
run_node_bundle: ${{ inputs.node_run_webpack }}
enable_node_cache: ${{ inputs.runs_on != 'self-hosted' }}
enable_python_cache: ${{ inputs.runs_on != 'self-hosted' }}
chromatic_enable: ${{ inputs.chromatic_enable }}
chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
build-python:
name: Python
timeout-minutes: ${{ fromJSON(inputs.timeout) }}
if: ${{ (!inputs.cypress_enable || (!inputs.cypress_run_because_flag && inputs.cypress_run_because_branch != 'true')) && (!inputs.playwright_enable || (!inputs.playwright_run_because_flag && inputs.playwright_run_because_branch != 'true')) }}
concurrency:
group: "python-${{ github.workflow }}-${{ github.ref || github.head_ref }}-${{ inputs.branch }}"
cancel-in-progress: ${{ inputs.cancel_in_progress }}
permissions:
id-token: write
contents: write
runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }}
steps:
- name: Checkout source repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || github.sha }}
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
- name: Checkout github-workflows
uses: actions/checkout@v4
with:
repository: datavisyn/github-workflows
ref: ${{ env.WORKFLOW_BRANCH }}
path: ./tmp/github-workflows
- name: Build python
uses: ./tmp/github-workflows/.github/actions/build-node-python
with:
enable_node: false
enable_python: true
enable_rust: ${{ inputs.rust_enable }}
run_parallel: ${{ inputs.run_parallel }}
node_version: ${{ inputs.node_version || secrets.NODE_VERSION || env.NODE_VERSION }}
npm_registry: ${{ env.NPM_REGISTRY }}
python_version: ${{ inputs.python_version || secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
run_node_bundle: ${{ inputs.node_run_webpack }}
enable_node_cache: ${{ inputs.runs_on != 'self-hosted' }}
enable_python_cache: ${{ inputs.runs_on != 'self-hosted' }}
# If cypress is used, build node and python sequentially as it is avoiding the duplicate install overhead
build-node-python-cypress:
name: Node, Python, Cypress
timeout-minutes: ${{ fromJSON(inputs.timeout) }}
if: ${{ inputs.cypress_enable && (inputs.cypress_run_because_flag || inputs.cypress_run_because_branch == 'true') }}
concurrency:
group: "node-python-cypress-${{ github.workflow }}-${{ github.ref || github.head_ref }}-${{ inputs.branch }}"
cancel-in-progress: ${{ inputs.cancel_in_progress }}
permissions:
id-token: write
contents: write
runs-on: ${{ inputs.cypress_runs_on || inputs.runs_on || 'ubuntu-22.04-large' }}
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: admin
POSTGRES_PASSWORD: admin
POSTGRES_DB: db
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
--name postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
ports:
# will assign a random free host port
- 5432/tcp
redis:
image: redis:6
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
--name redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
ports:
- 6379/tcp
steps:
- name: Set self-hosted env variable to github env
run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV"
- name: Set up custom postgres and redis hostname, port and docker network for self-hosted runner
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != ''
env:
REDIS_HOSTNAME: redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
REDIS_PORT: 6379
run: |
{
echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
echo "POSTGRES_PORT=5432"
echo "REDIS_HOSTNAME=redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
echo "REDIS_PORT=6379"
} >> "$GITHUB_ENV"
docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }}
docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.REDIS_HOSTNAME }}
- name: Set service connection details to hosted runner
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == ''
run: |
{
echo "POSTGRES_HOSTNAME=localhost"
echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}"
echo "REDIS_HOSTNAME=localhost"
echo "REDIS_PORT=${{ job.services.redis.ports['6379'] }}"
} >> "$GITHUB_ENV"
- name: Checkout source repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || github.sha }}
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
fetch-depth: ${{ inputs.chromatic_enable && '0' || '1' }}
- name: Checkout github-workflows
uses: actions/checkout@v4
with:
repository: datavisyn/github-workflows
ref: ${{ env.WORKFLOW_BRANCH }}
path: ./tmp/github-workflows
- name: Build node and python
uses: ./tmp/github-workflows/.github/actions/build-node-python
with:
enable_rust: ${{ inputs.rust_enable }}
run_parallel: ${{ inputs.run_parallel }}
node_version: ${{ inputs.node_version || secrets.NODE_VERSION || env.NODE_VERSION }}
npm_registry: ${{ env.NPM_REGISTRY }}
python_version: ${{ inputs.python_version || secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
run_node_bundle: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install
enable_node_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
enable_python_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
chromatic_enable: ${{ inputs.chromatic_enable }}
chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
- name: Decrypt .env.enc and <app>/.env.enc
run: |
yarn run env:decrypt -pass env:ENV_PASSWORD || true
make env_decrypt || true
env:
ENV_PASSWORD: ${{ secrets.ENV_PASSWORD }}
- name: Bundle build
run: yarn run bundle:prod || yarn run webpack:dev
shell: bash
- name: Run cypress e2e tests
uses: cypress-io/github-action@v6
with:
# Currently, the videos are buggy because of low-cpu/electron environment: https://github.com/cypress-io/cypress/issues/8210
config: video=false,modifyObstructiveCode=false
browser: chrome
install: false
# TODO: Create customizable variables for these?
start: |
yarn start dev_server_only=true
make start
wait-on: "http://localhost:8080, http://localhost:9000/health"
spec: ${{ inputs.cypress_spec_param }}
env:
POSTGRES_HOSTNAME: ${{ env.POSTGRES_HOSTNAME }}
POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
# rspack currently does not support Cypress component tests
# - name: Run cypress component tests
# uses: cypress-io/github-action@v6
# with:
# # Currently, the videos are buggy because of low-cpu/electron environment: https://github.com/cypress-io/cypress/issues/8210
# config: video=false,modifyObstructiveCode=false
# browser: chrome
# install: false
# component: true
# # TODO: Do component tests need the frontend/backend?
# env: ${{ secrets.CYPRESS_ENV }}
- name: Upload cypress screenshots
uses: actions/upload-artifact@v4
if: failure()
with:
name: cypress-screenshots
path: cypress/screenshots
# - name: Upload cypress videos
# uses: actions/upload-artifact@v4
# if: always()
# with:
# name: cypress-videos
# path: cypress/videos
# If playwright is used, build node and python sequentially as it is avoiding the duplicate install overhead
build-node-python-playwright:
name: Node, Python, Playwright
timeout-minutes: ${{ fromJSON(inputs.timeout) }}
if: ${{ inputs.playwright_enable && (inputs.playwright_run_because_flag || inputs.playwright_run_because_branch == 'true') }}
concurrency:
group: "node-python-playwright-${{ github.workflow }}-${{ github.ref || github.head_ref }}-${{ inputs.branch }}"
cancel-in-progress: ${{ inputs.cancel_in_progress }}
permissions:
id-token: write
contents: write
runs-on: ${{ inputs.playwright_runs_on || inputs.runs_on || 'ubuntu-22.04-large' }}
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: admin
POSTGRES_PASSWORD: admin
POSTGRES_DB: db
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
--name postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
ports:
# will assign a random free host port
- 5432/tcp
redis:
image: redis:6
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
--name redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
ports:
- 6379/tcp
steps:
- name: Set system env variable to github env
run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV"
- name: Set up custom postgres and redis hostname, port and docker network for self-hosted runner
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != ''
env:
REDIS_HOSTNAME: redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
REDIS_PORT: 6379
run: |
{
echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
echo "POSTGRES_PORT=5432"
echo "REDIS_HOSTNAME=redis_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
echo "REDIS_PORT=6379"
} >> "$GITHUB_ENV"
docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }}
docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.REDIS_HOSTNAME }}
- name: Set service connection details to hosted runner
if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == ''
run: |
{
echo "POSTGRES_HOSTNAME=localhost"
echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}"
echo "REDIS_HOSTNAME=localhost"
echo "REDIS_PORT=${{ job.services.redis.ports['6379'] }}"
} >> "$GITHUB_ENV"
- name: Checkout source repository
uses: actions/checkout@v4
with:
ref: ${{ inputs.branch || github.sha }}
token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
fetch-depth: ${{ inputs.chromatic_enable && '0' || '1' }}
- name: Checkout github-workflows
uses: actions/checkout@v4
with:
repository: datavisyn/github-workflows
ref: ${{ env.WORKFLOW_BRANCH }}
path: ./tmp/github-workflows
- name: Build node and python
uses: ./tmp/github-workflows/.github/actions/build-node-python
with:
enable_rust: ${{ inputs.rust_enable }}
run_parallel: ${{ inputs.run_parallel }}
node_version: ${{ inputs.node_version || secrets.NODE_VERSION || env.NODE_VERSION }}
npm_registry: ${{ env.NPM_REGISTRY }}
python_version: ${{ inputs.python_version || secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
run_node_bundle: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install
run_playwright_browser_install: true
enable_node_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
enable_python_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
chromatic_enable: false # Set to false as we run chromatic below w/ playwright integration
chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
- name: Decrypt .env.enc and <app>/.env.enc
run: |
yarn run env:decrypt -pass env:ENV_PASSWORD || true
make env_decrypt || true
env:
ENV_PASSWORD: ${{ secrets.ENV_PASSWORD }}
- name: Bundle build
run: yarn run bundle:prod || yarn run webpack:dev
shell: bash
- name: Run playwright e2e tests
run: yarn run pw:run ${{ inputs.playwright_spec_param }}
shell: bash
env:
VISYN_CORE__SENTRY__FRONTEND_DSN: ${{ vars.PLAYWRIGHT__VISYN_CORE__SENTRY__FRONTEND_DSN }}
VISYN_CORE__SENTRY__BACKEND_DSN: ${{ vars.PLAYWRIGHT__VISYN_CORE__SENTRY__BACKEND_DSN }}
# Disable upload of bundles as there is no real value...
# - name: Upload bundle
# uses: actions/upload-artifact@v4
# if: always()
# with:
# name: "bundles-${{ inputs.branch || github.sha }}"
# path: bundles/
- name: Upload playwright report
uses: actions/upload-artifact@v4
if: always()
with:
name: "playwright-report-${{ inputs.branch || github.sha }}"
path: playwright-report/
- name: Run Chromatic Playwright
if: ${{ inputs.chromatic_enable }}
uses: chromaui/[email protected]
with:
playwright: true
exitZeroOnChanges: true
exitOnceUploaded: true
projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}