changes for self-hosted runner (#64)

* self-hosted changes

* remove venv

* remove venv

* test main branch

* reset workflow branch

* add venv, change workflow branch

* activate venv before pw:run

* remove venv for test

* add venv, add pip update

* activate pip cache

* add pip upgrade to venv

* activate venv in cyspress tests

* remove venv

* test different activation of venv

* test without venv

* remove condition from setup python

* remove venv

* install addtional req

* remove comp tests

* clean up

* clean up

* clean up

* remove "disable_cache" input

* fix line change, add cypress_spec, remove pg_ready check

* Rename deduplication-id to deduplication_id

* use vdelev_selfhosted as main branch and add enable_cache flags

* Revert back to main

---------

Co-authored-by: Michael Pühringer <[email protected]>
Co-authored-by: Michael Puehringer <[email protected]>

Author: dvviktordelev

.github/actions/build-node/action.yml

@@ -15,6 +15,10 @@ inputs:
     description: "npm registry"
     default: "https://registry.npmjs.org/"
     required: true
+  enable_cache:
+    description: "enables the yarn cache download and upload"
+    required: false
+    default: true
   download_yarn_lock:
     description: "downloads the node-yarn-lock artifact (only available when build-node was called before)"
     required: false
@@ -47,6 +51,10 @@ runs:
     - name: Git config
       if: inputs.github_ro_token != ''
       run: |
+        if [ -f ~/.gitconfig ]; then
+          rm ~/.gitconfig
+          touch ~/.gitconfig
+        fi
         git config --global --replace-all url."https://[email protected]/".insteadOf ssh://[email protected]/
         git config --add --global url."https://[email protected]/".insteadOf [email protected]:
       env:
@@ -60,6 +68,7 @@ runs:
     # Enable yarn download cache, @see https://github.com/actions/cache/tree/main/save#always-save-cache and https://github.com/actions/setup-node/issues/325
     - name: Restore yarn cache
       uses: actions/cache/restore@v3
+      if: inputs.enable_cache == 'true'
       with:
         # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder
         path: ~/.yarn/berry/cache/
@@ -72,7 +81,7 @@ runs:
       shell: bash
     - name: Save yarn cache
       uses: actions/cache/save@v3
-      if: steps.install.outcome == 'success'
+      if: steps.install.outcome == 'success' && inputs.enable_cache == 'true'
       with:
         # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder
         path: ~/.yarn/berry/cache/

.github/actions/build-python/action.yml

@@ -11,6 +11,10 @@ inputs:
     description: "github read-only token"
     default: "admin"
     required: true
+  enable_cache:
+    description: "enables the pip cache download and upload"
+    required: false
+    default: true
   run_lint:
     default: true
     required: false
@@ -25,10 +29,10 @@ runs:
   using: "composite"
   steps:
     - name: Set up python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python_version }}
-        cache: 'pip'
+        cache: ${{ inputs.enable_cache == 'true' && 'pip' || null }}
     # needs wheel for make
     - name: install additional requirements
       run: |

.github/workflows/build-node-python.yml

@@ -66,6 +66,11 @@ on:
         type: string
         required: false
         default: "ubuntu-22.04-large"
+      deduplication_id:
+        type: string
+        required: false
+        description: Unique id per workflow run. Must be set to unique value if dispatched multiple times for a single workflow.
+        default: ""
       chromatic_enable:
         description: 'Enable Chromatic Tests'
         required: false
@@ -92,6 +97,7 @@ env:
   PYPI_USERNAME: "datavisyn"
   PYTHON_VERSION: "3.10"
   WORKFLOW_BRANCH: "main"
+  POSTGRES_HOSTNAME: postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
 
 permissions:
   contents: write
@@ -112,13 +118,13 @@ jobs:
     runs-on: ${{ inputs.runs_on }}
     steps:
       - name: Checkout source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
           token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN  || github.token  }}
           fetch-depth: ${{ inputs.chromatic_enable && '0' || '1' }}
       - name: Checkout github-workflows
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: datavisyn/github-workflows
           ref: ${{ env.WORKFLOW_BRANCH }}
@@ -130,6 +136,7 @@ jobs:
           npm_registry: ${{ env.NPM_REGISTRY }}
           github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token  }}
           run_webpack: ${{ inputs.node_run_webpack }}
+          enable_cache: ${{ inputs.runs_on != 'self-hosted' }}
       - name: Run Chromatic
         if: ${{ inputs.chromatic_enable }}
         uses: chromaui/[email protected]
@@ -145,12 +152,12 @@ jobs:
     runs-on: ${{ inputs.runs_on }}
     steps:
       - name: Checkout source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
           token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN  || github.token  }}
       - name: Checkout github-workflows
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: datavisyn/github-workflows
           ref: ${{ env.WORKFLOW_BRANCH }}
@@ -160,6 +167,7 @@ jobs:
         with:
           python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
           github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
+          enable_cache: ${{ inputs.runs_on != 'self-hosted' }}
 
   # If cypress is used, build node and python sequentially as it is avoiding the duplicate install overhead
   build-node-python-cypress:
@@ -185,17 +193,34 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --name postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
         ports:
-          - 5432:5432
+          # will assign a random free host port
+          - 5432/tcp
 
     steps:
+      - name: Set self-hosted env variable to github env
+        run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV"
+      - name: Set github token, hostname, port and docker network for self-hosted runner
+        if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != ''
+        run: |
+          {
+            echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
+            echo "POSTGRES_PORT=5432"
+          } >> "$GITHUB_ENV"
+          docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} "postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
+      - name: Set postgres connection details to hosted runner
+        if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == ''
+        run: |
+          echo "POSTGRES_HOSTNAME=localhost" >> "$GITHUB_ENV"
+          echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" >> "$GITHUB_ENV"
       - name: Checkout source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
           token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN  || github.token  }}
       - name: Checkout github-workflows
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: datavisyn/github-workflows
           ref: ${{ env.WORKFLOW_BRANCH }}
@@ -207,11 +232,13 @@ jobs:
           npm_registry: ${{ env.NPM_REGISTRY }}
           github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token  }}
           run_webpack: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install
+          enable_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
       - name: Build python
         uses: ./tmp/github-workflows/.github/actions/build-python
         with:
           python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
           github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
+          enable_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
       - name: Decrypt .env.enc and <app>/.env.enc
         run: |
           yarn run env:decrypt -pass env:ENV_PASSWORD || true
@@ -233,8 +260,10 @@ jobs:
             yarn start dev_server_only=true
             make start
           wait-on: "http://localhost:8080, http://localhost:9000/health"
-          env: ${{ secrets.CYPRESS_ENV }}
           spec: ${{ inputs.cypress_spec_param }}
+        env:
+          POSTGRES_HOSTNAME: ${{ env.POSTGRES_HOSTNAME }}
+          POSTGRES_PORT: ${{ env.POSTGRES_PORT }}
       # rspack currently does not support Cypress component tests
       # - name: Run cypress component tests
       #   uses: cypress-io/github-action@v6
@@ -247,7 +276,7 @@ jobs:
       #     # TODO: Do component tests need the frontend/backend?
       #     env: ${{ secrets.CYPRESS_ENV }}
       - name: Upload cypress screenshots
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: cypress-screenshots
@@ -283,17 +312,34 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+          --name postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}
         ports:
-          - 5432:5432
+          # will assign a random free host port
+          - 5432/tcp
 
     steps:
+      - name: Set system env variable to github env
+        run: echo "GH_ACTIONS_SELF_HOSTED_NETWORK_NAME=${GH_ACTIONS_SELF_HOSTED_NETWORK_NAME}" >> "$GITHUB_ENV"
+      - name: Set github token, hostname, port and docker network for self-hosted runner
+        if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME != ''
+        run: | 
+          {
+            echo "POSTGRES_HOSTNAME=postgres_${{ github.job }}_${{ inputs.deduplication_id }}_${{ github.run_id }}_${{ github.run_attempt }}"
+            echo "POSTGRES_PORT=5432"
+          } >> "$GITHUB_ENV"
+          docker network connect ${{ env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME }} ${{ env.POSTGRES_HOSTNAME }}
+      - name: Set postgres connection details to hosted runner
+        if: env.GH_ACTIONS_SELF_HOSTED_NETWORK_NAME == ''
+        run: |
+          echo "POSTGRES_HOSTNAME=localhost" >> "$GITHUB_ENV"
+          echo "POSTGRES_PORT=${{ job.services.postgres.ports['5432'] }}" >> "$GITHUB_ENV"
       - name: Checkout source repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ inputs.branch }}
           token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN  || github.token  }}
       - name: Checkout github-workflows
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: datavisyn/github-workflows
           ref: ${{ env.WORKFLOW_BRANCH }}
@@ -305,11 +351,13 @@ jobs:
           npm_registry: ${{ env.NPM_REGISTRY }}
           github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token  }}
           run_webpack: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install
+          enable_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
       - name: Build python
         uses: ./tmp/github-workflows/.github/actions/build-python
         with:
           python_version: ${{ secrets.PYTHON_VERSION || env.PYTHON_VERSION }}
           github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}
+          enable_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }}
       - name: Decrypt .env.enc and <app>/.env.enc
         run: |
           yarn run env:decrypt -pass env:ENV_PASSWORD || true