From f0dffb9b94a563582913d59c432a845ad0c28415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 12:41:49 +0100 Subject: [PATCH 01/41] chore: cache NPM dependencies in the smoke-test phase --- .github/workflows/test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58e0cc032..b4b6a693b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,17 @@ jobs: node-version: ${{ matrix.node-version }} - name: Install dependencies run: npm ci --omit=optional + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@v4 + id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-{{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node-{{ matrix.node-version }}- - name: Build workspaces run: npm run build -ws - name: Remove dev dependencies From ba4222aed6cb047a38c567a917df9fa7909646ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 13:03:03 +0100 Subject: [PATCH 02/41] chore: extract to reusable workflow --- .github/workflows/npm-ci.yml | 35 +++++++++++++++++++++++++++++++++++ .github/workflows/test.yml | 17 ++--------------- 2 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/npm-ci.yml diff --git a/.github/workflows/npm-ci.yml b/.github/workflows/npm-ci.yml new file mode 100644 index 000000000..23231fe8a --- /dev/null +++ b/.github/workflows/npm-ci.yml @@ -0,0 +1,35 @@ +name: NPM Install +run-name: "Runner (${{ inputs.runner }}) / Node (${{ inputs.node-version }})" + +on: + workflow_call: + inputs: + runner: + required: true + type: string + node-version: + required: true + type: string + +jobs: + npm-install: + name: "NPM Install (${{ inputs.runner }}) / Node (${{ inputs.node-version }})" + runs-on: ${{ inputs.runner }} + steps: + - name: Install NodeJS ${{ inputs.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + - name: Get npm cache directory + id: npm-cache-dir + shell: bash + run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} + - uses: actions/cache@v4 + id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ inputs.runner }}-node-{{ inputs.node-version }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ inputs.runner }}-node-{{ inputs.node-version }}- + - name: Install dependencies + run: npm ci --omit=optional diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4b6a693b..0552942ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,23 +29,10 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v4 - - name: Install NodeJS ${{ matrix.node-version }} - uses: actions/setup-node@v4 + - uses: ./.github/workflows/npm-ci.yml with: + runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} - - name: Install dependencies - run: npm ci --omit=optional - - name: Get npm cache directory - id: npm-cache-dir - shell: bash - run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} - - uses: actions/cache@v4 - id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-{{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-node-{{ matrix.node-version }}- - name: Build workspaces run: npm run build -ws - name: Remove dev dependencies From f9bd8ff5fd3e6d7da3baef2196ad0a8ca2872eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 13:18:02 +0100 Subject: [PATCH 03/41] fix: use a composite action --- .../npm-setup/action.yml} | 33 +++++++++---------- .github/workflows/test.yml | 5 ++- 2 files changed, 19 insertions(+), 19 deletions(-) rename .github/{workflows/npm-ci.yml => actions/npm-setup/action.yml} (51%) diff --git a/.github/workflows/npm-ci.yml b/.github/actions/npm-setup/action.yml similarity index 51% rename from .github/workflows/npm-ci.yml rename to .github/actions/npm-setup/action.yml index 23231fe8a..517f21b5e 100644 --- a/.github/workflows/npm-ci.yml +++ b/.github/actions/npm-setup/action.yml @@ -1,21 +1,17 @@ -name: NPM Install -run-name: "Runner (${{ inputs.runner }}) / Node (${{ inputs.node-version }})" +name: 'NPM Setup' +description: 'Sets up Node.js and installs NPM dependencies with caching' -on: - workflow_call: - inputs: - runner: - required: true - type: string - node-version: - required: true - type: string +inputs: + runner: + description: 'Runner to use' + required: true + node-version: + description: 'Node.js version to use' + required: true -jobs: - npm-install: - name: "NPM Install (${{ inputs.runner }}) / Node (${{ inputs.node-version }})" - runs-on: ${{ inputs.runner }} - steps: +runs: + using: "composite" + steps: - name: Install NodeJS ${{ inputs.node-version }} uses: actions/setup-node@v4 with: @@ -28,8 +24,9 @@ jobs: id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' with: path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ inputs.runner }}-node-{{ inputs.node-version }}-${{ hashFiles('**/package-lock.json') }} + key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ hashFiles('**/package-lock.json') }} restore-keys: | - ${{ inputs.runner }}-node-{{ inputs.node-version }}- + ${{ inputs.runner }}-node-${{ inputs.node-version }}- - name: Install dependencies + shell: bash run: npm ci --omit=optional diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0552942ea..9cbb9a669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,10 +29,13 @@ jobs: steps: - name: Code checkout uses: actions/checkout@v4 - - uses: ./.github/workflows/npm-ci.yml + + # Uses a composite action for a consistent NPM install including cache + - uses: ./.github/actions/npm-setup with: runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} + - name: Build workspaces run: npm run build -ws - name: Remove dev dependencies From 4fd8da16bbd4793cc66e99001dc65f6a72fa56f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 13:20:19 +0100 Subject: [PATCH 04/41] chore: name step --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9cbb9a669..1f3d47fc5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,8 @@ jobs: uses: actions/checkout@v4 # Uses a composite action for a consistent NPM install including cache - - uses: ./.github/actions/npm-setup + - name: Install Node ${{ matrix.node-version }} and Dependencies + uses: ./.github/actions/npm-setup with: runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} From d374502d5694a225311f191680ba724bc93125d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 13:42:54 +0100 Subject: [PATCH 05/41] fix: save cache --- .github/actions/npm-setup/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 517f21b5e..35ec2bd01 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -30,3 +30,9 @@ runs: - name: Install dependencies shell: bash run: npm ci --omit=optional + - name: Cache npm + id: cache-npm-save + uses: actions/cache/save@v4 + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ steps.npm-cache.outputs.cache-primary-key }} From 56334fa6363d0a58c9a098c8ec8c4b9bd5ebd139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 14:11:20 +0100 Subject: [PATCH 06/41] chore: immediately cache dependencies, not wait until the end --- .github/actions/npm-setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 35ec2bd01..37f98a16b 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -20,7 +20,7 @@ runs: id: npm-cache-dir shell: bash run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} - - uses: actions/cache@v4 + - uses: actions/cache/restore@v4 id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' with: path: ${{ steps.npm-cache-dir.outputs.dir }} From 380e478adfb13d65f6f6b0812291baeb127d984b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 14:47:31 +0100 Subject: [PATCH 07/41] chore: refine cache keys --- .github/actions/npm-setup/action.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 37f98a16b..9ab453f85 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -16,23 +16,21 @@ runs: uses: actions/setup-node@v4 with: node-version: ${{ inputs.node-version }} - - name: Get npm cache directory - id: npm-cache-dir - shell: bash - run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT} - uses: actions/cache/restore@v4 - id: npm-cache # use this to check for `cache-hit` ==> if: steps.npm-cache.outputs.cache-hit != 'true' + id: npm-cache with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ hashFiles('**/package-lock.json') }} + path: node_modules + key: ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ hashFiles('**/package-lock.json') }} restore-keys: | - ${{ inputs.runner }}-node-${{ inputs.node-version }}- + ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}- - name: Install dependencies + if: steps.npm-cache.outputs.cache-hit != 'true' shell: bash run: npm ci --omit=optional - name: Cache npm id: cache-npm-save + if: steps.npm-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: ${{ steps.npm-cache-dir.outputs.dir }} + path: node_modules key: ${{ steps.npm-cache.outputs.cache-primary-key }} From 06371a5c1748e1c2345b39bad8a17a64e3e4c35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 17:23:23 +0100 Subject: [PATCH 08/41] chore: reuse cache in workflows --- .github/workflows/lint.yml | 7 +++++-- .github/workflows/test-template.yml | 10 ++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8923d23fa..d4bddd996 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,10 @@ jobs: uses: actions/setup-node@v4 with: node-version: 22.x - - name: Install dependencies - run: npm ci --omit=optional + - name: Install Node and Dependencies + uses: ./.github/actions/npm-setup + with: + runner: ubuntu-22.04 + node-version: 22.x - name: Code linting run: npm run lint:ci diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index d1437b123..3dfacfae7 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -130,13 +130,11 @@ jobs: - name: Code checkout uses: actions/checkout@v4 - - name: Install NodeJS ${{ inputs.node-version }} - uses: actions/setup-node@v4 + - name: Install Node ${{ matrix.node-version }} and Dependencies + uses: ./.github/actions/npm-setup with: - node-version: ${{ inputs.node-version }} - - - name: Install dependencies - run: npm ci --omit=optional + runner: ${{ matrix.runner }} + node-version: ${{ matrix.node-version }} - name: Run tests run: npm run test:ci -- ${{ inputs.workspace }} From b4abbe5aae21b8d57ac84c5bc27dd691313f450b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 17:27:04 +0100 Subject: [PATCH 09/41] fix: use inputs --- .github/workflows/test-template.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 3dfacfae7..a249a2213 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -130,11 +130,11 @@ jobs: - name: Code checkout uses: actions/checkout@v4 - - name: Install Node ${{ matrix.node-version }} and Dependencies + - name: Install Node ${{ inputs.node-version }} and Dependencies uses: ./.github/actions/npm-setup with: - runner: ${{ matrix.runner }} - node-version: ${{ matrix.node-version }} + runner: ${{ inputs.runner }} + node-version: ${{ inputs.node-version }} - name: Run tests run: npm run test:ci -- ${{ inputs.workspace }} From 1cdcd9738fdb15ae48da66c53dfdefc391638b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 5 Mar 2025 23:23:10 +0100 Subject: [PATCH 10/41] chore: add workspace to cache key --- .github/actions/npm-setup/action.yml | 7 +++++-- .github/workflows/lint.yml | 1 + .github/workflows/test-template.yml | 1 + .github/workflows/test.yml | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 9ab453f85..e3a087652 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -8,6 +8,9 @@ inputs: node-version: description: 'Node.js version to use' required: true + workspace: + description: 'Key for the cache' + required: true runs: using: "composite" @@ -20,9 +23,9 @@ runs: id: npm-cache with: path: node_modules - key: ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ hashFiles('**/package-lock.json') }} + key: ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles('**/package-lock.json') }} restore-keys: | - ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}- + ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}- - name: Install dependencies if: steps.npm-cache.outputs.cache-hit != 'true' shell: bash diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d4bddd996..d53bf4d00 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,5 +19,6 @@ jobs: with: runner: ubuntu-22.04 node-version: 22.x + workspace: "packages/testcontainers" - name: Code linting run: npm run lint:ci diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index a249a2213..a48afb24b 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -135,6 +135,7 @@ jobs: with: runner: ${{ inputs.runner }} node-version: ${{ inputs.node-version }} + workspace: "${{ inputs.workspace }}" - name: Run tests run: npm run test:ci -- ${{ inputs.workspace }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f3d47fc5..dada6181a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,7 @@ jobs: with: runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} + workspace: "packages/testcontainers" - name: Build workspaces run: npm run build -ws From 93d5784227b23bc85913611f279d9b3f1536ff0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 14:32:05 +0100 Subject: [PATCH 11/41] fix: cache node_modules from the packages --- .github/actions/npm-setup/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index e3a087652..cfcf13ef8 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -35,5 +35,7 @@ runs: if: steps.npm-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: node_modules + path: | + node_modules + ${{ inputs.workspace }}/node_modules key: ${{ steps.npm-cache.outputs.cache-primary-key }} From 602ea5495128ba37ca824e2ccd6aa0104aeb25cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 17:58:55 +0100 Subject: [PATCH 12/41] feat: add shell script that detect changes --- .github/scripts/changed-modules.sh | 152 +++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100755 .github/scripts/changed-modules.sh diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh new file mode 100755 index 000000000..0ad76020d --- /dev/null +++ b/.github/scripts/changed-modules.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash + +# exit on error, unset variables, print commands, fail on pipe errors +set -euxo pipefail + +# How to test this script, run it with the required environment variables: +# 1. A modified file from the root, but not the package.json or package-lock.json: +# ALL_CHANGED_FILES="README.md" ./.github/scripts/changed-modules.sh +# Expected output: [], as no module should be built +# +# 2. The package.json or package-lock.json are modified: +# ALL_CHANGED_FILES="package.json" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the dependencies have been modified +# ALL_CHANGED_FILES="package-lock.json" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the dependencies have been modified +# +# 3. A file in the testcontainers module is modified: +# ALL_CHANGED_FILES="packages/testcontainers/a.txt" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the core has been modified +# +# 4. A file in a module is modified: +# ALL_CHANGED_FILES="packages/modules/arangodb/a.txt" ./.github/scripts/changed-modules.sh +# Expected output: [arangodb], only +# +# 5. Three files in three different modules are modified: +# ALL_CHANGED_FILES="packages/modules/arangodb/a.txt packages/modules/cassandra/b.txt packages/modules/chromadb/c.txt" ./.github/scripts/changed-modules.sh +# Expected output: [arangodb, cassandra, chromadb] +# +# 6. Core files and module files are modified: +# ALL_CHANGED_FILES="packages/testcontainers/a.txt packages/modules/chromadb/b.txt" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the core has been modified +# +# 7. This script is modified: +# ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the build script has been modified +# +# 8. This script is modified: +# ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the build script has been modified +# +# 9. This script is modified: +# ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the build script has been modified +# +# There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. +# But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. + +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) + +# define an array of directories that won't be included in the list +readonly excluded_modules=(".devcontainer" ".vscode" ".husky" "docs" ".github/ISSUE_TEMPLATE") + +# define an array of files that won't be included in the list +# Get all files in the root directory except package.json and package-lock.json +# Create array of excluded files by finding all files in root dir except package.json and package-lock.json +excluded_files=("${ROOT_DIR}/.github/release-drafter.yml") +while IFS= read -r file; do + excluded_files+=("\"${file}\"") +done < <(find "${ROOT_DIR}" -maxdepth 1 -type f -not -name "package.json" -not -name "package-lock.json") + +# define an array of modules that won't be part of the build +readonly no_build_modules=("packages/modules/k6") + +# modules is an array that will store the paths of all the modules in the repository. +modules=() + +# Find all package.json files in the repository, building a list of all the available modules. +for modFile in $(find "${ROOT_DIR}" -name "package.json" -not -path "*/node_modules/*"); do + modules+=("\"$(basename "$(dirname "${modFile}")")\"") +done +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) +unset IFS + +# capture the root module +readonly rootModule="\"packages/testcontainers\"" + +# merge all modules into a single array +allModules=(${rootModule} "${modules[@]}") + +# sort allModules array +IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) +unset IFS + +# Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES. +# On CI, this value will come from a Github Action retrieving the list of modified files from the pull request. +readonly modified_files=${ALL_CHANGED_FILES[@]} + +# Initialize variables +modified_modules=() + +# Check the modified files and determine which modules to build, following these rules: +# - if the modified files contain any file in the root module, include all modules in the list +# - if the modified files only contain files in one of the modules, include that module in the list +for file in $modified_files; do + # check if the file is in one of the excluded files + for exclude_file in ${excluded_files[@]}; do + # Remove quotes from exclude_file for comparison + clean_exclude_file=$(echo $exclude_file | tr -d '"') + if [[ "${ROOT_DIR}/${file}" == "${clean_exclude_file}" ]]; then + # if the file is in the excluded files, skip the rest of the loop. + # Execution continues at the loop control of the 2nd enclosing loop. + continue 2 + fi + done + + if [[ $file == packages/modules/* ]]; then + module_name=$(echo $file | cut -d'/' -f3) + if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then + modified_modules+=("\"packages/modules/$module_name\"") + fi + else + # a file from the core module (packages/testcontainers) is modified, so include all modules in the list and stop the loop + # check if the file is in one of the excluded modules + for exclude_module in ${excluded_modules[@]}; do + if [[ $file == $exclude_module/* ]]; then + # continue skips to the next iteration of an enclosing for, select, until, or while loop in a shell script. + # Execution continues at the loop control of the nth enclosing loop, in this case two levels up. + continue 2 + fi + done + + modified_modules=${allModules[@]} + break + fi +done + +# print all modules with this format: +# each module will be enclosed in double quotes +# each module will be separated by a comma +# the entire list will be enclosed in square brackets +# the list will be sorted and unique +sorted_unique_modules=($(echo "${modified_modules[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + +# remove modules that won't be part of the build from the list +filtered_modules=() +for module in "${sorted_unique_modules[@]}"; do + skip=false + for no_build_module in "${no_build_modules[@]}"; do + if [[ ${module} == \"${no_build_module}\" ]]; then + skip=true + break + fi + done + if [[ $skip == false ]]; then + filtered_modules+=(${module}) + fi +done +sorted_unique_modules=("${filtered_modules[@]}") + +echo "["$(IFS=,; echo "${sorted_unique_modules[*]}" | sed 's/ /,/g')"]" From aebadac941137dacc27ea82369fe02be255d1ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 18:06:07 +0100 Subject: [PATCH 13/41] chore: use script in the pipeline --- .github/workflows/lint.yml | 24 ---------- .github/workflows/test.yml | 97 ++++++++++++++++++++++++-------------- 2 files changed, 62 insertions(+), 59 deletions(-) delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index d53bf4d00..000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Lint - -on: - pull_request: - branches: [ main ] - -jobs: - lint: - runs-on: ubuntu-22.04 - steps: - - name: Code checkout - uses: actions/checkout@v4 - - name: Install NodeJS - uses: actions/setup-node@v4 - with: - node-version: 22.x - - name: Install Node and Dependencies - uses: ./.github/actions/npm-setup - with: - runner: ubuntu-22.04 - node-version: 22.x - workspace: "packages/testcontainers" - - name: Code linting - run: npm run lint:ci diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dada6181a..778dee0a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,13 +19,64 @@ concurrency: cancel-in-progress: true jobs: + detect-modules: + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.set-modified-modules.outputs.modules }} + modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }} + steps: + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 + + - id: set-modified-modules + name: Set all modified modules + env: + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + run: echo "modules=$(./.github/scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + + - id: set-modified-modules-count + name: Set all modified modules count + run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT + + - name: Print out the modules to be used + run: | + echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" + echo "${{ steps.set-modified-modules.outputs.modules }}" + + lint: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + strategy: + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + runs-on: ubuntu-22.04 + steps: + - name: Code checkout + uses: actions/checkout@v4 + - name: Install Node and Dependencies + uses: ./.github/actions/npm-setup + with: + runner: ubuntu-22.04 + node-version: 22.x + workspace: "packages/testcontainers" + - name: Code linting + run: npm run lint:ci -- ${{ matrix.module }} + smoke-test: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} name: Smoke test - runs-on: ${{ matrix.runner }} strategy: matrix: runner: [ ubuntu-22.04 ] node-version: [ 18.x, 20.x, 22.x ] + runs-on: ${{ matrix.runner }} steps: - name: Code checkout uses: actions/checkout@v4 @@ -47,40 +98,17 @@ jobs: - name: Run ES module smoke test run: node packages/testcontainers/smoke-test.mjs - test-testcontainers: - name: Testcontainers - needs: smoke-test - strategy: - matrix: - node-version: [ 18.x, 20.x, 22.x ] - container-runtime: [ docker, podman ] - include: - - container-runtime: docker - runner: ubuntu-22.04 - - container-runtime: podman - runner: ubuntu-22.04 - uses: ./.github/workflows/test-template.yml - with: - runner: ${{ matrix.runner }} - node-version: ${{ matrix.node-version }} - container-runtime: ${{ matrix.container-runtime }} - workspace: packages/testcontainers - - list-modules: - name: List modules - runs-on: ubuntu-22.04 - needs: test-testcontainers - outputs: - modules: ${{ steps.list_modules.outputs.modules }} - steps: - - uses: actions/checkout@v2 - - id: list_modules - run: echo "modules=$(ls packages/modules | jq -cnR '[inputs | select(length>0) | select(. != "couchbase")]')" >> $GITHUB_OUTPUT - test-modules: - name: Module (${{ matrix.module }}) - needs: [ test-testcontainers, list-modules ] + test: + name: Run tests + # only run if there are modules to test + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint + - smoke-test strategy: matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} node-version: [ 18.x, 20.x, 22.x ] container-runtime: [ docker, podman ] include: @@ -88,10 +116,9 @@ jobs: runner: ubuntu-22.04 - container-runtime: podman runner: ubuntu-22.04 - module: ${{ fromJson(needs.list-modules.outputs.modules) }} uses: ./.github/workflows/test-template.yml with: runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} container-runtime: ${{ matrix.container-runtime }} - workspace: packages/modules/${{ matrix.module }} + workspace: "${{ matrix.module }}" From 7afe84896bb1768c2dc8553ba8968f8934a9823c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 18:47:57 +0100 Subject: [PATCH 14/41] chore: run lint in the right dir --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 778dee0a3..84edff0c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,13 +64,17 @@ jobs: with: runner: ubuntu-22.04 node-version: 22.x - workspace: "packages/testcontainers" + workspace: "${{ matrix.module }}" - name: Code linting + working-directory: ${{ matrix.module }} run: npm run lint:ci -- ${{ matrix.module }} smoke-test: # only run if there are modules to lint if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint name: Smoke test strategy: matrix: From ad5296b98850390c51c3b429da0e4aef8f7c82d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 20:14:12 +0100 Subject: [PATCH 15/41] chore: add a closing node that will be required by GH --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84edff0c3..c718db866 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -126,3 +126,19 @@ jobs: node-version: ${{ matrix.node-version }} container-runtime: ${{ matrix.container-runtime }} workspace: "${{ matrix.module }}" + + # This job serves as confirmation that all test jobs finished + end: + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint + - smoke-test + - test + runs-on: ubuntu-latest + steps: + - name: Check if any jobs failed + if: ${{ failure() || cancelled() }} + run: exit 1 + + - run: echo "All tests completed successfully!" From 0b2f080b62ccacdb625aad924aa1c013445087ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 20:53:27 +0100 Subject: [PATCH 16/41] fix: pass the workspace dirs correctly --- .github/scripts/changed-modules.sh | 35 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh index 0ad76020d..ff2a1c2a1 100755 --- a/.github/scripts/changed-modules.sh +++ b/.github/scripts/changed-modules.sh @@ -34,13 +34,9 @@ set -euxo pipefail # ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh # Expected output: all modules, as the build script has been modified # -# 8. This script is modified: -# ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh -# Expected output: all modules, as the build script has been modified -# -# 9. This script is modified: -# ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh -# Expected output: all modules, as the build script has been modified +# 8. A .github file is modified: +# ALL_CHANGED_FILES=".github/release-drafter.yml" ./.github/scripts/changed-modules.sh +# Expected output: [] # # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. # But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. @@ -66,21 +62,24 @@ readonly no_build_modules=("packages/modules/k6") modules=() # Find all package.json files in the repository, building a list of all the available modules. -for modFile in $(find "${ROOT_DIR}" -name "package.json" -not -path "*/node_modules/*"); do - modules+=("\"$(basename "$(dirname "${modFile}")")\"") +# The list of modules is stored in the modules array, but the testcontainers-node module is excluded +# as it is not a module. +for packageJSONFile in $(find "${ROOT_DIR}" -name "package.json" -not -path "*/node_modules/*"); do + name=$(basename "$(dirname "${packageJSONFile}")") + if [[ "${name}" != "testcontainers-node" ]]; then + if [[ "${name}" != "testcontainers" ]]; then + modules+=("\"packages/modules/${name}\"") + else + modules+=("\"packages/testcontainers\"") + fi + fi done # sort modules array IFS=$'\n' modules=($(sort <<<"${modules[*]}")) unset IFS -# capture the root module -readonly rootModule="\"packages/testcontainers\"" - -# merge all modules into a single array -allModules=(${rootModule} "${modules[@]}") - -# sort allModules array -IFS=$'\n' allModules=($(sort <<<"${allModules[*]}")) +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) unset IFS # Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES. @@ -121,7 +120,7 @@ for file in $modified_files; do fi done - modified_modules=${allModules[@]} + modified_modules=${modules[@]} break fi done From 89fc46f0c4c0a2bace0f7d64329ecfa79d33ec1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Thu, 6 Mar 2025 21:01:02 +0100 Subject: [PATCH 17/41] fix: honor npm workspaces --- .github/scripts/changed-modules.sh | 10 +++------- .github/workflows/test.yml | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh index ff2a1c2a1..bff6e4bbe 100755 --- a/.github/scripts/changed-modules.sh +++ b/.github/scripts/changed-modules.sh @@ -56,7 +56,7 @@ while IFS= read -r file; do done < <(find "${ROOT_DIR}" -maxdepth 1 -type f -not -name "package.json" -not -name "package-lock.json") # define an array of modules that won't be part of the build -readonly no_build_modules=("packages/modules/k6") +readonly no_build_modules=() # modules is an array that will store the paths of all the modules in the repository. modules=() @@ -67,11 +67,7 @@ modules=() for packageJSONFile in $(find "${ROOT_DIR}" -name "package.json" -not -path "*/node_modules/*"); do name=$(basename "$(dirname "${packageJSONFile}")") if [[ "${name}" != "testcontainers-node" ]]; then - if [[ "${name}" != "testcontainers" ]]; then - modules+=("\"packages/modules/${name}\"") - else - modules+=("\"packages/testcontainers\"") - fi + modules+=("\"${name}\"") fi done # sort modules array @@ -107,7 +103,7 @@ for file in $modified_files; do if [[ $file == packages/modules/* ]]; then module_name=$(echo $file | cut -d'/' -f3) if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then - modified_modules+=("\"packages/modules/$module_name\"") + modified_modules+=("\"$module_name\"") fi else # a file from the core module (packages/testcontainers) is modified, so include all modules in the list and stop the loop diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c718db866..f08e369db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,6 @@ jobs: node-version: 22.x workspace: "${{ matrix.module }}" - name: Code linting - working-directory: ${{ matrix.module }} run: npm run lint:ci -- ${{ matrix.module }} smoke-test: From 0373fffee9a31f7161d400f950756335f2e5e02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 07:57:00 +0100 Subject: [PATCH 18/41] chore: remove branch from the cache key --- .github/actions/npm-setup/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index cfcf13ef8..d37cb83bc 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -23,13 +23,14 @@ runs: id: npm-cache with: path: node_modules - key: ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles('**/package-lock.json') }} + # the key is built using both package-lock.json files: the one in the root dir and the one for the module to be built + key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles(format('{0}-{1}/{2}', 'package-lock.json', inputs.workspace, 'package-lock.json')) }} restore-keys: | - ${{ github.head_ref || 'main' }}-${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}- + ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}- - name: Install dependencies if: steps.npm-cache.outputs.cache-hit != 'true' shell: bash - run: npm ci --omit=optional + run: npm ci --omit=optional -- ${{ inputs.workspace }} - name: Cache npm id: cache-npm-save if: steps.npm-cache.outputs.cache-hit != 'true' From dabcce658e924b30cf528df1784e6151de0d2625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 08:02:15 +0100 Subject: [PATCH 19/41] chore: exclude couchbase from the build --- .github/scripts/changed-modules.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh index bff6e4bbe..109b68d23 100755 --- a/.github/scripts/changed-modules.sh +++ b/.github/scripts/changed-modules.sh @@ -38,6 +38,10 @@ set -euxo pipefail # ALL_CHANGED_FILES=".github/release-drafter.yml" ./.github/scripts/changed-modules.sh # Expected output: [] # +# 9. A excluded module is modified: +# ALL_CHANGED_FILES="packages/modules/couchbase/a.txt" ./.github/scripts/changed-modules.sh +# Expected output: [] +# # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. # But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. @@ -56,7 +60,7 @@ while IFS= read -r file; do done < <(find "${ROOT_DIR}" -maxdepth 1 -type f -not -name "package.json" -not -name "package-lock.json") # define an array of modules that won't be part of the build -readonly no_build_modules=() +readonly no_build_modules=("couchbase") # modules is an array that will store the paths of all the modules in the repository. modules=() From 23f144552bac4f6394dc50eb1511fa5d71b04702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 09:48:33 +0100 Subject: [PATCH 20/41] fix: configure cache paths correctly --- .github/actions/npm-setup/action.yml | 34 +++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index d37cb83bc..600625e98 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -19,24 +19,42 @@ runs: uses: actions/setup-node@v4 with: node-version: ${{ inputs.node-version }} + + - name: Set cache configuration + shell: bash + run: | + if [ "${{ inputs.workspace }}" = "testcontainers" ]; then + echo CACHE_PATHS<> $GITHUB_ENV + echo node_modules >> $GITHUB_ENV + echo packages/testcontainers/node_modules >> $GITHUB_ENV + echo EOF >> $GITHUB_ENV + echo HASH_PATHS="'package-lock.json','packages/testcontainers/package-lock.json'" >> $GITHUB_ENV + else + echo CACHE_PATHS<> $GITHUB_ENV + echo node_modules >> $GITHUB_ENV + echo packages/testcontainers/node_modules >> $GITHUB_ENV + echo packages/modules/${{ inputs.workspace }}/node_modules >> $GITHUB_ENV + echo EOF >> $GITHUB_ENV + echo HASH_PATHS="'package-lock.json','packages/testcontainers/package-lock.json','packages/modules/${{ inputs.workspace }}/package-lock.json'" >> $GITHUB_ENV + fi + - uses: actions/cache/restore@v4 id: npm-cache with: - path: node_modules - # the key is built using both package-lock.json files: the one in the root dir and the one for the module to be built - key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles(format('{0}-{1}/{2}', 'package-lock.json', inputs.workspace, 'package-lock.json')) }} + path: ${{ env.CACHE_PATHS }} + key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles(env.HASH_PATHS) }} restore-keys: | ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}- + ${{ inputs.runner }}-node-${{ inputs.node-version }}- + - name: Install dependencies if: steps.npm-cache.outputs.cache-hit != 'true' shell: bash - run: npm ci --omit=optional -- ${{ inputs.workspace }} + run: npm ci --workspace ${{ inputs.workspace }} --include-workspace-root + - name: Cache npm - id: cache-npm-save if: steps.npm-cache.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: - path: | - node_modules - ${{ inputs.workspace }}/node_modules + path: ${{ env.CACHE_PATHS }} key: ${{ steps.npm-cache.outputs.cache-primary-key }} From a5169c61c478cdedba0077dc548492c4ff2ea195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 09:51:42 +0100 Subject: [PATCH 21/41] chore: fail fast the lint and smoke test stages --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f08e369db..f29a9a95a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,7 @@ jobs: needs: - detect-modules strategy: + fail-fast: true matrix: module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} runs-on: ubuntu-22.04 @@ -76,6 +77,7 @@ jobs: - lint name: Smoke test strategy: + fail-fast: true matrix: runner: [ ubuntu-22.04 ] node-version: [ 18.x, 20.x, 22.x ] From 48469a79e731ff3389f45f193425c53df4e883f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 09:57:16 +0100 Subject: [PATCH 22/41] fix: syntax --- .github/actions/npm-setup/action.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 600625e98..e4ecad0d8 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -24,18 +24,18 @@ runs: shell: bash run: | if [ "${{ inputs.workspace }}" = "testcontainers" ]; then - echo CACHE_PATHS<> $GITHUB_ENV - echo node_modules >> $GITHUB_ENV - echo packages/testcontainers/node_modules >> $GITHUB_ENV - echo EOF >> $GITHUB_ENV - echo HASH_PATHS="'package-lock.json','packages/testcontainers/package-lock.json'" >> $GITHUB_ENV + echo "CACHE_PATHS<> $GITHUB_ENV + echo "node_modules" >> $GITHUB_ENV + echo "packages/testcontainers/node_modules" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "HASH_PATHS='package-lock.json','packages/testcontainers/package-lock.json'" >> $GITHUB_ENV else - echo CACHE_PATHS<> $GITHUB_ENV - echo node_modules >> $GITHUB_ENV - echo packages/testcontainers/node_modules >> $GITHUB_ENV - echo packages/modules/${{ inputs.workspace }}/node_modules >> $GITHUB_ENV - echo EOF >> $GITHUB_ENV - echo HASH_PATHS="'package-lock.json','packages/testcontainers/package-lock.json','packages/modules/${{ inputs.workspace }}/package-lock.json'" >> $GITHUB_ENV + echo "CACHE_PATHS<> $GITHUB_ENV + echo "node_modules" >> $GITHUB_ENV + echo "packages/testcontainers/node_modules" >> $GITHUB_ENV + echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "HASH_PATHS='package-lock.json','packages/testcontainers/package-lock.json','packages/modules/${{ inputs.workspace }}/package-lock.json'" >> $GITHUB_ENV fi - uses: actions/cache/restore@v4 From bde0cf487e9f36fa7d8b3112e32175e5f1f1a5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:01:04 +0100 Subject: [PATCH 23/41] fix: cache key --- .github/actions/npm-setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index e4ecad0d8..a7737529c 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -28,14 +28,14 @@ runs: echo "node_modules" >> $GITHUB_ENV echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS='package-lock.json','packages/testcontainers/package-lock.json'" >> $GITHUB_ENV + echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV else echo "CACHE_PATHS<> $GITHUB_ENV echo "node_modules" >> $GITHUB_ENV echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS='package-lock.json','packages/testcontainers/package-lock.json','packages/modules/${{ inputs.workspace }}/package-lock.json'" >> $GITHUB_ENV + echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV fi - uses: actions/cache/restore@v4 From 23f82efea9503ea4a3f6cc36f35e60d48eeb3881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:02:49 +0100 Subject: [PATCH 24/41] fix: use module path in npm install --- .github/actions/npm-setup/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index a7737529c..86f2de335 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -29,6 +29,7 @@ runs: echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV + echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV else echo "CACHE_PATHS<> $GITHUB_ENV echo "node_modules" >> $GITHUB_ENV @@ -36,6 +37,7 @@ runs: echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV + echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV fi - uses: actions/cache/restore@v4 @@ -50,7 +52,7 @@ runs: - name: Install dependencies if: steps.npm-cache.outputs.cache-hit != 'true' shell: bash - run: npm ci --workspace ${{ inputs.workspace }} --include-workspace-root + run: npm ci --workspace ${{ env.WORKSPACE_PATH }} --include-workspace-root - name: Cache npm if: steps.npm-cache.outputs.cache-hit != 'true' From e9692d44d3bb1635e984352d963ce8a60ff888c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:07:27 +0100 Subject: [PATCH 25/41] chore: get the workspace output from the composite action --- .github/actions/npm-setup/action.yml | 7 +++++++ .github/workflows/test-template.yml | 3 ++- .github/workflows/test.yml | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 86f2de335..70ecd1af7 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -12,6 +12,11 @@ inputs: description: 'Key for the cache' required: true +outputs: + workspace_path: + description: "Full path to the workspace directory" + value: ${{ steps.set-env.outputs.workspace_path }} + runs: using: "composite" steps: @@ -30,6 +35,7 @@ runs: echo "EOF" >> $GITHUB_ENV echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV + echo "workspace_path=packages/testcontainers" >> $GITHUB_OUTPUT else echo "CACHE_PATHS<> $GITHUB_ENV echo "node_modules" >> $GITHUB_ENV @@ -38,6 +44,7 @@ runs: echo "EOF" >> $GITHUB_ENV echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV + echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> $GITHUB_OUTPUT fi - uses: actions/cache/restore@v4 diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index a48afb24b..04d14ee1c 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -131,6 +131,7 @@ jobs: uses: actions/checkout@v4 - name: Install Node ${{ inputs.node-version }} and Dependencies + id: npm-install uses: ./.github/actions/npm-setup with: runner: ${{ inputs.runner }} @@ -138,4 +139,4 @@ jobs: workspace: "${{ inputs.workspace }}" - name: Run tests - run: npm run test:ci -- ${{ inputs.workspace }} + run: npm run test:ci -- ${{ steps.npm-install.outputs.workspace_path }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f29a9a95a..c8d218f8b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,13 +61,14 @@ jobs: - name: Code checkout uses: actions/checkout@v4 - name: Install Node and Dependencies + id: npm-install uses: ./.github/actions/npm-setup with: runner: ubuntu-22.04 node-version: 22.x workspace: "${{ matrix.module }}" - name: Code linting - run: npm run lint:ci -- ${{ matrix.module }} + run: npm run lint:ci -- ${{ steps.npm-install.outputs.workspace_path }} smoke-test: # only run if there are modules to lint From edcb394fa8256e84fd3212c3c90c0c097d9e2cce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:12:51 +0100 Subject: [PATCH 26/41] chore: escape quotes --- .github/actions/npm-setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 70ecd1af7..28caa1534 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -33,7 +33,7 @@ runs: echo "node_modules" >> $GITHUB_ENV echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV + echo "HASH_PATHS=\"'package-lock.json', 'packages/testcontainers/package-lock.json'\"" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV echo "workspace_path=packages/testcontainers" >> $GITHUB_OUTPUT else @@ -42,7 +42,7 @@ runs: echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV + echo "HASH_PATHS=\"'package-lock.json', 'packages/testcontainers/package-lock.json', 'packages/modules/${{ inputs.workspace }}/package-lock.json'\"" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> $GITHUB_OUTPUT fi From ba8826072a05996e70c14476f0aba1a7a7a3b96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:15:56 +0100 Subject: [PATCH 27/41] fix: pass the module correctly in smoke tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c8d218f8b..9333c5969 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -93,7 +93,7 @@ jobs: with: runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} - workspace: "packages/testcontainers" + workspace: "testcontainers" - name: Build workspaces run: npm run build -ws From 4b3d1e00143aeea4e4e3873bd47c59b4054bdb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:18:52 +0100 Subject: [PATCH 28/41] fix: remove escaped quotes --- .github/actions/npm-setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 28caa1534..03906176f 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -33,7 +33,7 @@ runs: echo "node_modules" >> $GITHUB_ENV echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS=\"'package-lock.json', 'packages/testcontainers/package-lock.json'\"" >> $GITHUB_ENV + echo "HASH_PATHS='package-lock.json', 'packages/testcontainers/package-lock.json'" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV echo "workspace_path=packages/testcontainers" >> $GITHUB_OUTPUT else @@ -42,7 +42,7 @@ runs: echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS=\"'package-lock.json', 'packages/testcontainers/package-lock.json', 'packages/modules/${{ inputs.workspace }}/package-lock.json'\"" >> $GITHUB_ENV + echo "HASH_PATHS='package-lock.json', 'packages/testcontainers/package-lock.json', 'packages/modules/${{ inputs.workspace }}/package-lock.json'" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> $GITHUB_OUTPUT fi From 5a2d9676bda57a092a6400387ee77f968d91d7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:27:13 +0100 Subject: [PATCH 29/41] fix: simplify --- .github/actions/npm-setup/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 03906176f..70ecd1af7 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -33,7 +33,7 @@ runs: echo "node_modules" >> $GITHUB_ENV echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS='package-lock.json', 'packages/testcontainers/package-lock.json'" >> $GITHUB_ENV + echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV echo "workspace_path=packages/testcontainers" >> $GITHUB_OUTPUT else @@ -42,7 +42,7 @@ runs: echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS='package-lock.json', 'packages/testcontainers/package-lock.json', 'packages/modules/${{ inputs.workspace }}/package-lock.json'" >> $GITHUB_ENV + echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> $GITHUB_OUTPUT fi From 2b827bcb6526d289537590f12b181fe69504e0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:37:51 +0100 Subject: [PATCH 30/41] fix: set output correctly --- .github/actions/npm-setup/action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index 70ecd1af7..a9dda939d 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -27,6 +27,7 @@ runs: - name: Set cache configuration shell: bash + id: set-env run: | if [ "${{ inputs.workspace }}" = "testcontainers" ]; then echo "CACHE_PATHS<> $GITHUB_ENV @@ -35,7 +36,7 @@ runs: echo "EOF" >> $GITHUB_ENV echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV - echo "workspace_path=packages/testcontainers" >> $GITHUB_OUTPUT + echo "workspace_path=packages/testcontainers" >> "$GITHUB_OUTPUT" else echo "CACHE_PATHS<> $GITHUB_ENV echo "node_modules" >> $GITHUB_ENV @@ -44,7 +45,7 @@ runs: echo "EOF" >> $GITHUB_ENV echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV - echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> $GITHUB_OUTPUT + echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> "$GITHUB_OUTPUT" fi - uses: actions/cache/restore@v4 From d6c3532ec8a2907b8b37f792e814e03336612d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:39:38 +0100 Subject: [PATCH 31/41] fix: build the cache key directly --- .github/actions/npm-setup/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml index a9dda939d..e8325e5a6 100644 --- a/.github/actions/npm-setup/action.yml +++ b/.github/actions/npm-setup/action.yml @@ -34,7 +34,6 @@ runs: echo "node_modules" >> $GITHUB_ENV echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV echo "workspace_path=packages/testcontainers" >> "$GITHUB_OUTPUT" else @@ -43,7 +42,6 @@ runs: echo "packages/testcontainers/node_modules" >> $GITHUB_ENV echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - echo "HASH_PATHS=package-lock.json,packages/testcontainers/package-lock.json,packages/modules/${{ inputs.workspace }}/package-lock.json" >> $GITHUB_ENV echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> "$GITHUB_OUTPUT" fi @@ -52,7 +50,7 @@ runs: id: npm-cache with: path: ${{ env.CACHE_PATHS }} - key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles(env.HASH_PATHS) }} + key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles('package-lock.json', 'packages/testcontainers/package-lock.json', format('packages/modules/{0}/package-lock.json', inputs.workspace)) }} restore-keys: | ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}- ${{ inputs.runner }}-node-${{ inputs.node-version }}- From aa95ebfdb322e579388a3df62820c8b8d8c01ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:47:40 +0100 Subject: [PATCH 32/41] fix: use consistent format --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9333c5969..23a08c040 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: node-version: 22.x workspace: "${{ matrix.module }}" - name: Code linting - run: npm run lint:ci -- ${{ steps.npm-install.outputs.workspace_path }} + run: npm run lint:ci --workspace ${{ steps.npm-install.outputs.workspace_path }} --include-workspace-root smoke-test: # only run if there are modules to lint From 898fde72b0129a10d9f3c6669380ffbfefd439af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 10:58:47 +0100 Subject: [PATCH 33/41] fix: CI must run the ilnt in the current workspace --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 732e02c60..96ef969b7 100755 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "test:ci": "npm run test -- --runInBand --coverage", "format": "prettier --write package.json \"packages/**/*.ts\"", "lint": "eslint --fix package.json \"packages/**/*.ts\"", - "lint:ci": "eslint package.json \"packages/**/*.ts\" --max-warnings=0", + "lint:ci": "eslint package.json \"./**/*.ts\" --max-warnings=0", "update-deps": "npm-check-updates --workspaces --root -u", "validate-engine": "ls-engines" }, From fd135cd731ec63d203b31d6ad6a6c9a9967577b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 11:06:36 +0100 Subject: [PATCH 34/41] fix: not needed --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23a08c040..89275c241 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: node-version: 22.x workspace: "${{ matrix.module }}" - name: Code linting - run: npm run lint:ci --workspace ${{ steps.npm-install.outputs.workspace_path }} --include-workspace-root + run: npm run lint:ci --workspace ${{ steps.npm-install.outputs.workspace_path }} smoke-test: # only run if there are modules to lint From 6e7f641c1400ceee9fddde9114b19d0e167705cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 11:12:06 +0100 Subject: [PATCH 35/41] chore: define proper path for lint:ci --- .github/workflows/test.yml | 4 +++- package.json | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 89275c241..88796b3a8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,9 @@ jobs: node-version: 22.x workspace: "${{ matrix.module }}" - name: Code linting - run: npm run lint:ci --workspace ${{ steps.npm-install.outputs.workspace_path }} + env: + WORKSPACE_PATH: ${{ steps.npm-install.outputs.workspace_path }} + run: npm run lint:ci smoke-test: # only run if there are modules to lint diff --git a/package.json b/package.json index 96ef969b7..c30082b7f 100755 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "test:ci": "npm run test -- --runInBand --coverage", "format": "prettier --write package.json \"packages/**/*.ts\"", "lint": "eslint --fix package.json \"packages/**/*.ts\"", - "lint:ci": "eslint package.json \"./**/*.ts\" --max-warnings=0", + "lint:ci": "eslint package.json \"${WORKSPACE_PATH}/**/*.ts\" --max-warnings=0", "update-deps": "npm-check-updates --workspaces --root -u", "validate-engine": "ls-engines" }, From e68118f8d2dd33f96c41e3045664b28f881e56d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 11:20:48 +0100 Subject: [PATCH 36/41] chore: build just the testcontainers workspace for the smoke tests --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88796b3a8..812a753d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,10 +97,10 @@ jobs: node-version: ${{ matrix.node-version }} workspace: "testcontainers" - - name: Build workspaces - run: npm run build -ws + - name: Build testcontainers + run: npm run build --workspace packages/testcontainers - name: Remove dev dependencies - run: npm prune --omit=dev + run: npm prune --omit=dev --workspace packages/testcontainers - name: Run CommonJS module smoke test run: node packages/testcontainers/smoke-test.js - name: Run ES module smoke test From 9e5dec0477e76cf54b26011d91b3409300710bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 11:25:15 +0100 Subject: [PATCH 37/41] chre: we do not want to fail fast the tests in order to understand which module is failing --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 812a753d7..005fd6f3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,6 +115,7 @@ jobs: - lint - smoke-test strategy: + fail-fast: false matrix: module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} node-version: [ 18.x, 20.x, 22.x ] From c2fb3080beae5603ff86b5c92d18b5734a8f0d98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 11:27:46 +0100 Subject: [PATCH 38/41] chore: run podman after docker This will reduce the number of concurrent workers, still running podman --- .github/workflows/test.yml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 005fd6f3a..f53674f7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,17 +119,32 @@ jobs: matrix: module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} node-version: [ 18.x, 20.x, 22.x ] - container-runtime: [ docker, podman ] - include: - - container-runtime: docker - runner: ubuntu-22.04 - - container-runtime: podman - runner: ubuntu-22.04 uses: ./.github/workflows/test-template.yml with: - runner: ${{ matrix.runner }} + runner: ubuntu-22.04 node-version: ${{ matrix.node-version }} - container-runtime: ${{ matrix.container-runtime }} + container-runtime: docker + workspace: "${{ matrix.module }}" + + test-podman: + name: Run tests + # only run if there are modules to test + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint + - smoke-test + - test + strategy: + fail-fast: false + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + node-version: [ 18.x, 20.x, 22.x ] + uses: ./.github/workflows/test-template.yml + with: + runner: ubuntu-22.04 + node-version: ${{ matrix.node-version }} + container-runtime: podman workspace: "${{ matrix.module }}" # This job serves as confirmation that all test jobs finished @@ -140,6 +155,7 @@ jobs: - lint - smoke-test - test + - test-podman runs-on: ubuntu-latest steps: - name: Check if any jobs failed From f1118d1dce237f5dd3d706ecee9a0347670185a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 12:05:56 +0100 Subject: [PATCH 39/41] fix: skip neo4j --- .github/scripts/changed-modules.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh index 109b68d23..af9180a34 100755 --- a/.github/scripts/changed-modules.sh +++ b/.github/scripts/changed-modules.sh @@ -39,7 +39,7 @@ set -euxo pipefail # Expected output: [] # # 9. A excluded module is modified: -# ALL_CHANGED_FILES="packages/modules/couchbase/a.txt" ./.github/scripts/changed-modules.sh +# ALL_CHANGED_FILES="packages/modules/couchbase/a.txt packages/modules/neo4j/b.txt" ./.github/scripts/changed-modules.sh # Expected output: [] # # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. @@ -60,7 +60,7 @@ while IFS= read -r file; do done < <(find "${ROOT_DIR}" -maxdepth 1 -type f -not -name "package.json" -not -name "package-lock.json") # define an array of modules that won't be part of the build -readonly no_build_modules=("couchbase") +readonly no_build_modules=("couchbase" "neo4j") # modules is an array that will store the paths of all the modules in the repository. modules=() From a6843d11c4672f21fdbd82adebe18324afbf44fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Fri, 7 Mar 2025 13:39:35 +0100 Subject: [PATCH 40/41] chore: add neo4j again --- .github/scripts/changed-modules.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh index af9180a34..109b68d23 100755 --- a/.github/scripts/changed-modules.sh +++ b/.github/scripts/changed-modules.sh @@ -39,7 +39,7 @@ set -euxo pipefail # Expected output: [] # # 9. A excluded module is modified: -# ALL_CHANGED_FILES="packages/modules/couchbase/a.txt packages/modules/neo4j/b.txt" ./.github/scripts/changed-modules.sh +# ALL_CHANGED_FILES="packages/modules/couchbase/a.txt" ./.github/scripts/changed-modules.sh # Expected output: [] # # There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. @@ -60,7 +60,7 @@ while IFS= read -r file; do done < <(find "${ROOT_DIR}" -maxdepth 1 -type f -not -name "package.json" -not -name "package-lock.json") # define an array of modules that won't be part of the build -readonly no_build_modules=("couchbase" "neo4j") +readonly no_build_modules=("couchbase") # modules is an array that will store the paths of all the modules in the repository. modules=() From 39b904dc66b051a47aefd520091f78f40c82de87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Mon, 10 Mar 2025 17:52:24 +0100 Subject: [PATCH 41/41] fix: remove duplication --- .github/scripts/changed-modules.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh index 109b68d23..dcda2c6b2 100755 --- a/.github/scripts/changed-modules.sh +++ b/.github/scripts/changed-modules.sh @@ -74,9 +74,6 @@ for packageJSONFile in $(find "${ROOT_DIR}" -name "package.json" -not -path "*/n modules+=("\"${name}\"") fi done -# sort modules array -IFS=$'\n' modules=($(sort <<<"${modules[*]}")) -unset IFS # sort modules array IFS=$'\n' modules=($(sort <<<"${modules[*]}"))