diff --git a/.github/actions/npm-setup/action.yml b/.github/actions/npm-setup/action.yml new file mode 100644 index 000000000..e8325e5a6 --- /dev/null +++ b/.github/actions/npm-setup/action.yml @@ -0,0 +1,68 @@ +name: 'NPM Setup' +description: 'Sets up Node.js and installs NPM dependencies with caching' + +inputs: + runner: + description: 'Runner to use' + required: true + node-version: + description: 'Node.js version to use' + required: true + workspace: + description: 'Key for the cache' + required: true + +outputs: + workspace_path: + description: "Full path to the workspace directory" + value: ${{ steps.set-env.outputs.workspace_path }} + +runs: + using: "composite" + steps: + - name: Install NodeJS ${{ inputs.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - name: Set cache configuration + shell: bash + id: set-env + run: | + if [ "${{ inputs.workspace }}" = "testcontainers" ]; then + echo "CACHE_PATHS<> $GITHUB_ENV + echo "node_modules" >> $GITHUB_ENV + echo "packages/testcontainers/node_modules" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "WORKSPACE_PATH=packages/testcontainers" >> $GITHUB_ENV + echo "workspace_path=packages/testcontainers" >> "$GITHUB_OUTPUT" + else + echo "CACHE_PATHS<> $GITHUB_ENV + echo "node_modules" >> $GITHUB_ENV + echo "packages/testcontainers/node_modules" >> $GITHUB_ENV + echo "packages/modules/${{ inputs.workspace }}/node_modules" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "WORKSPACE_PATH=packages/modules/${{ inputs.workspace }}" >> $GITHUB_ENV + echo "workspace_path=packages/modules/${{ inputs.workspace }}" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/cache/restore@v4 + id: npm-cache + with: + path: ${{ env.CACHE_PATHS }} + key: ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}-${{ hashFiles('package-lock.json', 'packages/testcontainers/package-lock.json', format('packages/modules/{0}/package-lock.json', inputs.workspace)) }} + restore-keys: | + ${{ inputs.runner }}-node-${{ inputs.node-version }}-${{ inputs.workspace }}- + ${{ inputs.runner }}-node-${{ inputs.node-version }}- + + - name: Install dependencies + if: steps.npm-cache.outputs.cache-hit != 'true' + shell: bash + run: npm ci --workspace ${{ env.WORKSPACE_PATH }} --include-workspace-root + + - name: Cache npm + if: steps.npm-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ env.CACHE_PATHS }} + key: ${{ steps.npm-cache.outputs.cache-primary-key }} diff --git a/.github/scripts/changed-modules.sh b/.github/scripts/changed-modules.sh new file mode 100755 index 000000000..dcda2c6b2 --- /dev/null +++ b/.github/scripts/changed-modules.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +# exit on error, unset variables, print commands, fail on pipe errors +set -euxo pipefail + +# How to test this script, run it with the required environment variables: +# 1. A modified file from the root, but not the package.json or package-lock.json: +# ALL_CHANGED_FILES="README.md" ./.github/scripts/changed-modules.sh +# Expected output: [], as no module should be built +# +# 2. The package.json or package-lock.json are modified: +# ALL_CHANGED_FILES="package.json" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the dependencies have been modified +# ALL_CHANGED_FILES="package-lock.json" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the dependencies have been modified +# +# 3. A file in the testcontainers module is modified: +# ALL_CHANGED_FILES="packages/testcontainers/a.txt" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the core has been modified +# +# 4. A file in a module is modified: +# ALL_CHANGED_FILES="packages/modules/arangodb/a.txt" ./.github/scripts/changed-modules.sh +# Expected output: [arangodb], only +# +# 5. Three files in three different modules are modified: +# ALL_CHANGED_FILES="packages/modules/arangodb/a.txt packages/modules/cassandra/b.txt packages/modules/chromadb/c.txt" ./.github/scripts/changed-modules.sh +# Expected output: [arangodb, cassandra, chromadb] +# +# 6. Core files and module files are modified: +# ALL_CHANGED_FILES="packages/testcontainers/a.txt packages/modules/chromadb/b.txt" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the core has been modified +# +# 7. This script is modified: +# ALL_CHANGED_FILES=".github/scripts/changed-modules.sh" ./.github/scripts/changed-modules.sh +# Expected output: all modules, as the build script has been modified +# +# 8. A .github file is modified: +# ALL_CHANGED_FILES=".github/release-drafter.yml" ./.github/scripts/changed-modules.sh +# Expected output: [] +# +# 9. A excluded module is modified: +# ALL_CHANGED_FILES="packages/modules/couchbase/a.txt" ./.github/scripts/changed-modules.sh +# Expected output: [] +# +# There is room for improvement in this script. For example, it could detect if the changes applied to the docs or the .github dirs, and then do not include any module in the list. +# But then we would need to verify the CI scripts to ensure that the job receives the correct modules to build. + +# ROOT_DIR is the root directory of the repository. +readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) + +# define an array of directories that won't be included in the list +readonly excluded_modules=(".devcontainer" ".vscode" ".husky" "docs" ".github/ISSUE_TEMPLATE") + +# define an array of files that won't be included in the list +# Get all files in the root directory except package.json and package-lock.json +# Create array of excluded files by finding all files in root dir except package.json and package-lock.json +excluded_files=("${ROOT_DIR}/.github/release-drafter.yml") +while IFS= read -r file; do + excluded_files+=("\"${file}\"") +done < <(find "${ROOT_DIR}" -maxdepth 1 -type f -not -name "package.json" -not -name "package-lock.json") + +# define an array of modules that won't be part of the build +readonly no_build_modules=("couchbase") + +# modules is an array that will store the paths of all the modules in the repository. +modules=() + +# Find all package.json files in the repository, building a list of all the available modules. +# The list of modules is stored in the modules array, but the testcontainers-node module is excluded +# as it is not a module. +for packageJSONFile in $(find "${ROOT_DIR}" -name "package.json" -not -path "*/node_modules/*"); do + name=$(basename "$(dirname "${packageJSONFile}")") + if [[ "${name}" != "testcontainers-node" ]]; then + modules+=("\"${name}\"") + fi +done + +# sort modules array +IFS=$'\n' modules=($(sort <<<"${modules[*]}")) +unset IFS + +# Get the list of modified files, retrieved from the environment variable ALL_CHANGED_FILES. +# On CI, this value will come from a Github Action retrieving the list of modified files from the pull request. +readonly modified_files=${ALL_CHANGED_FILES[@]} + +# Initialize variables +modified_modules=() + +# Check the modified files and determine which modules to build, following these rules: +# - if the modified files contain any file in the root module, include all modules in the list +# - if the modified files only contain files in one of the modules, include that module in the list +for file in $modified_files; do + # check if the file is in one of the excluded files + for exclude_file in ${excluded_files[@]}; do + # Remove quotes from exclude_file for comparison + clean_exclude_file=$(echo $exclude_file | tr -d '"') + if [[ "${ROOT_DIR}/${file}" == "${clean_exclude_file}" ]]; then + # if the file is in the excluded files, skip the rest of the loop. + # Execution continues at the loop control of the 2nd enclosing loop. + continue 2 + fi + done + + if [[ $file == packages/modules/* ]]; then + module_name=$(echo $file | cut -d'/' -f3) + if [[ ! " ${modified_modules[@]} " =~ " ${module_name} " ]]; then + modified_modules+=("\"$module_name\"") + fi + else + # a file from the core module (packages/testcontainers) is modified, so include all modules in the list and stop the loop + # check if the file is in one of the excluded modules + for exclude_module in ${excluded_modules[@]}; do + if [[ $file == $exclude_module/* ]]; then + # continue skips to the next iteration of an enclosing for, select, until, or while loop in a shell script. + # Execution continues at the loop control of the nth enclosing loop, in this case two levels up. + continue 2 + fi + done + + modified_modules=${modules[@]} + break + fi +done + +# print all modules with this format: +# each module will be enclosed in double quotes +# each module will be separated by a comma +# the entire list will be enclosed in square brackets +# the list will be sorted and unique +sorted_unique_modules=($(echo "${modified_modules[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')) + +# remove modules that won't be part of the build from the list +filtered_modules=() +for module in "${sorted_unique_modules[@]}"; do + skip=false + for no_build_module in "${no_build_modules[@]}"; do + if [[ ${module} == \"${no_build_module}\" ]]; then + skip=true + break + fi + done + if [[ $skip == false ]]; then + filtered_modules+=(${module}) + fi +done +sorted_unique_modules=("${filtered_modules[@]}") + +echo "["$(IFS=,; echo "${sorted_unique_modules[*]}" | sed 's/ /,/g')"]" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 8923d23fa..000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Lint - -on: - pull_request: - branches: [ main ] - -jobs: - lint: - runs-on: ubuntu-22.04 - steps: - - name: Code checkout - uses: actions/checkout@v4 - - name: Install NodeJS - uses: actions/setup-node@v4 - with: - node-version: 22.x - - name: Install dependencies - run: npm ci --omit=optional - - name: Code linting - run: npm run lint:ci diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index d1437b123..04d14ee1c 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -130,13 +130,13 @@ jobs: - name: Code checkout uses: actions/checkout@v4 - - name: Install NodeJS ${{ inputs.node-version }} - uses: actions/setup-node@v4 + - name: Install Node ${{ inputs.node-version }} and Dependencies + id: npm-install + uses: ./.github/actions/npm-setup with: + runner: ${{ inputs.runner }} node-version: ${{ inputs.node-version }} - - - name: Install dependencies - run: npm ci --omit=optional + workspace: "${{ inputs.workspace }}" - name: Run tests - run: npm run test:ci -- ${{ inputs.workspace }} + run: npm run test:ci -- ${{ steps.npm-install.outputs.workspace_path }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58e0cc032..f53674f7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,76 +19,147 @@ concurrency: cancel-in-progress: true jobs: + detect-modules: + runs-on: ubuntu-latest + outputs: + modules: ${{ steps.set-modified-modules.outputs.modules }} + modules_count: ${{ steps.set-modified-modules-count.outputs.modules_count }} + steps: + - name: Check out code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - id: changed-files + name: Get changed files + uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 + + - id: set-modified-modules + name: Set all modified modules + env: + ALL_CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + run: echo "modules=$(./.github/scripts/changed-modules.sh)" >> $GITHUB_OUTPUT + + - id: set-modified-modules-count + name: Set all modified modules count + run: echo "modules_count=$(echo ${{ toJSON(steps.set-modified-modules.outputs.modules) }} | jq '. | length')" >> $GITHUB_OUTPUT + + - name: Print out the modules to be used + run: | + echo "${{ steps.set-modified-modules-count.outputs.modules_count }} modules in the build" + echo "${{ steps.set-modified-modules.outputs.modules }}" + + lint: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + strategy: + fail-fast: true + matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} + runs-on: ubuntu-22.04 + steps: + - name: Code checkout + uses: actions/checkout@v4 + - name: Install Node and Dependencies + id: npm-install + uses: ./.github/actions/npm-setup + with: + runner: ubuntu-22.04 + node-version: 22.x + workspace: "${{ matrix.module }}" + - name: Code linting + env: + WORKSPACE_PATH: ${{ steps.npm-install.outputs.workspace_path }} + run: npm run lint:ci + smoke-test: + # only run if there are modules to lint + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint name: Smoke test - runs-on: ${{ matrix.runner }} strategy: + fail-fast: true matrix: runner: [ ubuntu-22.04 ] node-version: [ 18.x, 20.x, 22.x ] + runs-on: ${{ matrix.runner }} steps: - name: Code checkout uses: actions/checkout@v4 - - name: Install NodeJS ${{ matrix.node-version }} - uses: actions/setup-node@v4 + + # Uses a composite action for a consistent NPM install including cache + - name: Install Node ${{ matrix.node-version }} and Dependencies + uses: ./.github/actions/npm-setup with: + runner: ${{ matrix.runner }} node-version: ${{ matrix.node-version }} - - name: Install dependencies - run: npm ci --omit=optional - - name: Build workspaces - run: npm run build -ws + workspace: "testcontainers" + + - name: Build testcontainers + run: npm run build --workspace packages/testcontainers - name: Remove dev dependencies - run: npm prune --omit=dev + run: npm prune --omit=dev --workspace packages/testcontainers - name: Run CommonJS module smoke test run: node packages/testcontainers/smoke-test.js - name: Run ES module smoke test run: node packages/testcontainers/smoke-test.mjs - test-testcontainers: - name: Testcontainers - needs: smoke-test + test: + name: Run tests + # only run if there are modules to test + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint + - smoke-test strategy: + fail-fast: false matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} node-version: [ 18.x, 20.x, 22.x ] - container-runtime: [ docker, podman ] - include: - - container-runtime: docker - runner: ubuntu-22.04 - - container-runtime: podman - runner: ubuntu-22.04 uses: ./.github/workflows/test-template.yml with: - runner: ${{ matrix.runner }} + runner: ubuntu-22.04 node-version: ${{ matrix.node-version }} - container-runtime: ${{ matrix.container-runtime }} - workspace: packages/testcontainers + container-runtime: docker + workspace: "${{ matrix.module }}" - list-modules: - name: List modules - runs-on: ubuntu-22.04 - needs: test-testcontainers - outputs: - modules: ${{ steps.list_modules.outputs.modules }} - steps: - - uses: actions/checkout@v2 - - id: list_modules - run: echo "modules=$(ls packages/modules | jq -cnR '[inputs | select(length>0) | select(. != "couchbase")]')" >> $GITHUB_OUTPUT - test-modules: - name: Module (${{ matrix.module }}) - needs: [ test-testcontainers, list-modules ] + test-podman: + name: Run tests + # only run if there are modules to test + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint + - smoke-test + - test strategy: + fail-fast: false matrix: + module: ${{ fromJSON(needs.detect-modules.outputs.modules) }} node-version: [ 18.x, 20.x, 22.x ] - container-runtime: [ docker, podman ] - include: - - container-runtime: docker - runner: ubuntu-22.04 - - container-runtime: podman - runner: ubuntu-22.04 - module: ${{ fromJson(needs.list-modules.outputs.modules) }} uses: ./.github/workflows/test-template.yml with: - runner: ${{ matrix.runner }} + runner: ubuntu-22.04 node-version: ${{ matrix.node-version }} - container-runtime: ${{ matrix.container-runtime }} - workspace: packages/modules/${{ matrix.module }} + container-runtime: podman + workspace: "${{ matrix.module }}" + + # This job serves as confirmation that all test jobs finished + end: + if: ${{ needs.detect-modules.outputs.modules_count > 0 }} + needs: + - detect-modules + - lint + - smoke-test + - test + - test-podman + runs-on: ubuntu-latest + steps: + - name: Check if any jobs failed + if: ${{ failure() || cancelled() }} + run: exit 1 + + - run: echo "All tests completed successfully!" diff --git a/package.json b/package.json index 732e02c60..c30082b7f 100755 --- a/package.json +++ b/package.json @@ -12,7 +12,7 @@ "test:ci": "npm run test -- --runInBand --coverage", "format": "prettier --write package.json \"packages/**/*.ts\"", "lint": "eslint --fix package.json \"packages/**/*.ts\"", - "lint:ci": "eslint package.json \"packages/**/*.ts\" --max-warnings=0", + "lint:ci": "eslint package.json \"${WORKSPACE_PATH}/**/*.ts\" --max-warnings=0", "update-deps": "npm-check-updates --workspaces --root -u", "validate-engine": "ls-engines" },