diff --git a/.editorconfig b/.editorconfig index 9396e053..85fceb7b 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,7 +1,7 @@ root = true [*] -indent_style = space +indent_style = tabs indent_size = 4 charset = utf-8 trim_trailing_whitespace = true @@ -9,6 +9,15 @@ insert_final_newline = true end_of_line = lf # editorconfig-tools is unable to ignore longs strings or urls max_line_length = null +quote_type = single -[{*.yaml, *.yml}] +[*.md] indent_size = 2 + +[*.yml] +indent_size = 2 +indent_style = spaces + +[*.yaml] +indent_size = 2 +indent_style = spaces diff --git a/.github/actions/cache-builder/.gitignore b/.github/actions/cache-builder/.gitignore new file mode 100644 index 00000000..d5cd56bd --- /dev/null +++ b/.github/actions/cache-builder/.gitignore @@ -0,0 +1,180 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store + +# caches +data/npm +data/yarn +data/yarn_tmp diff --git a/.github/actions/cache-builder/README.md b/.github/actions/cache-builder/README.md new file mode 100644 index 00000000..8dc1ab6d --- /dev/null +++ b/.github/actions/cache-builder/README.md @@ -0,0 +1,72 @@ +# Version Matrix Actions Script + +This folder contains scripts that are used to create Actions matrices for building specific Docker images with the right version combinations of Apify SDK, Playwright/Puppeteer, and Crawlee. + +These scripts are ran using the [bun](https://bun.sh) runtime (for no reason other than ease of use). + +## Adding a new Node version to the matrix + +When a new version of Node is released, just update the `supportedNodeVersions` array in the `src/shares/constants.ts` file. + +Then, run `SKIP_CACHE_SET=true bun node:normal` locally to preview the new matrix. (you can append `| jq -r '.include[] | "node-version=\(.["node-version"]) apify-version=\(.["apify-version"]) is-latest=\(.["is-latest"])"'` to get a nicer output from the big JSON blob) + +## Adding a new Python version to the matrix + +When a new version of Python is released, just update the `supportedPythonVersions` array in the `src/shares/constants.ts` file. + +Then, run `SKIP_CACHE_SET=true bun python:normal` locally to preview the new matrix. (you can append `| jq -r '.include[] | "python-version=\(.["python-version"]) playwright-version=\(.["playwright-version"]) apify-version=\(.["apify-version"]) is-latest=\(.["is-latest"])"'` to get a nicer output from the big JSON blob) + +## Adding a new Python version range for specific Playwright version ranges + +Sometimes, newer Python is not compatible with Playwright versions that were released before a specific one (at the time of writing, this is the case for Playwright 1.48.0 and Python 3.13 -> Python 3.13.x can only run Playwright 1.48.0 and newer). + +To add a new Python version range for a specific Playwright version, add a new entry to the `playwrightPythonVersionConstraints` array in the `python.ts` file. + +The key represents the Python version range where this starts taking effect. The value is the Playwright version range that is required for the Python version. + +## Updating the runtime version that will be used for images that are referenced with just the build tag + +When we build images, we also include a specific runtime version in the tag (as an example, we have `apify/actor-node:20`). We also provide images tagged with `latest` or `beta`. These images will default to the "latest" runtime version that is specified in the `src/shares/constants.ts` file under `latestPythonVersion` or `latestNodeVersion`. + +When the time comes to bump these, just make a PR, edit those values, and merge it. Next time images get built, the `latest` or `beta` tags will use those new versions for the tag. + +## Creating new matrices + +The structure for a GitHub Actions matrix is as follows: + +```ts +interface Matrix { + include: MatrixEntry[]; +} + +type MatrixEntry = Record; +``` + +When trying to integrate a new matrix into a flow, you need to follow the following steps: + +- have a step that outputs the matrix as a JSON blob + + ```yaml + matrix: + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + steps: + - name: Generate matrix + id: set-matrix + run: echo "matrix=$(bun python:normal)" >> $GITHUB_OUTPUT + working-directory: ./.github/actions/version-matrix + ``` + +(optionally you can also add in a print step to ensure the matrix is correct. Feel free to copy it from any that uses previous matrices) + +- ensure the actual build step needs the matrix and uses it like this (the if check if optional if the matrix will always have at least one entry): + + ```yaml + needs: [matrix] + if: ${{ toJson(fromJson(needs.matrix.outputs.matrix).include) != '[]' }} + strategy: + matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} + ``` + +- reference matrix values based on the keys in the objects in the `include` array. For example, to get the Python version, you can use `${{ matrix.python-version }}`. diff --git a/.github/actions/cache-builder/data/.gitkeep b/.github/actions/cache-builder/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/.github/actions/cache-builder/data/npm_state.json b/.github/actions/cache-builder/data/npm_state.json new file mode 100644 index 00000000..3a14cb4d --- /dev/null +++ b/.github/actions/cache-builder/data/npm_state.json @@ -0,0 +1,37 @@ +{ + "crawlee": [ + "3.13.1", + "3.13.2", + "3.13.3", + "3.13.4", + "3.13.5" + ], + "apify": [ + "3.3.1", + "3.3.2", + "3.4.0", + "3.4.1", + "3.4.2" + ], + "playwright": [ + "1.50.0", + "1.50.1", + "1.51.0", + "1.51.1", + "1.52.0" + ], + "puppeteer": [ + "24.7.2", + "24.8.0", + "24.8.1", + "24.8.2", + "24.9.0" + ], + "typescript": [ + "5.6.3", + "5.7.2", + "5.7.3", + "5.8.2", + "5.8.3" + ] +} \ No newline at end of file diff --git a/.github/actions/cache-builder/data/yarn_state.json b/.github/actions/cache-builder/data/yarn_state.json new file mode 100644 index 00000000..228a1a70 --- /dev/null +++ b/.github/actions/cache-builder/data/yarn_state.json @@ -0,0 +1,37 @@ +{ + "crawlee": [ + "3.13.1", + "3.13.2", + "3.13.3", + "3.13.4", + "3.13.5" + ], + "apify": [ + "3.3.1", + "3.3.2", + "3.4.0", + "3.4.1", + "3.4.2" + ], + "playwright": [ + "1.50.0", + "1.50.1", + "1.51.0", + "1.51.1", + "1.52.0" + ], + "puppeteer": [ + "24.8.0", + "24.8.1", + "24.8.2", + "24.9.0", + "24.10.0" + ], + "typescript": [ + "5.6.3", + "5.7.2", + "5.7.3", + "5.8.2", + "5.8.3" + ] +} \ No newline at end of file diff --git a/.github/actions/cache-builder/package.json b/.github/actions/cache-builder/package.json new file mode 100644 index 00000000..76e09670 --- /dev/null +++ b/.github/actions/cache-builder/package.json @@ -0,0 +1,24 @@ +{ + "name": "cache-builder", + "type": "module", + "private": true, + "scripts": { + "node:npm": "node src/caches/npm.ts", + "node:yarn": "node src/caches/yarn.ts", + "fmt": "biome format --write ./src", + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "@types/node": "^22.15.32", + "@types/semver": "^7.7.0", + "typescript": "^5.8.3" + }, + "dependencies": { + "nano-spawn": "^1.0.2", + "semver": "^7.7.2" + }, + "volta": { + "extends": "../../../package.json" + }, + "packageManager": "yarn@4.9.1" +} diff --git a/.github/actions/cache-builder/src/caches/npm.ts b/.github/actions/cache-builder/src/caches/npm.ts new file mode 100644 index 00000000..079300de --- /dev/null +++ b/.github/actions/cache-builder/src/caches/npm.ts @@ -0,0 +1,33 @@ +import { writeFile } from 'node:fs/promises'; +import spawn from 'nano-spawn'; +import { packagesToPrecache } from '../shared/constants.ts'; +import { fetchPackageVersions, getCachePathData } from '../shared/npm.ts'; + +const cachePath = getCachePathData(); + +console.log(`Cache location: ${cachePath.path}`); + +const cacheState: Record = {}; + +for (const packageName of packagesToPrecache) { + const lastFiveVersions = (await fetchPackageVersions(packageName)).slice(-5); + cacheState[packageName] = lastFiveVersions; + + for (const version of lastFiveVersions) { + console.log(`Fetching ${packageName}@${version}`); + + try { + await spawn('npm', ['cache', 'add', `${packageName}@${version}`], { + env: { + [cachePath.environmentVariable]: cachePath.path, + }, + }); + } catch (error) { + console.error(`Failed to fetch ${packageName}@${version}:`, error); + } finally { + console.log(`Done fetching ${packageName}@${version}`); + } + } +} + +await writeFile(new URL('../../data/npm_state.json', import.meta.url), JSON.stringify(cacheState, null, '\t')); diff --git a/.github/actions/cache-builder/src/caches/yarn.ts b/.github/actions/cache-builder/src/caches/yarn.ts new file mode 100644 index 00000000..d2130fbc --- /dev/null +++ b/.github/actions/cache-builder/src/caches/yarn.ts @@ -0,0 +1,62 @@ +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import spawn from 'nano-spawn'; +import { getCachePath, packagesToPrecache } from '../shared/constants.ts'; +import { fetchPackageVersions } from '../shared/npm.ts'; + +const environmentVariable = 'YARN_CACHE_FOLDER'; +const globalEnvironmentVariable = 'YARN_GLOBAL_FOLDER'; +const cachePath = getCachePath('yarn'); + +console.log(`Cache location: ${cachePath}`); + +const cacheState: Record = {}; + +const tmpDir = new URL('../../data/yarn_tmp/', import.meta.url); + +{ + // Warm up the temp directory + await mkdir(tmpDir, { recursive: true }); + + await writeFile(new URL('package.json', tmpDir), JSON.stringify({ private: true })); + await writeFile(new URL('yarn.lock', tmpDir), ''); + + await spawn('yarn', 'config set enableGlobalCache true'.split(' '), { + cwd: tmpDir, + }); + + await spawn('yarn', 'config set nodeLinker node-modules'.split(' '), { + cwd: tmpDir, + }); + + await spawn('yarn', 'set version stable --yarn-path'.split(' '), { + cwd: tmpDir, + }); +} + +for (const packageName of packagesToPrecache) { + const lastFiveVersions = (await fetchPackageVersions(packageName)).slice(-5); + cacheState[packageName] = lastFiveVersions; + + for (const version of lastFiveVersions) { + console.log(`Fetching ${packageName}@${version}`); + + try { + await spawn('yarn', ['add', `${packageName}@${version}`], { + env: { + [environmentVariable]: cachePath, + [globalEnvironmentVariable]: cachePath, + YARN_ENABLE_GLOBAL_CACHE: 'true', + }, + cwd: tmpDir, + }); + } catch (error) { + console.error(`Failed to fetch ${packageName}@${version}:`, error); + } finally { + console.log(`Done fetching ${packageName}@${version}`); + } + } +} + +await writeFile(new URL('../../data/yarn_state.json', import.meta.url), JSON.stringify(cacheState, null, '\t')); + +await rm(tmpDir, { recursive: true }); diff --git a/.github/actions/cache-builder/src/shared/constants.ts b/.github/actions/cache-builder/src/shared/constants.ts new file mode 100644 index 00000000..bfa3defa --- /dev/null +++ b/.github/actions/cache-builder/src/shared/constants.ts @@ -0,0 +1,14 @@ +import { fileURLToPath } from 'node:url'; + +export const packagesToPrecache = [ + // + 'crawlee', + 'apify', + 'playwright', + 'puppeteer', + 'typescript', +]; + +export function getCachePath(pm: string) { + return fileURLToPath(new URL(`../../data/${pm}/`, import.meta.url)); +} diff --git a/.github/actions/cache-builder/src/shared/npm.ts b/.github/actions/cache-builder/src/shared/npm.ts new file mode 100644 index 00000000..2f9b2578 --- /dev/null +++ b/.github/actions/cache-builder/src/shared/npm.ts @@ -0,0 +1,42 @@ +import { compare } from 'semver'; +import { getCachePath } from './constants.ts'; + +const npmPackageInfoRoute = (pkg: string) => `https://registry.npmjs.org/${pkg}`; + +interface PackageVersionInfo { + name: string; + version: string; + engines?: Record; +} + +interface PackageInfo { + name: string; + 'dist-tags': Record; + versions: Record; +} + +export async function fetchPackageVersions(packageName: string) { + const url = npmPackageInfoRoute(packageName); + + const response = await fetch(url); + + if (!response.ok) { + throw new Error(`Failed to fetch package info for ${packageName}`, { + cause: await response.text(), + }); + } + + const json: PackageInfo = await response.json(); + + // Avoid versions with suffixes for this + const versions = Object.keys(json.versions).filter((version) => !/[a-z]/.test(version)); + + return versions.sort((a, b) => compare(a, b)); +} + +export function getCachePathData() { + return { + path: getCachePath('npm'), + environmentVariable: 'npm_config_cache', + }; +} diff --git a/.github/actions/cache-builder/tsconfig.json b/.github/actions/cache-builder/tsconfig.json new file mode 100644 index 00000000..ffedd977 --- /dev/null +++ b/.github/actions/cache-builder/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "NodeNext", + "moduleDetection": "force", + "allowJs": true, + + // Bundler mode + "moduleResolution": "NodeNext", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + }, + "include": ["src/**/*.ts"] +} diff --git a/.github/actions/version-matrix/package.json b/.github/actions/version-matrix/package.json index fb88a32f..0e4a5291 100644 --- a/.github/actions/version-matrix/package.json +++ b/.github/actions/version-matrix/package.json @@ -1,5 +1,5 @@ { - "name": "python-matrix", + "name": "version-matrix", "type": "module", "private": true, "scripts": { @@ -9,23 +9,19 @@ "node:normal": "node src/matrices/node/normal.ts", "node:puppeteer": "node src/matrices/node/puppeteer.ts", "node:playwright": "node src/matrices/node/playwright.ts", - "fmt": "biome format --write . && biome format --write ../../../renovate.json", + "fmt": "biome format --write .", "typecheck": "tsc --noEmit" }, "devDependencies": { - "@biomejs/biome": "^2.0.0", - "@types/node": "^22.15.18", + "@types/node": "^22.15.32", "@types/semver": "^7.7.0", "typescript": "^5.8.3" }, "dependencies": { - "@actions/core": "^1.11.1", - "@actions/github": "^6.0.1", "semver": "^7.7.2" }, "volta": { - "node": "24.1.0", - "yarn": "4.9.1" + "extends": "../../../package.json" }, "packageManager": "yarn@4.9.1" } diff --git a/.github/actions/version-matrix/yarn.lock b/.github/actions/version-matrix/yarn.lock deleted file mode 100644 index fb41fbfc..00000000 --- a/.github/actions/version-matrix/yarn.lock +++ /dev/null @@ -1,394 +0,0 @@ -# This file is generated by running "yarn install" inside your project. -# Manual changes might be lost - proceed with caution! - -__metadata: - version: 8 - cacheKey: 10 - -"@actions/core@npm:^1.11.1": - version: 1.11.1 - resolution: "@actions/core@npm:1.11.1" - dependencies: - "@actions/exec": "npm:^1.1.1" - "@actions/http-client": "npm:^2.0.1" - checksum: 10/94f260e33607cc16567ce4c88014f069cd7da92baaa443b72cff80fdf4f1dcd18192e135df0d51ec29e8b82cfe214218715d482f2a7804efa5095737d1245f38 - languageName: node - linkType: hard - -"@actions/exec@npm:^1.1.1": - version: 1.1.1 - resolution: "@actions/exec@npm:1.1.1" - dependencies: - "@actions/io": "npm:^1.0.1" - checksum: 10/c04bd25191e522841c7e17862d70099de8db61278d2b4c744b69ac0197a48f85c75dba548e1c29c695c4cc5363d558846b4dcd3db9013463c18ba8cf36497c6d - languageName: node - linkType: hard - -"@actions/github@npm:^6.0.1": - version: 6.0.1 - resolution: "@actions/github@npm:6.0.1" - dependencies: - "@actions/http-client": "npm:^2.2.0" - "@octokit/core": "npm:^5.0.1" - "@octokit/plugin-paginate-rest": "npm:^9.2.2" - "@octokit/plugin-rest-endpoint-methods": "npm:^10.4.0" - "@octokit/request": "npm:^8.4.1" - "@octokit/request-error": "npm:^5.1.1" - undici: "npm:^5.28.5" - checksum: 10/ba6a162a5727dea2f3f3fc450e02c5b336ceb65a0e26ba9ad9c62b20f4f5b2625ca347a9311a4905ef3c92378ca022caba841a283cb7f2e4175d79e3d1ecaf12 - languageName: node - linkType: hard - -"@actions/http-client@npm:^2.0.1, @actions/http-client@npm:^2.2.0": - version: 2.2.3 - resolution: "@actions/http-client@npm:2.2.3" - dependencies: - tunnel: "npm:^0.0.6" - undici: "npm:^5.25.4" - checksum: 10/0c0a540c79e50f795d214f696710bb9c50bdf5bb1458be288140f2aae3686adec73fdb464c43da5ef94f985ac7736273efef21cb5ba5a3b09e85b403d852c04b - languageName: node - linkType: hard - -"@actions/io@npm:^1.0.1": - version: 1.1.3 - resolution: "@actions/io@npm:1.1.3" - checksum: 10/4de44e8d428ba9f20049c844b37ecd486b589ed201f8cc8c5b550a9e4c72d1f594271ee2a7a6cfe8a42ebfb5dd527ef65016454656db391a353d41eab4f147e1 - languageName: node - linkType: hard - -"@biomejs/biome@npm:^2.0.0": - version: 2.0.0 - resolution: "@biomejs/biome@npm:2.0.0" - dependencies: - "@biomejs/cli-darwin-arm64": "npm:2.0.0" - "@biomejs/cli-darwin-x64": "npm:2.0.0" - "@biomejs/cli-linux-arm64": "npm:2.0.0" - "@biomejs/cli-linux-arm64-musl": "npm:2.0.0" - "@biomejs/cli-linux-x64": "npm:2.0.0" - "@biomejs/cli-linux-x64-musl": "npm:2.0.0" - "@biomejs/cli-win32-arm64": "npm:2.0.0" - "@biomejs/cli-win32-x64": "npm:2.0.0" - dependenciesMeta: - "@biomejs/cli-darwin-arm64": - optional: true - "@biomejs/cli-darwin-x64": - optional: true - "@biomejs/cli-linux-arm64": - optional: true - "@biomejs/cli-linux-arm64-musl": - optional: true - "@biomejs/cli-linux-x64": - optional: true - "@biomejs/cli-linux-x64-musl": - optional: true - "@biomejs/cli-win32-arm64": - optional: true - "@biomejs/cli-win32-x64": - optional: true - bin: - biome: bin/biome - checksum: 10/d4df2e3a0d3ff9db4eca3b4f5458b9de2b180f3f56cdb874e57f0c933f72f68398803d563016dd23882cbae5090f50cae6538d9ec3609fe931db793d19cdc985 - languageName: node - linkType: hard - -"@biomejs/cli-darwin-arm64@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-darwin-arm64@npm:2.0.0" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - -"@biomejs/cli-darwin-x64@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-darwin-x64@npm:2.0.0" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - -"@biomejs/cli-linux-arm64-musl@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-linux-arm64-musl@npm:2.0.0" - conditions: os=linux & cpu=arm64 & libc=musl - languageName: node - linkType: hard - -"@biomejs/cli-linux-arm64@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-linux-arm64@npm:2.0.0" - conditions: os=linux & cpu=arm64 & libc=glibc - languageName: node - linkType: hard - -"@biomejs/cli-linux-x64-musl@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-linux-x64-musl@npm:2.0.0" - conditions: os=linux & cpu=x64 & libc=musl - languageName: node - linkType: hard - -"@biomejs/cli-linux-x64@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-linux-x64@npm:2.0.0" - conditions: os=linux & cpu=x64 & libc=glibc - languageName: node - linkType: hard - -"@biomejs/cli-win32-arm64@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-win32-arm64@npm:2.0.0" - conditions: os=win32 & cpu=arm64 - languageName: node - linkType: hard - -"@biomejs/cli-win32-x64@npm:2.0.0": - version: 2.0.0 - resolution: "@biomejs/cli-win32-x64@npm:2.0.0" - conditions: os=win32 & cpu=x64 - languageName: node - linkType: hard - -"@fastify/busboy@npm:^2.0.0": - version: 2.1.1 - resolution: "@fastify/busboy@npm:2.1.1" - checksum: 10/2bb8a7eca8289ed14c9eb15239bc1019797454624e769b39a0b90ed204d032403adc0f8ed0d2aef8a18c772205fa7808cf5a1b91f21c7bfc7b6032150b1062c5 - languageName: node - linkType: hard - -"@octokit/auth-token@npm:^4.0.0": - version: 4.0.0 - resolution: "@octokit/auth-token@npm:4.0.0" - checksum: 10/60e42701e341d700f73c518c7a35675d36d79fa9d5e838cc3ade96d147e49f5ba74db2e07b2337c2b95aaa540aa42088116df2122daa25633f9e70a2c8785c44 - languageName: node - linkType: hard - -"@octokit/core@npm:^5.0.1": - version: 5.2.1 - resolution: "@octokit/core@npm:5.2.1" - dependencies: - "@octokit/auth-token": "npm:^4.0.0" - "@octokit/graphql": "npm:^7.1.0" - "@octokit/request": "npm:^8.4.1" - "@octokit/request-error": "npm:^5.1.1" - "@octokit/types": "npm:^13.0.0" - before-after-hook: "npm:^2.2.0" - universal-user-agent: "npm:^6.0.0" - checksum: 10/9d95da740f350811dc5aadbf6670f3ddb8735f7c80029add497311ca0b13c4919fea473110e3fd9e85929e51ee64797fad8732e2f31703d55d80bdab1fdc846b - languageName: node - linkType: hard - -"@octokit/endpoint@npm:^9.0.6": - version: 9.0.6 - resolution: "@octokit/endpoint@npm:9.0.6" - dependencies: - "@octokit/types": "npm:^13.1.0" - universal-user-agent: "npm:^6.0.0" - checksum: 10/2bf776423365ee926bf3f722a664e52f1070758eff4a176279fb132103fd0c76e3541f83ace49bbad9a64f9c9b8de453be565ca8d6136989e9514dea65380ecf - languageName: node - linkType: hard - -"@octokit/graphql@npm:^7.1.0": - version: 7.1.1 - resolution: "@octokit/graphql@npm:7.1.1" - dependencies: - "@octokit/request": "npm:^8.4.1" - "@octokit/types": "npm:^13.0.0" - universal-user-agent: "npm:^6.0.0" - checksum: 10/9a7a65fa84df795b0acb5315dae5a4a5a042a01dde0c88974df180a1c02b9b8e61cae013be32461b11ee1d507a8f778f3b7f37dfa3b371771332cb8efcd01f29 - languageName: node - linkType: hard - -"@octokit/openapi-types@npm:^20.0.0": - version: 20.0.0 - resolution: "@octokit/openapi-types@npm:20.0.0" - checksum: 10/9f60572af1201dd92626c412253d83d986b8ab1956250b95f417013ee8e7baf25870eeb801d16672cabc2c420544bc9c2f0a979e07603ff5997eff038c71a8c3 - languageName: node - linkType: hard - -"@octokit/openapi-types@npm:^24.2.0": - version: 24.2.0 - resolution: "@octokit/openapi-types@npm:24.2.0" - checksum: 10/000897ebc6e247c2591049d6081e95eb5636f73798dadd695ee6048496772b58065df88823e74a760201828545a7ac601dd3c1bcd2e00079a62a9ee9d389409c - languageName: node - linkType: hard - -"@octokit/plugin-paginate-rest@npm:^9.2.2": - version: 9.2.2 - resolution: "@octokit/plugin-paginate-rest@npm:9.2.2" - dependencies: - "@octokit/types": "npm:^12.6.0" - peerDependencies: - "@octokit/core": 5 - checksum: 10/9afdd61d24a276ed7c2a8e436f735066d1b71601177deb97afa204a1f224257ca9c02681bc94dcda921d37c288a342124f7dfdd88393817306fe0b1ad1f0690f - languageName: node - linkType: hard - -"@octokit/plugin-rest-endpoint-methods@npm:^10.4.0": - version: 10.4.1 - resolution: "@octokit/plugin-rest-endpoint-methods@npm:10.4.1" - dependencies: - "@octokit/types": "npm:^12.6.0" - peerDependencies: - "@octokit/core": 5 - checksum: 10/1090fc5a1bebb7b48c512e178f8ad69a3ef8332e583274972f3a3035e9be9200093e22a5dbfe0f71aa1a7a8817e54bb915af3c2a3f88db1311a2873cef176552 - languageName: node - linkType: hard - -"@octokit/request-error@npm:^5.1.1": - version: 5.1.1 - resolution: "@octokit/request-error@npm:5.1.1" - dependencies: - "@octokit/types": "npm:^13.1.0" - deprecation: "npm:^2.0.0" - once: "npm:^1.4.0" - checksum: 10/6ad98626407ba57bb33fa197611be74bee1dd9abc8d5d845648d6a2a04aa6840c0eb7f4be341d55dfcab5bc19181ad5fd25194869a7aaac6245f74b3a14d9662 - languageName: node - linkType: hard - -"@octokit/request@npm:^8.4.1": - version: 8.4.1 - resolution: "@octokit/request@npm:8.4.1" - dependencies: - "@octokit/endpoint": "npm:^9.0.6" - "@octokit/request-error": "npm:^5.1.1" - "@octokit/types": "npm:^13.1.0" - universal-user-agent: "npm:^6.0.0" - checksum: 10/2b2c9131cc9b608baeeef8ce2943768cc9db5fbe36a665f734a099bd921561c760e4391fbdf39d5aefb725db26742db1488c65624940ef7cec522e10863caa5e - languageName: node - linkType: hard - -"@octokit/types@npm:^12.6.0": - version: 12.6.0 - resolution: "@octokit/types@npm:12.6.0" - dependencies: - "@octokit/openapi-types": "npm:^20.0.0" - checksum: 10/19b77a8d25af2a5df4561f8750f807edfc9fca5b07cfa9fb21dce4665e1b188c966688f5ed5e08089404428100dfe44ad353f8d8532f1d30fe47e61c5faa1440 - languageName: node - linkType: hard - -"@octokit/types@npm:^13.0.0, @octokit/types@npm:^13.1.0": - version: 13.10.0 - resolution: "@octokit/types@npm:13.10.0" - dependencies: - "@octokit/openapi-types": "npm:^24.2.0" - checksum: 10/32f8f5010d7faae128b0cdd0c221f0ca8c3781fe44483ecd87162b3da507db667f7369acda81340f6e2c9c374d9a938803409c6085c2c01d98210b6c58efb99a - languageName: node - linkType: hard - -"@types/node@npm:^22.15.18": - version: 22.15.29 - resolution: "@types/node@npm:22.15.29" - dependencies: - undici-types: "npm:~6.21.0" - checksum: 10/3426790c5aa22df445213d7f37e57ea261cf3013030fe9b3025d87c302097799a9db3b848e2a9bdb07cab8ec6c7e9947ac770cf11e12e420148296b63d63e7db - languageName: node - linkType: hard - -"@types/semver@npm:^7.7.0": - version: 7.7.0 - resolution: "@types/semver@npm:7.7.0" - checksum: 10/ee4514c6c852b1c38f951239db02f9edeea39f5310fad9396a00b51efa2a2d96b3dfca1ae84c88181ea5b7157c57d32d7ef94edacee36fbf975546396b85ba5b - languageName: node - linkType: hard - -"before-after-hook@npm:^2.2.0": - version: 2.2.3 - resolution: "before-after-hook@npm:2.2.3" - checksum: 10/e676f769dbc4abcf4b3317db2fd2badb4a92c0710e0a7da12cf14b59c3482d4febf835ad7de7874499060fd4e13adf0191628e504728b3c5bb4ec7a878c09940 - languageName: node - linkType: hard - -"deprecation@npm:^2.0.0": - version: 2.3.1 - resolution: "deprecation@npm:2.3.1" - checksum: 10/f56a05e182c2c195071385455956b0c4106fe14e36245b00c689ceef8e8ab639235176a96977ba7c74afb173317fac2e0ec6ec7a1c6d1e6eaa401c586c714132 - languageName: node - linkType: hard - -"once@npm:^1.4.0": - version: 1.4.0 - resolution: "once@npm:1.4.0" - dependencies: - wrappy: "npm:1" - checksum: 10/cd0a88501333edd640d95f0d2700fbde6bff20b3d4d9bdc521bdd31af0656b5706570d6c6afe532045a20bb8dc0849f8332d6f2a416e0ba6d3d3b98806c7db68 - languageName: node - linkType: hard - -"python-matrix@workspace:.": - version: 0.0.0-use.local - resolution: "python-matrix@workspace:." - dependencies: - "@actions/core": "npm:^1.11.1" - "@actions/github": "npm:^6.0.1" - "@biomejs/biome": "npm:^2.0.0" - "@types/node": "npm:^22.15.18" - "@types/semver": "npm:^7.7.0" - semver: "npm:^7.7.2" - typescript: "npm:^5.8.3" - languageName: unknown - linkType: soft - -"semver@npm:^7.7.2": - version: 7.7.2 - resolution: "semver@npm:7.7.2" - bin: - semver: bin/semver.js - checksum: 10/7a24cffcaa13f53c09ce55e05efe25cd41328730b2308678624f8b9f5fc3093fc4d189f47950f0b811ff8f3c3039c24a2c36717ba7961615c682045bf03e1dda - languageName: node - linkType: hard - -"tunnel@npm:^0.0.6": - version: 0.0.6 - resolution: "tunnel@npm:0.0.6" - checksum: 10/cf1ffed5e67159b901a924dbf94c989f20b2b3b65649cfbbe4b6abb35955ce2cf7433b23498bdb2c5530ab185b82190fce531597b3b4a649f06a907fc8702405 - languageName: node - linkType: hard - -"typescript@npm:^5.8.3": - version: 5.8.3 - resolution: "typescript@npm:5.8.3" - bin: - tsc: bin/tsc - tsserver: bin/tsserver - checksum: 10/65c40944c51b513b0172c6710ee62e951b70af6f75d5a5da745cb7fab132c09ae27ffdf7838996e3ed603bb015dadd099006658046941bd0ba30340cc563ae92 - languageName: node - linkType: hard - -"typescript@patch:typescript@npm%3A^5.8.3#optional!builtin": - version: 5.8.3 - resolution: "typescript@patch:typescript@npm%3A5.8.3#optional!builtin::version=5.8.3&hash=5786d5" - bin: - tsc: bin/tsc - tsserver: bin/tsserver - checksum: 10/b9b1e73dabac5dc730c041325dbd9c99467c1b0d239f1b74ec3b90d831384af3e2ba973946232df670519147eb51a2c20f6f96163cea2b359f03de1e2091cc4f - languageName: node - linkType: hard - -"undici-types@npm:~6.21.0": - version: 6.21.0 - resolution: "undici-types@npm:6.21.0" - checksum: 10/ec8f41aa4359d50f9b59fa61fe3efce3477cc681908c8f84354d8567bb3701fafdddf36ef6bff307024d3feb42c837cf6f670314ba37fc8145e219560e473d14 - languageName: node - linkType: hard - -"undici@npm:^5.25.4, undici@npm:^5.28.5": - version: 5.29.0 - resolution: "undici@npm:5.29.0" - dependencies: - "@fastify/busboy": "npm:^2.0.0" - checksum: 10/0ceca8924a32acdcc0cfb8dd2d368c217840970aa3f5e314fc169608474be6341c5b8e50cad7bd257dbe3b4e432bc5d0a0d000f83644b54fa11a48735ec52b93 - languageName: node - linkType: hard - -"universal-user-agent@npm:^6.0.0": - version: 6.0.1 - resolution: "universal-user-agent@npm:6.0.1" - checksum: 10/fdc8e1ae48a05decfc7ded09b62071f571c7fe0bd793d700704c80cea316101d4eac15cc27ed2bb64f4ce166d2684777c3198b9ab16034f547abea0d3aa1c93c - languageName: node - linkType: hard - -"wrappy@npm:1": - version: 1.0.2 - resolution: "wrappy@npm:1.0.2" - checksum: 10/159da4805f7e84a3d003d8841557196034155008f817172d4e986bd591f74aa82aa7db55929a54222309e01079a65a92a9e6414da5a6aa4b01ee44a511ac3ee5 - languageName: node - linkType: hard diff --git a/.github/scripts/prepare-node-image-tags.js b/.github/scripts/prepare-node-image-tags.js index cad9bc35..3f3c02e8 100644 --- a/.github/scripts/prepare-node-image-tags.js +++ b/.github/scripts/prepare-node-image-tags.js @@ -1,35 +1,36 @@ module.exports = () => { - const { CURRENT_NODE, LATEST_NODE, RELEASE_TAG, IMAGE_NAME, FRAMEWORK_VERSION, IS_LATEST_BROWSER_IMAGE } = process.env - const tags = []; + const { CURRENT_NODE, LATEST_NODE, RELEASE_TAG, IMAGE_NAME, FRAMEWORK_VERSION, IS_LATEST_BROWSER_IMAGE } = + process.env; + const tags = []; - if (CURRENT_NODE === LATEST_NODE && IS_LATEST_BROWSER_IMAGE === 'true') { - // apify/actor-node-x:latest - tags.push(`${IMAGE_NAME}:${RELEASE_TAG}`); - } + if (CURRENT_NODE === LATEST_NODE && IS_LATEST_BROWSER_IMAGE === 'true') { + // apify/actor-node-x:latest + tags.push(`${IMAGE_NAME}:${RELEASE_TAG}`); + } - // latest version - if (RELEASE_TAG === 'latest') { - if (FRAMEWORK_VERSION) { - // apify/actor-node-x:20-4.2.0 - tags.push(`${IMAGE_NAME}:${CURRENT_NODE}-${FRAMEWORK_VERSION}`) - } + // latest version + if (RELEASE_TAG === 'latest') { + if (FRAMEWORK_VERSION) { + // apify/actor-node-x:20-4.2.0 + tags.push(`${IMAGE_NAME}:${CURRENT_NODE}-${FRAMEWORK_VERSION}`); + } - // apify/actor-node-x:20 - // we want this only when the browser image is also latest - if (IS_LATEST_BROWSER_IMAGE === 'true') { - tags.push(`${IMAGE_NAME}:${CURRENT_NODE}`); - } - } else { - // beta and other tags - if (FRAMEWORK_VERSION) { - // apify/actor-node-x:20-4.2.0-beta - tags.push(`${IMAGE_NAME}:${CURRENT_NODE}-${FRAMEWORK_VERSION}-${RELEASE_TAG}`); - } + // apify/actor-node-x:20 + // we want this only when the browser image is also latest + if (IS_LATEST_BROWSER_IMAGE === 'true') { + tags.push(`${IMAGE_NAME}:${CURRENT_NODE}`); + } + } else { + // beta and other tags + if (FRAMEWORK_VERSION) { + // apify/actor-node-x:20-4.2.0-beta + tags.push(`${IMAGE_NAME}:${CURRENT_NODE}-${FRAMEWORK_VERSION}-${RELEASE_TAG}`); + } - // apify/actor-node-x:20-beta - // we don't care if the browser image is latest or not, as its a beta image - tags.push(`${IMAGE_NAME}:${CURRENT_NODE}-${RELEASE_TAG}`); - } + // apify/actor-node-x:20-beta + // we don't care if the browser image is latest or not, as its a beta image + tags.push(`${IMAGE_NAME}:${CURRENT_NODE}-${RELEASE_TAG}`); + } - return { allTags: tags.join(','), firstImageName: tags[0] }; -} + return { allTags: tags.join(','), firstImageName: tags[0] }; +}; diff --git a/.github/scripts/prepare-python-image-tags.js b/.github/scripts/prepare-python-image-tags.js index 33aac422..d000699b 100644 --- a/.github/scripts/prepare-python-image-tags.js +++ b/.github/scripts/prepare-python-image-tags.js @@ -1,35 +1,36 @@ module.exports = () => { - const { CURRENT_PYTHON, LATEST_PYTHON, RELEASE_TAG, IMAGE_NAME, FRAMEWORK_VERSION, IS_LATEST_BROWSER_IMAGE } = process.env - const tags = []; + const { CURRENT_PYTHON, LATEST_PYTHON, RELEASE_TAG, IMAGE_NAME, FRAMEWORK_VERSION, IS_LATEST_BROWSER_IMAGE } = + process.env; + const tags = []; - if (CURRENT_PYTHON === LATEST_PYTHON && IS_LATEST_BROWSER_IMAGE === 'true') { - // apify/actor-python-x:latest - tags.push(`${IMAGE_NAME}:${RELEASE_TAG}`); - } + if (CURRENT_PYTHON === LATEST_PYTHON && IS_LATEST_BROWSER_IMAGE === 'true') { + // apify/actor-python-x:latest + tags.push(`${IMAGE_NAME}:${RELEASE_TAG}`); + } - // latest version - if (RELEASE_TAG === 'latest') { - if (FRAMEWORK_VERSION) { - // apify/actor-python-x:3.13-4.2.0 - tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}-${FRAMEWORK_VERSION}`) - } + // latest version + if (RELEASE_TAG === 'latest') { + if (FRAMEWORK_VERSION) { + // apify/actor-python-x:3.13-4.2.0 + tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}-${FRAMEWORK_VERSION}`); + } - // apify/actor-python-x:3.13 - // we want this only when browser image is also latest - if (IS_LATEST_BROWSER_IMAGE === 'true') { - tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}`); - } - } else { - // beta and other tags - if (FRAMEWORK_VERSION) { - // apify/actor-python-x:3.13-4.2.0-beta - tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}-${FRAMEWORK_VERSION}-${RELEASE_TAG}`); - } + // apify/actor-python-x:3.13 + // we want this only when browser image is also latest + if (IS_LATEST_BROWSER_IMAGE === 'true') { + tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}`); + } + } else { + // beta and other tags + if (FRAMEWORK_VERSION) { + // apify/actor-python-x:3.13-4.2.0-beta + tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}-${FRAMEWORK_VERSION}-${RELEASE_TAG}`); + } - // apify/actor-python-x:3.13-beta - // we don't care if the browser image is latest or not, as its a beta image - tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}-${RELEASE_TAG}`); - } + // apify/actor-python-x:3.13-beta + // we don't care if the browser image is latest or not, as its a beta image + tags.push(`${IMAGE_NAME}:${CURRENT_PYTHON}-${RELEASE_TAG}`); + } - return { allTags: tags.join(','), firstImageName: tags[0] }; -} + return { allTags: tags.join(','), firstImageName: tags[0] }; +}; diff --git a/.github/workflows/cache-node.yaml b/.github/workflows/cache-node.yaml new file mode 100644 index 00000000..4b6163b6 --- /dev/null +++ b/.github/workflows/cache-node.yaml @@ -0,0 +1,75 @@ +name: Node package caches + +on: + workflow_dispatch: + + repository_dispatch: + types: + - build-node-cache + + pull_request: + + # schedule: + # - cron: 0 */2 * * * + +jobs: + cache-node: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version-file: package.json + cache: yarn + cache-dependency-path: yarn.lock + + - run: yarn + + - run: yarn node:npm + working-directory: .github/actions/cache-builder + + - run: yarn node:yarn + working-directory: .github/actions/cache-builder + + - name: Print npm state + run: | + cat .github/actions/cache-builder/data/npm_state.json + + - name: Print yarn state + run: | + cat .github/actions/cache-builder/data/yarn_state.json + + - name: Store npm output + uses: actions/cache/save@v4 + with: + path: .github/actions/cache-builder/data/npm/* + key: node-npm-${{ hashFiles('.github/actions/cache-builder/data/npm_state.json') }} + + - name: Store yarn output + uses: actions/cache/save@v4 + with: + path: .github/actions/cache-builder/data/yarn/* + key: node-yarn-${{ hashFiles('.github/actions/cache-builder/data/yarn_state.json') }} + + - name: Push updated states + if: github.event_name != 'pull_request' + run: | + # Setup git user + git config --global user.email "noreply@apify.com" + git config --global user.name "Apify CI Bot" + git config pull.rebase true + + # Add and commit if there are changes + git add ./.github/actions/cache-builder/data/*.json + git diff-index --quiet HEAD || git commit -m "chore(pkg-caches): update node caches" + + # Try to push 5 times, with pulls between retries + for i in {1..5}; do + git push && break || echo "Failed to push, retrying in 5 seconds..." && sleep 5 && git pull + done diff --git a/.github/workflows/release-node-playwright.yaml b/.github/workflows/release-node-playwright.yaml index 8319ce70..64716366 100644 --- a/.github/workflows/release-node-playwright.yaml +++ b/.github/workflows/release-node-playwright.yaml @@ -4,16 +4,16 @@ on: workflow_dispatch: inputs: release_tag: - description: "Tag for the images (e.g.: beta)" + description: 'Tag for the images (e.g.: beta)' required: true apify_version: - description: "Apify SDK version (e.g.: ^1.0.0). If missing, the latest version will be used." + description: 'Apify SDK version (e.g.: ^1.0.0). If missing, the latest version will be used.' required: false crawlee_version: - description: "Crawlee version (e.g.: ^1.0.0). If missing, the latest version will be used." + description: 'Crawlee version (e.g.: ^1.0.0). If missing, the latest version will be used.' required: false rebuild_images: - description: "Rebuilds images even if the cache state matches the current state." + description: 'Rebuilds images even if the cache state matches the current state.' required: false type: boolean @@ -48,12 +48,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version-file: .github/actions/version-matrix/package.json + node-version-file: package.json cache: yarn - cache-dependency-path: .github/actions/version-matrix/yarn.lock + cache-dependency-path: yarn.lock - run: yarn - working-directory: ./.github/actions/version-matrix - name: Generate matrix id: set-matrix @@ -99,7 +98,7 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} - name: "img: ${{ matrix.image-name }} node: ${{ matrix.node-version }}, apify: ${{ matrix.apify-version }}, crawlee: ${{ matrix.crawlee-version }}, pw: ${{ matrix.playwright-version }}, cf: ${{ matrix.camoufox-version }}, is-latest: ${{ matrix.is-latest }}" + name: 'img: ${{ matrix.image-name }} node: ${{ matrix.node-version }}, apify: ${{ matrix.apify-version }}, crawlee: ${{ matrix.crawlee-version }}, pw: ${{ matrix.playwright-version }}, cf: ${{ matrix.camoufox-version }}, is-latest: ${{ matrix.is-latest }}' steps: - name: Set default inputs if event is pull request diff --git a/.github/workflows/release-node-puppeteer.yaml b/.github/workflows/release-node-puppeteer.yaml index cf66e009..90df1f3f 100644 --- a/.github/workflows/release-node-puppeteer.yaml +++ b/.github/workflows/release-node-puppeteer.yaml @@ -4,16 +4,16 @@ on: workflow_dispatch: inputs: release_tag: - description: "Tag for the images (e.g.: beta)" + description: 'Tag for the images (e.g.: beta)' required: true apify_version: - description: "Apify SDK version (e.g.: ^1.0.0). If missing, the latest version will be used." + description: 'Apify SDK version (e.g.: ^1.0.0). If missing, the latest version will be used.' required: false crawlee_version: - description: "Crawlee version (e.g.: ^1.0.0). If missing, the latest version will be used." + description: 'Crawlee version (e.g.: ^1.0.0). If missing, the latest version will be used.' required: false rebuild_images: - description: "Rebuilds images even if the cache state matches the current state." + description: 'Rebuilds images even if the cache state matches the current state.' required: false type: boolean @@ -48,12 +48,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version-file: .github/actions/version-matrix/package.json + node-version-file: package.json cache: yarn - cache-dependency-path: .github/actions/version-matrix/yarn.lock + cache-dependency-path: yarn.lock - run: yarn - working-directory: ./.github/actions/version-matrix - name: Generate matrix id: set-matrix @@ -98,7 +97,7 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} - name: "node: ${{ matrix.node-version }}, apify: ${{ matrix.apify-version }}, crawlee: ${{ matrix.crawlee-version }}, pptr: ${{ matrix.puppeteer-version }}, is-latest: ${{ matrix.is-latest }}" + name: 'node: ${{ matrix.node-version }}, apify: ${{ matrix.apify-version }}, crawlee: ${{ matrix.crawlee-version }}, pptr: ${{ matrix.puppeteer-version }}, is-latest: ${{ matrix.is-latest }}' steps: - name: Set default inputs if event is pull request diff --git a/.github/workflows/release-node.yaml b/.github/workflows/release-node.yaml index b6a8ed79..d374b7e8 100644 --- a/.github/workflows/release-node.yaml +++ b/.github/workflows/release-node.yaml @@ -4,16 +4,16 @@ on: workflow_dispatch: inputs: release_tag: - description: "Tag for the images (e.g.: beta)" + description: 'Tag for the images (e.g.: beta)' required: true apify_version: - description: "Apify SDK version (e.g.: ^1.0.0). If missing, the latest version will be used." + description: 'Apify SDK version (e.g.: ^1.0.0). If missing, the latest version will be used.' required: false crawlee_version: - description: "Crawlee version (e.g.: ^1.0.0). If missing, the latest version will be used." + description: 'Crawlee version (e.g.: ^1.0.0). If missing, the latest version will be used.' required: false rebuild_images: - description: "Rebuilds images even if the cache state matches the current state." + description: 'Rebuilds images even if the cache state matches the current state.' required: false type: boolean @@ -47,12 +47,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version-file: .github/actions/version-matrix/package.json + node-version-file: package.json cache: yarn - cache-dependency-path: .github/actions/version-matrix/yarn.lock + cache-dependency-path: yarn.lock - run: yarn - working-directory: ./.github/actions/version-matrix - name: Generate matrix id: set-matrix @@ -95,7 +94,7 @@ jobs: fail-fast: false matrix: ${{ fromJson(needs.matrix.outputs.matrix) }} - name: "node: ${{ matrix.node-version }} apify: ${{ matrix.apify-version }} crawlee: ${{ matrix.crawlee-version }}" + name: 'node: ${{ matrix.node-version }} apify: ${{ matrix.apify-version }} crawlee: ${{ matrix.crawlee-version }}' steps: - name: Set default inputs if event is pull request @@ -156,7 +155,7 @@ jobs: RELEASE_TAG: ${{ env.RELEASE_TAG }} IMAGE_NAME: apify/actor-${{ matrix.image-name }} # Force this to true, as these images have no browsers - IS_LATEST_BROWSER_IMAGE: "true" + IS_LATEST_BROWSER_IMAGE: 'true' with: script: | const generateTags = require("./.github/scripts/prepare-node-image-tags.js"); diff --git a/.github/workflows/release-python-playwright.yaml b/.github/workflows/release-python-playwright.yaml index f8cd5bf2..f03d0d1f 100644 --- a/.github/workflows/release-python-playwright.yaml +++ b/.github/workflows/release-python-playwright.yaml @@ -43,12 +43,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version-file: .github/actions/version-matrix/package.json + node-version-file: package.json cache: yarn - cache-dependency-path: .github/actions/version-matrix/yarn.lock + cache-dependency-path: yarn.lock - run: yarn - working-directory: ./.github/actions/version-matrix - name: Generate matrix id: set-matrix diff --git a/.github/workflows/release-python-selenium.yaml b/.github/workflows/release-python-selenium.yaml index 23b3f64c..2369fe18 100644 --- a/.github/workflows/release-python-selenium.yaml +++ b/.github/workflows/release-python-selenium.yaml @@ -43,12 +43,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version-file: .github/actions/version-matrix/package.json + node-version-file: package.json cache: yarn - cache-dependency-path: .github/actions/version-matrix/yarn.lock + cache-dependency-path: yarn.lock - run: yarn - working-directory: ./.github/actions/version-matrix - name: Generate matrix id: set-matrix diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 30ecc5b8..493a2cc7 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -42,12 +42,11 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 with: - node-version-file: .github/actions/version-matrix/package.json + node-version-file: package.json cache: yarn - cache-dependency-path: .github/actions/version-matrix/yarn.lock + cache-dependency-path: yarn.lock - run: yarn - working-directory: ./.github/actions/version-matrix - name: Generate matrix id: set-matrix diff --git a/.gitignore b/.gitignore index a3374f62..c977c837 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ npm-debug.log node_modules .DS_Store +.turbo +.yarn/install-state.gz diff --git a/.github/actions/version-matrix/.yarn/releases/yarn-4.9.1.cjs b/.yarn/releases/yarn-4.9.1.cjs similarity index 100% rename from .github/actions/version-matrix/.yarn/releases/yarn-4.9.1.cjs rename to .yarn/releases/yarn-4.9.1.cjs diff --git a/.github/actions/version-matrix/.yarnrc.yml b/.yarnrc.yml similarity index 100% rename from .github/actions/version-matrix/.yarnrc.yml rename to .yarnrc.yml diff --git a/package.json b/package.json new file mode 100644 index 00000000..ff19982a --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "@apify/docker-repository", + "private": true, + "workspaces": [ + ".github/actions/*" + ], + "packageManager": "yarn@4.9.1", + "volta": { + "node": "24.1.0", + "yarn": "4.9.1" + }, + "devDependencies": { + "@biomejs/biome": "^2.0.0", + "turbo": "^2.5.4" + }, + "scripts": { + "fmt": "biome format --write renovate.json && turbo fmt" + } +} diff --git a/turbo.json b/turbo.json new file mode 100644 index 00000000..de46cce5 --- /dev/null +++ b/turbo.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://turborepo.com/schema.json", + "tasks": { + "fmt": { + "cache": false + } + } +} diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 00000000..65c9a785 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,259 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10 + +"@apify/docker-repository@workspace:.": + version: 0.0.0-use.local + resolution: "@apify/docker-repository@workspace:." + dependencies: + "@biomejs/biome": "npm:^2.0.0" + turbo: "npm:^2.5.4" + languageName: unknown + linkType: soft + +"@biomejs/biome@npm:^2.0.0": + version: 2.0.0 + resolution: "@biomejs/biome@npm:2.0.0" + dependencies: + "@biomejs/cli-darwin-arm64": "npm:2.0.0" + "@biomejs/cli-darwin-x64": "npm:2.0.0" + "@biomejs/cli-linux-arm64": "npm:2.0.0" + "@biomejs/cli-linux-arm64-musl": "npm:2.0.0" + "@biomejs/cli-linux-x64": "npm:2.0.0" + "@biomejs/cli-linux-x64-musl": "npm:2.0.0" + "@biomejs/cli-win32-arm64": "npm:2.0.0" + "@biomejs/cli-win32-x64": "npm:2.0.0" + dependenciesMeta: + "@biomejs/cli-darwin-arm64": + optional: true + "@biomejs/cli-darwin-x64": + optional: true + "@biomejs/cli-linux-arm64": + optional: true + "@biomejs/cli-linux-arm64-musl": + optional: true + "@biomejs/cli-linux-x64": + optional: true + "@biomejs/cli-linux-x64-musl": + optional: true + "@biomejs/cli-win32-arm64": + optional: true + "@biomejs/cli-win32-x64": + optional: true + bin: + biome: bin/biome + checksum: 10/d4df2e3a0d3ff9db4eca3b4f5458b9de2b180f3f56cdb874e57f0c933f72f68398803d563016dd23882cbae5090f50cae6538d9ec3609fe931db793d19cdc985 + languageName: node + linkType: hard + +"@biomejs/cli-darwin-arm64@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-darwin-arm64@npm:2.0.0" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@biomejs/cli-darwin-x64@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-darwin-x64@npm:2.0.0" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@biomejs/cli-linux-arm64-musl@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-linux-arm64-musl@npm:2.0.0" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"@biomejs/cli-linux-arm64@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-linux-arm64@npm:2.0.0" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"@biomejs/cli-linux-x64-musl@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-linux-x64-musl@npm:2.0.0" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"@biomejs/cli-linux-x64@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-linux-x64@npm:2.0.0" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"@biomejs/cli-win32-arm64@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-win32-arm64@npm:2.0.0" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@biomejs/cli-win32-x64@npm:2.0.0": + version: 2.0.0 + resolution: "@biomejs/cli-win32-x64@npm:2.0.0" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@types/node@npm:^22.15.32": + version: 22.15.32 + resolution: "@types/node@npm:22.15.32" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10/10b4c106d0c512a1d35ec08142bd7fb5cf2e1df93fc5627b3c69dd843dec4be07a47f1fa7ede232ad84762d75a372ea35028b79ee1e753b6f2adecd0b2cb2f71 + languageName: node + linkType: hard + +"@types/semver@npm:^7.7.0": + version: 7.7.0 + resolution: "@types/semver@npm:7.7.0" + checksum: 10/ee4514c6c852b1c38f951239db02f9edeea39f5310fad9396a00b51efa2a2d96b3dfca1ae84c88181ea5b7157c57d32d7ef94edacee36fbf975546396b85ba5b + languageName: node + linkType: hard + +"cache-builder@workspace:.github/actions/cache-builder": + version: 0.0.0-use.local + resolution: "cache-builder@workspace:.github/actions/cache-builder" + dependencies: + "@types/node": "npm:^22.15.32" + "@types/semver": "npm:^7.7.0" + nano-spawn: "npm:^1.0.2" + semver: "npm:^7.7.2" + typescript: "npm:^5.8.3" + languageName: unknown + linkType: soft + +"nano-spawn@npm:^1.0.2": + version: 1.0.2 + resolution: "nano-spawn@npm:1.0.2" + checksum: 10/6ce9e60846d2e37c0e3cd048472683c81dbcaadef9ebe73bfc8754ee7da2a574f724436d3dcdeda5d807aedc857cc8cbc278a9882529164b5ef4b170b95cfe0b + languageName: node + linkType: hard + +"semver@npm:^7.7.2": + version: 7.7.2 + resolution: "semver@npm:7.7.2" + bin: + semver: bin/semver.js + checksum: 10/7a24cffcaa13f53c09ce55e05efe25cd41328730b2308678624f8b9f5fc3093fc4d189f47950f0b811ff8f3c3039c24a2c36717ba7961615c682045bf03e1dda + languageName: node + linkType: hard + +"turbo-darwin-64@npm:2.5.4": + version: 2.5.4 + resolution: "turbo-darwin-64@npm:2.5.4" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"turbo-darwin-arm64@npm:2.5.4": + version: 2.5.4 + resolution: "turbo-darwin-arm64@npm:2.5.4" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"turbo-linux-64@npm:2.5.4": + version: 2.5.4 + resolution: "turbo-linux-64@npm:2.5.4" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"turbo-linux-arm64@npm:2.5.4": + version: 2.5.4 + resolution: "turbo-linux-arm64@npm:2.5.4" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"turbo-windows-64@npm:2.5.4": + version: 2.5.4 + resolution: "turbo-windows-64@npm:2.5.4" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"turbo-windows-arm64@npm:2.5.4": + version: 2.5.4 + resolution: "turbo-windows-arm64@npm:2.5.4" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"turbo@npm:^2.5.4": + version: 2.5.4 + resolution: "turbo@npm:2.5.4" + dependencies: + turbo-darwin-64: "npm:2.5.4" + turbo-darwin-arm64: "npm:2.5.4" + turbo-linux-64: "npm:2.5.4" + turbo-linux-arm64: "npm:2.5.4" + turbo-windows-64: "npm:2.5.4" + turbo-windows-arm64: "npm:2.5.4" + dependenciesMeta: + turbo-darwin-64: + optional: true + turbo-darwin-arm64: + optional: true + turbo-linux-64: + optional: true + turbo-linux-arm64: + optional: true + turbo-windows-64: + optional: true + turbo-windows-arm64: + optional: true + bin: + turbo: bin/turbo + checksum: 10/43dd952192a1261de3845ecac96d4f42ea6d8e49eaa4c339c029dbe010a1323957ef4b0080f8f06e3cd0169c1f00c356d32cbabde1ee08c72b0708f90994a774 + languageName: node + linkType: hard + +"typescript@npm:^5.8.3": + version: 5.8.3 + resolution: "typescript@npm:5.8.3" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10/65c40944c51b513b0172c6710ee62e951b70af6f75d5a5da745cb7fab132c09ae27ffdf7838996e3ed603bb015dadd099006658046941bd0ba30340cc563ae92 + languageName: node + linkType: hard + +"typescript@patch:typescript@npm%3A^5.8.3#optional!builtin": + version: 5.8.3 + resolution: "typescript@patch:typescript@npm%3A5.8.3#optional!builtin::version=5.8.3&hash=5786d5" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10/b9b1e73dabac5dc730c041325dbd9c99467c1b0d239f1b74ec3b90d831384af3e2ba973946232df670519147eb51a2c20f6f96163cea2b359f03de1e2091cc4f + languageName: node + linkType: hard + +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10/ec8f41aa4359d50f9b59fa61fe3efce3477cc681908c8f84354d8567bb3701fafdddf36ef6bff307024d3feb42c837cf6f670314ba37fc8145e219560e473d14 + languageName: node + linkType: hard + +"version-matrix@workspace:.github/actions/version-matrix": + version: 0.0.0-use.local + resolution: "version-matrix@workspace:.github/actions/version-matrix" + dependencies: + "@types/node": "npm:^22.15.32" + "@types/semver": "npm:^7.7.0" + semver: "npm:^7.7.2" + typescript: "npm:^5.8.3" + languageName: unknown + linkType: soft