diff --git a/.github/download-artifacts/index.js b/.github/download-artifacts/index.js index 86759c134b..75d43aab0d 100644 --- a/.github/download-artifacts/index.js +++ b/.github/download-artifacts/index.js @@ -29,7 +29,7 @@ const rimraf = require('rimraf') const fetch = require('node-fetch') const crossZip = require('cross-zip') -const { mkdir, rename, readdir } = promises +const { mkdir, rename, readdir, unlink } = promises const pipeline = promisify(stream.pipeline) const unzip = promisify(crossZip.unzip) const rm = promisify(rimraf) @@ -40,23 +40,21 @@ const downloadedSpec = join(esFolder, 'rest-api-spec', 'api') const specFolder = join(__dirname, '..', '..', 'specification', '_json_spec') async function downloadArtifacts (opts) { - if (typeof opts.version !== 'string' && typeof opts.branch !== 'string') { - throw new Error('Missing version or branch') + if (typeof opts.branch !== 'string') { + throw new Error('Missing branch') } - core.info('Checking out spec and test') + core.info('Resolving artifact URL') - core.info('Resolving version') let resolved try { - resolved = await resolve(opts.version || fromBranch(opts.branch), opts.hash) + resolved = await resolve(opts.branch) } catch (err) { core.error(err.message) process.exit(1) } - opts.version = resolved.version - core.info(`Resolved version ${opts.version}`) + core.info(`Resolved artifact URL for ${resolved.commit_url}`) core.info('Cleanup') await rm(esFolder) @@ -85,77 +83,37 @@ async function downloadArtifacts (opts) { await rename(join(downloadedSpec, file), join(specFolder, file)) } - core.info('Done') -} - -async function resolve (version, hash) { - if (version === 'latest') { - const response = await fetch('https://artifacts-api.elastic.co/v1/versions') - if (!response.ok) { - throw new Error(`unexpected response ${response.statusText}`) + /** Delete files that weren't in the zip file */ + const specFiles = await readdir(specFolder) + for (const file of specFiles) { + if (!files.includes(file)) { + await unlink(join(specFolder, file)) } - const { versions } = await response.json() - version = versions.pop() } - core.info(`Resolving version ${version}`) - const response = await fetch(`https://artifacts-api.elastic.co/v1/versions/${version}`) + core.info('Done') +} + +async function resolve (branch) { + const url = `https://artifacts-snapshot.elastic.co/elasticsearch/latest/${branch}.json` + const response = await fetch(url) if (!response.ok) { - throw new Error(`unexpected response ${response.statusText}`) + throw new Error(`Unexpected response. Invalid version? ${url}: ${response.statusText}`) } - const data = await response.json() - const esBuilds = data.version.builds - .filter(build => build.projects.elasticsearch != null) - .map(build => { - return { - projects: build.projects.elasticsearch, - buildId: build.build_id, - date: build.start_time, - version: build.version - } - }) - .sort((a, b) => { - const dA = new Date(a.date) - const dB = new Date(b.date) - if (dA > dB) return -1 - if (dA < dB) return 1 - return 0 - }) - - if (hash != null) { - const build = esBuilds.find(build => build.projects.commit_hash === hash) - if (!build) { - throw new Error(`Can't find any build with hash '${hash}'`) - } - const zipKey = Object.keys(build.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip')) - return { - url: build.projects.packages[zipKey].url, - id: build.buildId, - hash: build.projects.commit_hash, - version: build.version - } - } - const lastBuild = esBuilds[0] - const zipKey = Object.keys(lastBuild.projects.packages).find(key => key.startsWith('rest-resources-zip-') && key.endsWith('.zip')) - return { - url: lastBuild.projects.packages[zipKey].url, - id: lastBuild.buildId, - hash: lastBuild.projects.commit_hash, - version: lastBuild.version + let manifest_url = data.manifest_url + const manifestResponse = await fetch(manifest_url) + if (!manifestResponse.ok) { + throw new Error(`Unexpected manifestResponse. ${manifest_url}: ${manifestResponse.statusText}`) } -} + const manifestData = await manifestResponse.json() + const elasticsearch = manifestData.projects.elasticsearch + const restResourceName = `rest-resources-zip-${manifestData.version}.zip` -function fromBranch (branch) { - if (branch === 'main') { - return 'latest' - } else if (branch === '7.x') { - return '7.x-SNAPSHOT' - } else if ((branch.startsWith('7.') || branch.startsWith('8.')) && !isNaN(Number(branch.split('.')[1]))) { - return `${branch}-SNAPSHOT` - } else { - throw new Error(`Cannot derive version from branch '${branch}'`) + return { + url: elasticsearch.packages[restResourceName].url, + commit_url: elasticsearch.commit_url, } } @@ -164,7 +122,7 @@ async function main (options) { } const options = minimist(process.argv.slice(2), { - string: ['id', 'version', 'hash', 'branch'] + string: ['branch'] }) main(options).catch(t => { core.error(t) diff --git a/.github/workflows/update-rest-api-json.yml b/.github/workflows/update-rest-api-json.yml index 78e041cad1..0a4067d623 100644 --- a/.github/workflows/update-rest-api-json.yml +++ b/.github/workflows/update-rest-api-json.yml @@ -3,7 +3,7 @@ name: Update rest-api-spec on: workflow_dispatch: schedule: - - cron: '0 */3 * * *' # At minute 0 past every 3rd hour. + - cron: '0 4 * * *' # At 04:00. jobs: update-rest-api: @@ -11,11 +11,12 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - branch: ['main', '8.1', '8.0', '7.17'] + branch: ['main', '8.x', '8.15', '7.17'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: ref: ${{ matrix.branch }} @@ -29,31 +30,36 @@ jobs: npm install --prefix .github/download-artifacts npm install --prefix compiler npm install --prefix typescript-generator + - name: Download artifacts run: | node .github/download-artifacts/index.js --branch ${{ matrix.branch }} + - name: Generate output run: | - SKIP_VERSION_UPDATE=true make contrib + make contrib + - name: Debug git status run: | git status --porcelain + - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v6 with: title: Update rest-api-spec ${{ matrix.branch }} body: 'As titled.' commit-message: 'Update rest-api-spec' labels: specification delete-branch: true - team-reviewers: elastic/clients-team + reviewers: Anaethelion,ezimuel,flobernd,JoshMock,l-trotta,miguelgrinberg,picandocodigo,pquentin,srikanthmanvi,swallez,technige branch: automated/rest-api-spec-update-${{ matrix.branch }} - name: Open an issue if the action fails if: ${{ failure() }} - uses: nashmaniac/create-issue-action@v1.1 + uses: nashmaniac/create-issue-action@v1.2 with: title: rest-api-spec update failed token: ${{ secrets.GITHUB_TOKEN }} labels: bug body: The rest-api-spec action is currently failing, see [here](https://github.com/elastic/elasticsearch-specification/actions/workflows/update-rest-api-json.yml). +