From 88a5619de7e001deb9fc3dea6a7eaf580b7149b8 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Wed, 31 Jul 2024 15:28:36 +0200 Subject: [PATCH 01/26] Add Gtilab functionalities Add code to handle Gitlab repository, add axios to project packages, add env variables for Gitlab --- .env.example | 2 + config/default.json | 1 + package-lock.json | 33 +- package.json | 3 +- .../dataset/assets/README.templateGitLab.js | 65 ++++ scripts/dataset/index.js | 25 +- scripts/dataset/publishGitLab/index.js | 102 ++++++ src/index.js | 19 + src/reporterGitlab/gitlab.js | 324 ++++++++++++++++++ src/reporterGitlab/index.js | 147 ++++++++ src/reporterGitlab/labels.json | 77 +++++ src/reporterGitlab/labels.test.js | 30 ++ 12 files changed, 814 insertions(+), 14 deletions(-) create mode 100644 scripts/dataset/assets/README.templateGitLab.js create mode 100644 scripts/dataset/publishGitLab/index.js create mode 100644 src/reporterGitlab/gitlab.js create mode 100644 src/reporterGitlab/index.js create mode 100644 src/reporterGitlab/labels.json create mode 100644 src/reporterGitlab/labels.test.js diff --git a/.env.example b/.env.example index 892ff5b3..08813d69 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,5 @@ OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…' OTA_ENGINE_SMTP_PASSWORD='password' OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX +OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX +OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX diff --git a/config/default.json b/config/default.json index 0d3d82ea..9009d52a 100644 --- a/config/default.json +++ b/config/default.json @@ -59,6 +59,7 @@ "dataset": { "title": "sandbox", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox", + "versionsRepositoryURLGitLab": "https://gitlab.com/p2b/contrib-versions", "publishingSchedule": "30 8 * * MON" } } diff --git a/package-lock.json b/package-lock.json index 7fb26663..dfed6a31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", + "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", @@ -171,6 +172,14 @@ "npm": ">=6" } }, + "node_modules/@accordproject/concerto-util/node_modules/axios": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", + "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@accordproject/markdown-cicero": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/@accordproject/markdown-cicero/-/markdown-cicero-0.15.2.tgz", @@ -324,6 +333,14 @@ "npm": ">=6" } }, + "node_modules/@accordproject/markdown-pdf/node_modules/axios": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", + "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@accordproject/markdown-pdf/node_modules/pdfjs-dist": { "version": "2.13.216", "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.13.216.tgz", @@ -3964,11 +3981,13 @@ } }, "node_modules/axios": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", - "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", "dependencies": { - "follow-redirects": "^1.14.4" + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "node_modules/b4a": { @@ -6478,9 +6497,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", diff --git a/package.json b/package.json index 8fccde58..8da099a3 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,8 @@ "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "winston": "^3.3.3", - "winston-mail": "^2.0.0" + "winston-mail": "^2.0.0", + "axios": "^1.7.2" }, "devDependencies": { "@commitlint/cli": "^19.0.3", diff --git a/scripts/dataset/assets/README.templateGitLab.js b/scripts/dataset/assets/README.templateGitLab.js new file mode 100644 index 00000000..c9f0bb8d --- /dev/null +++ b/scripts/dataset/assets/README.templateGitLab.js @@ -0,0 +1,65 @@ +import config from 'config'; + +const LOCALE = 'en-EN'; +const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; + +export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { + return `# Open Terms Archive — ${title({ releaseDate })} + +${body({ servicesCount, firstVersionDate, lastVersionDate })}`; +} + +export function title({ releaseDate }) { + releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + + const title = config.get('@opentermsarchive/engine.dataset.title'); + + return `${title} — ${releaseDate} dataset`; +} + +export function body({ servicesCount, firstVersionDate, lastVersionDate }) { + firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + + const versionsRepositoryURLGitLab = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. + +This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURLGitLab}](${versionsRepositoryURLGitLab}). + +It has been generated with [Open Terms Archive](https://opentermsarchive.org). + +### Dataset format + +This dataset represents each version of a document as a separate [Markdown](https://spec.commonmark.org/0.30/) file, nested in a directory with the name of the service provider and in a directory with the name of the terms type. The filesystem layout will look like below. + +\`\`\` +├ README.md +├┬ Service provider 1 (e.g. Facebook) +│├┬ Terms type 1 (e.g. Terms of Service) +││├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-08-01T01-03-12Z.md) +┆┆┆ +││└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-10-03T08-12-25Z.md) +┆┆ +│└┬ Terms type X (e.g. Privacy Policy) +│ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) +┆ ┆ +│ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) +┆ +└┬ Service provider Y (e.g. Google) + ├┬ Terms type 1 (e.g. Developer Terms) + │├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2019-03-12T04-18-22Z.md) + ┆┆ + │└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-12-04T22-47-05Z.md) + └┬ Terms type Z (e.g. Privacy Policy) + ┆ + ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) + ┆ + └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) +\`\`\` + +### License + +This dataset is made available under an [Open Database (OdBL) License](https://opendatacommons.org/licenses/odbl/1.0/) by Open Terms Archive Contributors. +`; +} diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 4c739686..d6b74a3b 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -6,6 +6,7 @@ import config from 'config'; import generateRelease from './export/index.js'; import logger from './logger/index.js'; import publishRelease from './publish/index.js'; +import publishReleaseGitLab from './publishGitLab/index.js'; export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) { const releaseDate = new Date(); @@ -24,13 +25,25 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start publishing dataset…'); - const releaseUrl = await publishRelease({ - archivePath, - releaseDate, - stats, - }); + if (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined') { + const releaseUrl = await publishRelease({ + archivePath, + releaseDate, + stats, + }); - logger.info(`Dataset published to ${releaseUrl}`); + logger.info(`Dataset published to ${releaseUrl}`); + } + + if (typeof process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN !== 'undefined') { + const releaseUrl = await publishReleaseGitLab({ + archivePath, + releaseDate, + stats, + }); + + logger.info(`Dataset published to ${releaseUrl}`); + } if (!shouldRemoveLocalCopy) { return; diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js new file mode 100644 index 00000000..2a1f9bbc --- /dev/null +++ b/scripts/dataset/publishGitLab/index.js @@ -0,0 +1,102 @@ +import fsApi from 'fs'; +import path from 'path'; +import url from 'url'; + +import axios from 'axios'; + +import config from 'config'; +import dotenv from 'dotenv'; +//import { Octokit } from 'octokit'; + +import FormData from 'form-data'; + +import * as readme from '../assets/README.templateGitLab.js'; + +dotenv.config(); + +const gitlabAPIUrl = "https://gitlab.com/api/v4"; +const gitlabUrl = "https://gitlab.com"; + +export default async function publishReleaseGitLab({ + archivePath, + releaseDate, + stats, +}) { + let projectId = null; + + // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); + + const [owner, repo] = url + .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) + .pathname.split('/') + .filter((component) => component); + const commonParams = { owner, repo }; + + try { + const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; + const response = await axios.get( + `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, + }, + }, + ); + projectId = response.data.id; + } catch (error) { + //logger.error(`🤖 Error while obtaining projectId: ${error}`); + projectId = null; + } + + const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag + + try { + // First, create the release + const releaseResponse = await axios.post( + `${gitlabAPIUrl}/projects/${projectId}/releases`, + { + ref: 'main', + tag_name: tagName, + name: readme.title({ releaseDate }), + description: readme.body(stats), + }, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + const releaseId = releaseResponse.data.commit.id; + + // Then, upload the ZIP file as an asset to the release + const formData = new FormData(); + formData.append('name', archivePath); + formData.append( + 'url', + `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`, + ); + formData.append('file', fsApi.createReadStream(archivePath), { + filename: path.basename(archivePath), + }); + + const uploadResponse = await axios.post( + `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, + formData, + { + headers: { + ...formData.getHeaders(), + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, + }, + }, + ); + + const releaseUrl = uploadResponse.data.direct_asset_url; + + return releaseUrl; + } catch (error) { + console.error('Failed to create release or upload ZIP file:', error); + throw error; + } +} diff --git a/src/index.js b/src/index.js index 2083e180..3f9c4813 100644 --- a/src/index.js +++ b/src/index.js @@ -8,6 +8,7 @@ import Archivist from './archivist/index.js'; import logger from './logger/index.js'; import Notifier from './notifier/index.js'; import Reporter from './reporter/index.js'; +import ReporterGitlab from './reporterGitlab/index.js'; const require = createRequire(import.meta.url); @@ -65,6 +66,7 @@ export default async function track({ services, types, extractOnly, schedule }) } catch (error) { logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); } + archivist.attach(reporter); } else { logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); } @@ -72,6 +74,23 @@ export default async function track({ services, types, extractOnly, schedule }) logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored'); } + if (process.env.OTA_ENGINE_GITLAB_TOKEN) { + if (config.has('@opentermsarchive/engine.reporter.gitlabIssues.repositories.declarations')) { + try { + const reporter = new ReporterGitlab(config.get('@opentermsarchive/engine.reporter')); + + await reporter.initialize(); + archivist.attach(reporter); + } catch (error) { + logger.error('Cannot instantiate the ReporterGitlab module; it will be ignored:', error); + } + } else { + logger.warn('Configuration key "reporter.gitlabIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + } + } else { + logger.warn('Environment variable "OTA_ENGINE_GITLAB_TOKEN" was not found; the ReporterGitlab module will be ignored'); + } + if (!schedule) { await archivist.track({ services, types }); diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js new file mode 100644 index 00000000..e688d33e --- /dev/null +++ b/src/reporterGitlab/gitlab.js @@ -0,0 +1,324 @@ +import { createRequire } from 'module'; + +import logger from '../logger/index.js'; + +const require = createRequire(import.meta.url); + +export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; + +const gitlabUrl = "https://gitlab.com/api/v4"; + +export default class GitLab { + static ISSUE_STATE_CLOSED = 'closed'; + static ISSUE_STATE_OPEN = 'opened'; + static ISSUE_STATE_ALL = 'all'; + + constructor(repository) { + //const { version } = require('../../package.json'); + + const [owner, repo] = repository.split('/'); + + this.commonParams = { owner, repo }; + } + + async initialize() { + const axios = require('axios'); + + try { + const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; + const response = await axios.get( + `${gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + }, + }, + ); + this.projectId = response.data.id; + } catch (error) { + logger.error(`🤖 Error while obtaining projectId: ${error}`); + this.projectId = null; + } + this.MANAGED_LABELS = require('./labels.json'); + + const existingLabels = await this.getRepositoryLabels(); + const existingLabelsNames = existingLabels.map((label) => label.name); + const missingLabels = this.MANAGED_LABELS.filter( + (label) => !existingLabelsNames.includes(label.name), + ); + + if (missingLabels.length) { + logger.info( + `🤖 Following required labels are not present on the repository: ${missingLabels.map((label) => `"${label.name}"`).join(', ')}. Creating them…`, + ); + + for (const label of missingLabels) { + await this.createLabel({ + /* eslint-disable-line no-await-in-loop */ name: label.name, + color: label.color, + description: `${label.description} ${MANAGED_BY_OTA_MARKER}`, + }); + } + } + } + + async getRepositoryLabels() { + try { + const response = await fetch( + `${gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + }, + }, + ); + if (response.status == 200) { + const labels = response.json(); + return labels; + } else { + logger.error( + `🤖 Failed to get labels: ${response.status_code} - ${response.text}`, + ); + return null; + } + } catch (error) { + logger.error(`🤖 Could get labels: ${error}`); + } + } + + async createLabel({ name, color, description }) { + const axios = require('axios'); + + try { + const label = { + name: name, + color: color, + description: description, + }; + const response = await axios.post( + `${gitlabUrl}/projects/${this.projectId}/labels`, + label, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + logger.info(`🤖 New label created: ${response.data.name}`); + } catch (error) { + logger.error(`🤖 Failed to create label: ${error}`); + } + } + + async createIssue({ title, description, labels }) { + const axios = require('axios'); + + try { + const issue = { + title: title, + labels: labels, + description: description, + }; + const response = await axios.post( + `${gitlabUrl}/projects/${this.projectId}/issues`, + issue, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + logger.info( + `🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`, + ); + + return response; + } catch (error) { + logger.error(`🤖 Could not create GitLab issue "${title}": ${error}`); + } + } + + async setIssueLabels({ issue, labels }) { + const axios = require('axios'); + + try { + const newLabels = { + labels: labels, + }; + const response = await axios.put( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + newLabels, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + } catch (error) { + logger.error( + `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async openIssue(issue) { + const axios = require('axios'); + + try { + const updateIssue = { + state_event: 'reopen', + }; + const response = await axios.put( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + updateIssue, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + logger.info(`🤖 Opened GitLab issue #${issue.iid}`); + } catch (error) { + logger.error( + `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async closeIssue(issue) { + const axios = require('axios'); + + try { + const updateIssue = { + state_event: 'close', + }; + const response = await axios.put( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + updateIssue, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + } catch (error) { + logger.error( + `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async getIssue({ title, ...searchParams }) { + const axios = require('axios'); + + try { + let apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + if (searchParams.state == 'all') + apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; + const response = await axios.get(apiUrl, { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + }, + }); + const issues = response.data; + + const [issue] = issues.filter((item) => item.title === title); // since only one is expected, use the first one + + setTimeout(() => { + console.log(title + ' - ' + apiUrl); + }, 5000); + + return issue; + } catch (error) { + logger.error(`🤖 Could not find GitLab issue "${title}": ${error}`); + } + } + + async addCommentToIssue({ issue, comment }) { + const axios = require('axios'); + const body = { + body: comment, + }; + + try { + const response = await axios.post( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, + body, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + logger.info( + `🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`, + ); + + return response.data.body; + } catch (error) { + logger.error( + `🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async closeIssueWithCommentIfExists({ title, comment }) { + const openedIssue = await this.getIssue({ + title, + state: GitLab.ISSUE_STATE_OPEN, + }); + + if (!openedIssue) { + return; + } + + await this.addCommentToIssue({ issue: openedIssue, comment }); + + return this.closeIssue(openedIssue); + } + + async createOrUpdateIssue({ title, description, label }) { + const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_ALL }); + + if (!issue) { + return this.createIssue({ title, description, labels: [label] }); + } + + if (issue.state == GitLab.ISSUE_STATE_CLOSED) { + await this.openIssue(issue); + } + + const managedLabelsNames = this.MANAGED_LABELS.map((label) => label.name); + const [managedLabel] = issue.labels.filter((label) => + managedLabelsNames.includes(label.name), + ); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive + + if (managedLabel?.name == label) { + // if the label is already assigned to the issue, the error is redundant with the one already reported and no further action is necessary + return; + } + + const labelsNotManagedToKeep = issue.labels + .map((label) => label.name) + .filter((label) => !managedLabelsNames.includes(label)); + + await this.setIssueLabels({ + issue, + labels: [label, ...labelsNotManagedToKeep], + }); + await this.addCommentToIssue({ issue, comment: description }); + } +} diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js new file mode 100644 index 00000000..d0917607 --- /dev/null +++ b/src/reporterGitlab/index.js @@ -0,0 +1,147 @@ +import mime from 'mime'; + +import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; + +import GitLab from './gitlab.js'; + +const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/'; +const DOC_URL = 'https://docs.opentermsarchive.org'; +const REPO_URL = 'https://gitlab.com/'; + +const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = { + 'has no match': 'selectors', + 'HTTP code 404': 'location', + 'HTTP code 403': '403', + 'HTTP code 429': '429', + 'HTTP code 500': '500', + 'HTTP code 502': '502', + 'HTTP code 503': '503', + 'Timed out after': 'timeout', + 'getaddrinfo EAI_AGAIN': 'EAI_AGAIN', + 'getaddrinfo ENOTFOUND': 'ENOTFOUND', + 'Response is empty': 'empty response', + 'unable to verify the first certificate': 'first certificate', + 'certificate has expired': 'certificate expired', + 'maximum redirect reached': 'redirects', +}; + +function getLabelNameFromError(error) { + return ERROR_MESSAGE_TO_ISSUE_LABEL_MAP[Object.keys(ERROR_MESSAGE_TO_ISSUE_LABEL_MAP).find(substring => error.toString().includes(substring))] || 'to clarify'; +} + +// In the following class, it is assumed that each issue is managed using its title as a unique identifier +export default class Reporter { + constructor(config) { + const { repositories } = config.gitlabIssues; + + for (const repositoryType of Object.keys(repositories)) { + if (!repositories[repositoryType].includes('/') || repositories[repositoryType].includes('https://')) { + throw new Error(`Configuration entry "reporter.gitlabIssues.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); + } + } + + this.gitlab = new GitLab(repositories.declarations); + this.repositories = repositories; + } + + initialize() { + return this.gitlab.initialize(); + } + + async onVersionRecorded(version) { + await this.gitlab.closeIssueWithCommentIfExists({ + title: Reporter.generateTitleID(version.serviceId, version.termsType), + comment: `### Tracking resumed + +A new version has been recorded.`, + }); + } + + async onVersionNotChanged(version) { + await this.gitlab.closeIssueWithCommentIfExists({ + title: Reporter.generateTitleID(version.serviceId, version.termsType), + comment: `### Tracking resumed + +No changes were found in the last run, so no new version has been recorded.`, + }); + } + + async onFirstVersionRecorded(version) { + return this.onVersionRecorded(version); + } + + async onInaccessibleContent(error, terms) { + await this.gitlab.createOrUpdateIssue({ + title: Reporter.generateTitleID(terms.service.id, terms.type), + description: this.generateDescription({ error, terms }), + label: getLabelNameFromError(error), + }); + } + + generateDescription({ error, terms }) { + const date = new Date(); + const currentFormattedDate = date.toLocaleDateString('en-GB', { year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric', timeZoneName: 'short', timeZone: 'UTC' }); + const validUntil = toISODateWithoutMilliseconds(date); + + const hasSnapshots = terms.sourceDocuments.every(sourceDocument => sourceDocument.snapshotId); + + const contributionToolParams = new URLSearchParams({ + json: JSON.stringify(terms.toPersistence()), + destination: this.repositories.declarations, + step: '2', + }); + const contributionToolUrl = `${CONTRIBUTION_TOOL_URL}?${contributionToolParams}`; + + const latestDeclarationLink = `[Latest declaration](${REPO_URL}/${this.repositories.declarations}/-/blob/main/declarations/${encodeURIComponent(terms.service.name)}.json)`; + const latestVersionLink = `[Latest version](${REPO_URL}/${this.repositories.versions}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}.md)`; + const snapshotsBaseUrl = `${REPO_URL}/${this.repositories.snapshots}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}`; + const latestSnapshotsLink = terms.hasMultipleSourceDocuments + ? `Latest snapshots:\n - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n - ')}` + : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`; + + /* eslint-disable no-irregular-whitespace */ + return ` +### No version of the \`${terms.type}\` of service \`${terms.service.name}\` is recorded anymore since ${currentFormattedDate} + +The source document${terms.hasMultipleSourceDocuments ? 's have' : ' has'}${hasSnapshots ? ' ' : ' not '}been recorded in ${terms.hasMultipleSourceDocuments ? 'snapshots' : 'a snapshot'}, ${hasSnapshots ? 'but ' : 'thus '} no version can be [extracted](${DOC_URL}/#tracking-terms). +${hasSnapshots ? 'After correction, it might still be possible to recover the missed versions.' : ''} + +### What went wrong + +- ${error.reasons.join('\n- ')} + +### How to resume tracking + +First of all, check if the source documents are accessible through a web browser: + +- [ ] ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.location}](${sourceDocument.location})`).join('\n- [ ] ')} + +#### If the source documents are accessible through a web browser + +[Edit the declaration](${contributionToolUrl}): +- Try updating the selectors. +- Try switching client scripts on with expert mode. + +#### If the source documents are not accessible anymore + +- If the source documents have moved, find their new location and [update it](${contributionToolUrl}). +- If these terms have been removed, move them from the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. +- If the service has closed, move the entire contents of the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. + +#### If none of the above works + +If the source documents are accessible in a browser but fetching them always fails from the Open Terms Archive server, this is most likely because the service provider has blocked the Open Terms Archive robots from accessing its content. In this case, updating the declaration will not enable resuming tracking. Only an agreement with the service provider, an engine upgrade, or some technical workarounds provided by the administrator of this collection’s server might resume tracking. + +### References + +- ${latestDeclarationLink} +${this.repositories.versions ? `- ${latestVersionLink}` : ''} +${this.repositories.snapshots ? `- ${latestSnapshotsLink}` : ''} +`; + /* eslint-enable no-irregular-whitespace */ + } + + static generateTitleID(serviceId, type) { + return `\`${serviceId}\` ‧ \`${type}\` ‧ not tracked anymore`; + } +} diff --git a/src/reporterGitlab/labels.json b/src/reporterGitlab/labels.json new file mode 100644 index 00000000..8d73f4e8 --- /dev/null +++ b/src/reporterGitlab/labels.json @@ -0,0 +1,77 @@ +[ + { + "name": "403", + "color": "#0b08a0", + "description": "Fetching fails with a 403 (forbidden) HTTP code" + }, + { + "name": "429", + "color": "#0b08a0", + "description": "Fetching fails with a 429 (too many requests) HTTP code" + }, + { + "name": "500", + "color": "#0b08a0", + "description": "Fetching fails with a 500 (internal server error) HTTP code" + }, + { + "name": "502", + "color": "#0b08a0", + "description": "Fetching fails with a 502 (bad gateway) HTTP code" + }, + { + "name": "503", + "color": "#0b08a0", + "description": "Fetching fails with a 503 (service unavailable) HTTP code" + }, + { + "name": "certificate expired", + "color": "#0b08a0", + "description": "Fetching fails because the domain SSL certificate has expired" + }, + { + "name": "EAI_AGAIN", + "color": "#0b08a0", + "description": "Fetching fails because the domain fails to resolve on DNS" + }, + { + "name": "ENOTFOUND", + "color": "#0b08a0", + "description": "Fetching fails because the domain fails to resolve on DNS" + }, + { + "name": "empty response", + "color": "#0b08a0", + "description": "Fetching fails with a “response is empty” error" + }, + { + "name": "first certificate", + "color": "#0b08a0", + "description": "Fetching fails with an “unable to verify the first certificate” error" + }, + { + "name": "redirects", + "color": "#0b08a0", + "description": "Fetching fails with a “too many redirects” error" + }, + { + "name": "timeout", + "color": "#0b08a0", + "description": "Fetching fails with a timeout error" + }, + { + "name": "to clarify", + "color": "#0496ff", + "description": "Default failure label" + }, + { + "name": "selectors", + "color": "#FBCA04", + "description": "Extraction selectors are outdated" + }, + { + "name": "location", + "color": "#FBCA04", + "description": "Fetch location is outdated" + } +] diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js new file mode 100644 index 00000000..0b4e9f75 --- /dev/null +++ b/src/reporterGitlab/labels.test.js @@ -0,0 +1,30 @@ +import { createRequire } from 'module'; + +import chai from 'chai'; + +import { MANAGED_BY_OTA_MARKER } from './gitlab.js'; + +const require = createRequire(import.meta.url); + +const { expect } = chai; +const labels = require('./labels.json'); + +const GITLAB_LABEL_DESCRIPTION_MAX_LENGTH = 255; + +describe('Reporter GitLab labels', () => { + labels.forEach(label => { + describe(`"${label.name}"`, () => { + it('complies with the GitLab character limit for descriptions', () => { + const descriptionLength = label.description.length + MANAGED_BY_OTA_MARKER.length; + + expect(descriptionLength).to.be.lessThan(GITLAB_LABEL_DESCRIPTION_MAX_LENGTH); + }); + + it('complies with the GitHub constraints for color', () => { + const validHexColorRegex = /^\#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` + + expect(validHexColorRegex.test(label.color)).to.be.true; + }); + }); + }); +}); From 8672ba9c9240c695a556d25feb60b896a601f53b Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Mon, 12 Aug 2024 11:14:32 +0200 Subject: [PATCH 02/26] Fix new gitlab files using eslint --- package.json | 4 +- scripts/dataset/index.js | 2 +- scripts/dataset/publishGitLab/index.js | 31 +++--- src/index.js | 1 - src/reporterGitlab/gitlab.js | 131 ++++++++++--------------- src/reporterGitlab/labels.test.js | 2 +- 6 files changed, 70 insertions(+), 101 deletions(-) diff --git a/package.json b/package.json index 8da099a3..85e90b22 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", + "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", @@ -100,8 +101,7 @@ "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "winston": "^3.3.3", - "winston-mail": "^2.0.0", - "axios": "^1.7.2" + "winston-mail": "^2.0.0" }, "devDependencies": { "@commitlint/cli": "^19.0.3", diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index d6b74a3b..735ad2d3 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -41,7 +41,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } releaseDate, stats, }); - + logger.info(`Dataset published to ${releaseUrl}`); } diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 2a1f9bbc..0e4d543c 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -3,19 +3,17 @@ import path from 'path'; import url from 'url'; import axios from 'axios'; - import config from 'config'; import dotenv from 'dotenv'; -//import { Octokit } from 'octokit'; - import FormData from 'form-data'; import * as readme from '../assets/README.templateGitLab.js'; +import logger from '../logger/index.js'; dotenv.config(); -const gitlabAPIUrl = "https://gitlab.com/api/v4"; -const gitlabUrl = "https://gitlab.com"; +const gitlabAPIUrl = 'https://gitlab.com/api/v4'; +const gitlabUrl = 'https://gitlab.com'; export default async function publishReleaseGitLab({ archivePath, @@ -26,25 +24,22 @@ export default async function publishReleaseGitLab({ // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); - const [owner, repo] = url + const [ owner, repo ] = url .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) .pathname.split('/') - .filter((component) => component); + .filter(component => component); const commonParams = { owner, repo }; try { const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; const response = await axios.get( `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, - }, - }, + { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}` } }, ); + projectId = response.data.id; } catch (error) { - //logger.error(`🤖 Error while obtaining projectId: ${error}`); + // logger.error(`🤖 Error while obtaining projectId: ${error}`); projectId = null; } @@ -67,19 +62,19 @@ export default async function publishReleaseGitLab({ }, }, ); - const releaseId = releaseResponse.data.commit.id; + logger.info(`Created release with releaseId: ${releaseId}`); + // Then, upload the ZIP file as an asset to the release const formData = new FormData(); + formData.append('name', archivePath); formData.append( 'url', `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`, ); - formData.append('file', fsApi.createReadStream(archivePath), { - filename: path.basename(archivePath), - }); + formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); const uploadResponse = await axios.post( `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, @@ -96,7 +91,7 @@ export default async function publishReleaseGitLab({ return releaseUrl; } catch (error) { - console.error('Failed to create release or upload ZIP file:', error); + logger.error('Failed to create release or upload ZIP file:', error); throw error; } } diff --git a/src/index.js b/src/index.js index 3f9c4813..517512e0 100644 --- a/src/index.js +++ b/src/index.js @@ -66,7 +66,6 @@ export default async function track({ services, types, extractOnly, schedule }) } catch (error) { logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); } - archivist.attach(reporter); } else { logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); } diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index e688d33e..bf7b8fa5 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -6,7 +6,7 @@ const require = createRequire(import.meta.url); export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; -const gitlabUrl = "https://gitlab.com/api/v4"; +const gitlabUrl = 'https://gitlab.com/api/v4'; export default class GitLab { static ISSUE_STATE_CLOSED = 'closed'; @@ -14,9 +14,9 @@ export default class GitLab { static ISSUE_STATE_ALL = 'all'; constructor(repository) { - //const { version } = require('../../package.json'); + // const { version } = require('../../package.json'); - const [owner, repo] = repository.split('/'); + const [ owner, repo ] = repository.split('/'); this.commonParams = { owner, repo }; } @@ -28,12 +28,9 @@ export default class GitLab { const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; const response = await axios.get( `${gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - }, - }, + { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }, ); + this.projectId = response.data.id; } catch (error) { logger.error(`🤖 Error while obtaining projectId: ${error}`); @@ -42,19 +39,15 @@ export default class GitLab { this.MANAGED_LABELS = require('./labels.json'); const existingLabels = await this.getRepositoryLabels(); - const existingLabelsNames = existingLabels.map((label) => label.name); - const missingLabels = this.MANAGED_LABELS.filter( - (label) => !existingLabelsNames.includes(label.name), - ); + const existingLabelsNames = existingLabels.map(label => label.name); + const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name)); if (missingLabels.length) { - logger.info( - `🤖 Following required labels are not present on the repository: ${missingLabels.map((label) => `"${label.name}"`).join(', ')}. Creating them…`, - ); + logger.info(`🤖 Following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); for (const label of missingLabels) { - await this.createLabel({ - /* eslint-disable-line no-await-in-loop */ name: label.name, + await this.createLabel({ /* eslint-disable-line no-await-in-loop */ + name: label.name, color: label.color, description: `${label.description} ${MANAGED_BY_OTA_MARKER}`, }); @@ -65,23 +58,21 @@ export default class GitLab { async getRepositoryLabels() { try { const response = await fetch( - `${gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + `https://gitlab.com/api/v4/projects/4/labels?with_counts=true`, { method: 'GET', - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - }, + headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` }, }, ); + if (response.status == 200) { const labels = response.json(); + return labels; - } else { - logger.error( - `🤖 Failed to get labels: ${response.status_code} - ${response.text}`, - ); - return null; } + logger.error(`🤖 Failed to get labels: ${response.status_code} - ${response.text}`); + + return null; } catch (error) { logger.error(`🤖 Could get labels: ${error}`); } @@ -92,9 +83,9 @@ export default class GitLab { try { const label = { - name: name, - color: color, - description: description, + name, + color, + description, }; const response = await axios.post( `${gitlabUrl}/projects/${this.projectId}/labels`, @@ -106,6 +97,7 @@ export default class GitLab { }, }, ); + logger.info(`🤖 New label created: ${response.data.name}`); } catch (error) { logger.error(`🤖 Failed to create label: ${error}`); @@ -117,9 +109,9 @@ export default class GitLab { try { const issue = { - title: title, - labels: labels, - description: description, + title, + labels, + description, }; const response = await axios.post( `${gitlabUrl}/projects/${this.projectId}/issues`, @@ -131,9 +123,8 @@ export default class GitLab { }, }, ); - logger.info( - `🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`, - ); + + logger.info(`🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`); return response; } catch (error) { @@ -145,9 +136,7 @@ export default class GitLab { const axios = require('axios'); try { - const newLabels = { - labels: labels, - }; + const newLabels = { labels }; const response = await axios.put( `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, newLabels, @@ -159,11 +148,11 @@ export default class GitLab { }, ); + logger.debug(`response data: ${response.data}`); + logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); } catch (error) { - logger.error( - `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -171,9 +160,7 @@ export default class GitLab { const axios = require('axios'); try { - const updateIssue = { - state_event: 'reopen', - }; + const updateIssue = { state_event: 'reopen' }; const response = await axios.put( `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, updateIssue, @@ -185,11 +172,11 @@ export default class GitLab { }, ); + logger.debug(`response data: ${response.data}`); + logger.info(`🤖 Opened GitLab issue #${issue.iid}`); } catch (error) { - logger.error( - `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -197,9 +184,7 @@ export default class GitLab { const axios = require('axios'); try { - const updateIssue = { - state_event: 'close', - }; + const updateIssue = { state_event: 'close' }; const response = await axios.put( `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, updateIssue, @@ -211,11 +196,11 @@ export default class GitLab { }, ); + logger.debug(`response data: ${response.data}`); + logger.info(`🤖 Closed GitLab issue #${issue.iid}`); } catch (error) { - logger.error( - `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -224,20 +209,16 @@ export default class GitLab { try { let apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; - if (searchParams.state == 'all') - apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + + if (searchParams.state == 'all') apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; - const response = await axios.get(apiUrl, { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - }, - }); + const response = await axios.get(apiUrl, { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }); const issues = response.data; - const [issue] = issues.filter((item) => item.title === title); // since only one is expected, use the first one + const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one setTimeout(() => { - console.log(title + ' - ' + apiUrl); + console.log(`${title} - ${apiUrl}`); }, 5000); return issue; @@ -248,9 +229,7 @@ export default class GitLab { async addCommentToIssue({ issue, comment }) { const axios = require('axios'); - const body = { - body: comment, - }; + const body = { body: comment }; try { const response = await axios.post( @@ -263,15 +242,12 @@ export default class GitLab { }, }, ); - logger.info( - `🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`, - ); + + logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`); return response.data.body; } catch (error) { - logger.error( - `🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -301,10 +277,9 @@ export default class GitLab { await this.openIssue(issue); } - const managedLabelsNames = this.MANAGED_LABELS.map((label) => label.name); - const [managedLabel] = issue.labels.filter((label) => - managedLabelsNames.includes(label.name), - ); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive + const managedLabelsNames = this.MANAGED_LABELS.map(label => label.name); + const [managedLabel] = issue.labels.filter(label => + managedLabelsNames.includes(label.name)); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive if (managedLabel?.name == label) { // if the label is already assigned to the issue, the error is redundant with the one already reported and no further action is necessary @@ -312,12 +287,12 @@ export default class GitLab { } const labelsNotManagedToKeep = issue.labels - .map((label) => label.name) - .filter((label) => !managedLabelsNames.includes(label)); + .map(label => label.name) + .filter(label => !managedLabelsNames.includes(label)); await this.setIssueLabels({ issue, - labels: [label, ...labelsNotManagedToKeep], + labels: [ label, ...labelsNotManagedToKeep ], }); await this.addCommentToIssue({ issue, comment: description }); } diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index 0b4e9f75..3a0c7876 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -21,7 +21,7 @@ describe('Reporter GitLab labels', () => { }); it('complies with the GitHub constraints for color', () => { - const validHexColorRegex = /^\#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` + const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` expect(validHexColorRegex.test(label.color)).to.be.true; }); From 5dbe093229f693429da27d1735fed1a23592f81e Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Fri, 6 Sep 2024 17:57:35 +0200 Subject: [PATCH 03/26] Add tests for GitLab class and format fixes Add test file for GitLab, fix formatting in files, refactor GitLab code to remove axios library --- .env.example | 2 + package-lock.json | 11 - package.json | 1 - scripts/dataset/publishGitLab/index.js | 84 ++-- src/reporterGitlab/gitlab.js | 295 ++++++++------ src/reporterGitlab/gitlab.test.js | 532 +++++++++++++++++++++++++ src/reporterGitlab/index.js | 2 +- src/reporterGitlab/labels.test.js | 2 +- 8 files changed, 768 insertions(+), 161 deletions(-) create mode 100644 src/reporterGitlab/gitlab.test.js diff --git a/.env.example b/.env.example index 08813d69..bae1061c 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,5 @@ OTA_ENGINE_SMTP_PASSWORD='password' OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX +OTA_ENGINE_GITLAB_API_BASE_URL=https://gitlab.com/api/v4 +OTA_ENGINE_GITLAB_BASE_URL=https://gitlab.com \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index dfed6a31..55828f58 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,6 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", - "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", @@ -3980,16 +3979,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/axios": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", - "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/b4a": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", diff --git a/package.json b/package.json index 85e90b22..8fccde58 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", - "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 0e4d543c..c752c1ab 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -2,18 +2,19 @@ import fsApi from 'fs'; import path from 'path'; import url from 'url'; -import axios from 'axios'; import config from 'config'; import dotenv from 'dotenv'; import FormData from 'form-data'; +import nodeFetch from 'node-fetch'; +import GitLab from '../../../src/reporterGitlab/gitlab.js'; import * as readme from '../assets/README.templateGitLab.js'; import logger from '../logger/index.js'; dotenv.config(); -const gitlabAPIUrl = 'https://gitlab.com/api/v4'; -const gitlabUrl = 'https://gitlab.com'; +const gitlabAPIUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; +const gitlabUrl = process.env.OTA_ENGINE_GITLAB_BASE_URL; export default async function publishReleaseGitLab({ archivePath, @@ -22,8 +23,6 @@ export default async function publishReleaseGitLab({ }) { let projectId = null; - // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); - const [ owner, repo ] = url .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) .pathname.split('/') @@ -32,37 +31,53 @@ export default async function publishReleaseGitLab({ try { const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; - const response = await axios.get( + + const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + + options.method = 'GET'; + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}` } }, + options, ); + const res = await response.json(); - projectId = response.data.id; + projectId = res.id; } catch (error) { - // logger.error(`🤖 Error while obtaining projectId: ${error}`); + logger.error(`🤖 Error while obtaining projectId: ${error}`); projectId = null; } const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag try { - // First, create the release - const releaseResponse = await axios.post( + let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + + options.method = 'POST'; + options.body = { + ref: 'main', + tag_name: tagName, + name: readme.title({ releaseDate }), + description: readme.body(stats), + }; + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + options.body = JSON.stringify(options.body); + + const releaseResponse = await nodeFetch( `${gitlabAPIUrl}/projects/${projectId}/releases`, - { - ref: 'main', - tag_name: tagName, - name: readme.title({ releaseDate }), - description: readme.body(stats), - }, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + options, ); - const releaseId = releaseResponse.data.commit.id; + const releaseRes = await releaseResponse.json(); + + const releaseId = releaseRes.commit.id; logger.info(`Created release with releaseId: ${releaseId}`); @@ -76,18 +91,21 @@ export default async function publishReleaseGitLab({ ); formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); - const uploadResponse = await axios.post( + options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + options.method = 'POST'; + options.headers = { + ...formData.getHeaders(), + ...options.headers, + }; + options.body = formData; + + const uploadResponse = await nodeFetch( `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, - formData, - { - headers: { - ...formData.getHeaders(), - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, - }, - }, + options, ); + const uploadRes = await uploadResponse.json(); - const releaseUrl = uploadResponse.data.direct_asset_url; + const releaseUrl = uploadRes.direct_asset_url; return releaseUrl; } catch (error) { diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index bf7b8fa5..392b5b48 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -1,37 +1,48 @@ import { createRequire } from 'module'; +import HttpProxyAgent from 'http-proxy-agent'; +import HttpsProxyAgent from 'https-proxy-agent'; +import nodeFetch from 'node-fetch'; + import logger from '../logger/index.js'; const require = createRequire(import.meta.url); export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; -const gitlabUrl = 'https://gitlab.com/api/v4'; - export default class GitLab { static ISSUE_STATE_CLOSED = 'closed'; static ISSUE_STATE_OPEN = 'opened'; static ISSUE_STATE_ALL = 'all'; constructor(repository) { - // const { version } = require('../../package.json'); - const [ owner, repo ] = repository.split('/'); this.commonParams = { owner, repo }; + this.projectId = null; + const gitlabUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; + + this.gitlabUrl = gitlabUrl; } async initialize() { - const axios = require('axios'); + const options = GitLab.baseOptionsHttpReq(); try { const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; - const response = await axios.get( - `${gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, + options, ); - this.projectId = response.data.id; + const res = await response.json(); + + if (response.ok) { + this.projectId = res.id; + } else { + logger.error(`🤖 Error while obtaining projectId: ${JSON.strinfigy(res)}`); + this.projectId = null; + } } catch (error) { logger.error(`🤖 Error while obtaining projectId: ${error}`); this.projectId = null; @@ -57,20 +68,18 @@ export default class GitLab { async getRepositoryLabels() { try { - const response = await fetch( - `https://gitlab.com/api/v4/projects/4/labels?with_counts=true`, - { - method: 'GET', - headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` }, - }, + const options = GitLab.baseOptionsHttpReq(); + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + options, ); + const res = await response.json(); - if (response.status == 200) { - const labels = response.json(); - - return labels; + if (response.ok) { + return res; } - logger.error(`🤖 Failed to get labels: ${response.status_code} - ${response.text}`); + + logger.error(`🤖 Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); return null; } catch (error) { @@ -79,191 +88,235 @@ export default class GitLab { } async createLabel({ name, color, description }) { - const axios = require('axios'); - try { const label = { name, color, description, }; - const response = await axios.post( - `${gitlabUrl}/projects/${this.projectId}/labels`, - label, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(label); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/labels`, + options, ); - logger.info(`🤖 New label created: ${response.data.name}`); + const res = await response.json(); + + if (response.ok) { + logger.info(`🤖 New label created: ${res.name} , color: ${res.color}`); + } else { + logger.error(`createLabel response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Failed to create label: ${error}`); } } async createIssue({ title, description, labels }) { - const axios = require('axios'); - try { const issue = { title, labels, description, }; - const response = await axios.post( - `${gitlabUrl}/projects/${this.projectId}/issues`, - issue, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(issue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues`, + options, ); - logger.info(`🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`); + const res = await response.json(); + + if (response.ok) { + logger.info(`🤖 Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); + + return res; + } - return response; + logger.error(`createIssue response: ${JSON.stringify(res)}`); } catch (error) { logger.error(`🤖 Could not create GitLab issue "${title}": ${error}`); } } async setIssueLabels({ issue, labels }) { - const axios = require('axios'); + const newLabels = { labels }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(newLabels); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; try { - const newLabels = { labels }; - const response = await axios.put( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, - newLabels, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, ); - logger.debug(`response data: ${response.data}`); + const res = await response.json(); - logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + if (response.ok) { + logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + } else { + logger.error(`setIssueLabels response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async openIssue(issue) { - const axios = require('axios'); + const updateIssue = { state_event: 'reopen' }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(updateIssue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; try { - const updateIssue = { state_event: 'reopen' }; - const response = await axios.put( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, - updateIssue, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, ); + const res = await response.json(); - logger.debug(`response data: ${response.data}`); - - logger.info(`🤖 Opened GitLab issue #${issue.iid}`); + if (response.ok) { + logger.info(`🤖 Opened GitLab issue #${res.iid}`); + } else { + logger.error(`openIssue response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async closeIssue(issue) { - const axios = require('axios'); + const updateIssue = { state_event: 'close' }; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(updateIssue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; try { - const updateIssue = { state_event: 'close' }; - const response = await axios.put( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, - updateIssue, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, ); + const res = await response.json(); - logger.debug(`response data: ${response.data}`); - - logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + if (response.ok) { + logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + } else { + logger.error(`closeIssue response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async getIssue({ title, ...searchParams }) { - const axios = require('axios'); - try { - let apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + let apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + + if (searchParams.state == 'all') apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'GET'; + + const response = await nodeFetch(apiUrl, options); + const res = await response.json(); - if (searchParams.state == 'all') apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; - apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; - const response = await axios.get(apiUrl, { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }); - const issues = response.data; + if (response.ok) { + logger.debug(`response data: ${JSON.stringify(res)}`); + const issues = res; - const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one + const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one - setTimeout(() => { - console.log(`${title} - ${apiUrl}`); - }, 5000); + return issue; + } - return issue; + logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { logger.error(`🤖 Could not find GitLab issue "${title}": ${error}`); } } async addCommentToIssue({ issue, comment }) { - const axios = require('axios'); const body = { body: comment }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(body); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + try { - const response = await axios.post( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, - body, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, + options, ); + const res = await response.json(); - logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`); + if (response.ok) { + logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); - return response.data.body; + return res.body; + } + + logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { logger.error(`🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async closeIssueWithCommentIfExists({ title, comment }) { - const openedIssue = await this.getIssue({ + const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_OPEN, }); - if (!openedIssue) { + // if issue does not exist in the "opened" state + if (!issue) { return; } - await this.addCommentToIssue({ issue: openedIssue, comment }); + await this.addCommentToIssue({ issue, comment }); - return this.closeIssue(openedIssue); + return this.closeIssue(issue); } async createOrUpdateIssue({ title, description, label }) { @@ -296,4 +349,18 @@ export default class GitLab { }); await this.addCommentToIssue({ issue, comment: description }); } + + static baseOptionsHttpReq(token = process.env.OTA_ENGINE_GITLAB_TOKEN) { + const options = {}; + + if (process.env.HTTPS_PROXY) { + options.agent = new HttpsProxyAgent(process.env.HTTPS_PROXY); + } else if (process.env.HTTP_PROXY) { + options.agent = new HttpProxyAgent(process.env.HTTP_PROXY); + } + + options.headers = { Authorization: `Bearer ${token}` }; + + return options; + } } diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js new file mode 100644 index 00000000..ede97cd1 --- /dev/null +++ b/src/reporterGitlab/gitlab.test.js @@ -0,0 +1,532 @@ +import { createRequire } from 'module'; + +import { expect } from 'chai'; +import nock from 'nock'; + +import GitLab from './gitlab.js'; + +const require = createRequire(import.meta.url); + +describe('GitLab', function () { + this.timeout(5000); + + let MANAGED_LABELS; + let gitlab; + let gitlabApiUrl = ''; + let reqHeaders; + const projectId = '4'; + + before(() => { + MANAGED_LABELS = require('./labels.json'); + gitlab = new GitLab('owner/repo'); + gitlab.projectId = projectId; + gitlabApiUrl = gitlab.gitlabUrl; + reqHeaders = { reqheaders: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }; + }); + + describe('#Gitlab_initialize', () => { + const scopes = []; + + before(async () => { + const existingLabels = MANAGED_LABELS.slice(0, -2); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${encodeURIComponent('owner/repo')}`) + .reply(200, { id: 4 }); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, existingLabels); + + const missingLabels = MANAGED_LABELS.slice(-2); + + for (const label of missingLabels) { + scopes.push(nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/labels`) + .reply(200, { name: label.name })); + } + + await gitlab.initialize(); + }); + + after(nock.cleanAll); + + it('should create missing labels', () => { + scopes.forEach(scope => expect(scope.isDone()).to.be.true); + }); + }); + + describe('#Gitlab_getRepositoryLabels', () => { + let scope; + let result; + const LABELS = [{ name: 'bug' }, { name: 'enhancement' }]; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, LABELS); + + result = await gitlab.getRepositoryLabels(); + }); + + after(nock.cleanAll); + + it('fetches repository labels', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the repository labels', () => { + expect(result).to.deep.equal(LABELS); + }); + }); + + describe('#Gitlab_createLabel', () => { + let scope; + const LABEL = { name: 'new_label', color: 'ffffff' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/labels`, body => body.name === LABEL.name) + .reply(200, LABEL); + + await gitlab.createLabel(LABEL); + }); + + after(nock.cleanAll); + + it('creates the new label', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_createIssue', () => { + let scope; + let result; + + const ISSUE = { + title: 'New Issue', + description: 'Description of the new issue', + labels: ['bug'], + }; + const CREATED_ISSUE = { + title: 'New Issue', + description: 'Description of the new issue', + labels: ['bug'], + iid: 555, + web_url: 'https://example.com/test/test', + }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues`) + .reply(200, CREATED_ISSUE); + + result = await gitlab.createIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('creates the new issue', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the created issue', () => { + expect(result).to.deep.equal(CREATED_ISSUE); + }); + }); + + describe('#Gitlab_setIssueLabels', () => { + let scope; + const issue = { + iid: 123, + title: 'test issue', + }; + const labels = [ 'bug', 'enhancement' ]; + + const response = { + iid: 123, + labels, + }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${issue.iid}`, { labels }) + .reply(200, response); + + await gitlab.setIssueLabels({ issue, labels }); + }); + + after(nock.cleanAll); + + it('sets labels on the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_openIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'issue reopened' }; + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const response = { iid: 123 }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + .reply(200, response); + + await gitlab.openIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('opens the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_closeIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'close issue' }; + const EXPECTED_REQUEST_BODY = { state_event: 'close' }; + const response = { iid: 123 }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + .reply(200, response); + + await gitlab.closeIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('closes the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_getIssue', () => { + let scope; + let result; + + const ISSUE = { number: 123, title: 'Test Issue' }; + const ANOTHER_ISSUE = { number: 124, title: 'Test Issue 2' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [ ISSUE, ANOTHER_ISSUE ]); + + result = await gitlab.getIssue({ title: ISSUE.title }); + }); + + after(nock.cleanAll); + + it('searches for the issue', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the expected issue', () => { + expect(result).to.deep.equal(ISSUE); + }); + }); + + describe('#Gitlab_addCommentToIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'Test Issue' }; + const COMMENT = 'Test comment'; + const response = { iid: 123, id: 23, body: 'Test comment' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, response); + + await gitlab.addCommentToIssue({ issue: ISSUE, comment: COMMENT }); + }); + + after(nock.cleanAll); + + it('adds the comment to the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_closeIssueWithCommentIfExists', () => { + after(nock.cleanAll); + + context('when the issue exists and is open', () => { + const ISSUE = { + iid: 123, + title: 'Open Issue', + state: GitLab.ISSUE_STATE_OPEN, + }; + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [ISSUE]); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + + it('closes the issue', () => { + expect(closeIssueScope.isDone()).to.be.true; + }); + }); + + context('when the issue exists and is closed', () => { + const ISSUE = { + number: 123, + title: 'Closed Issue', + state: GitLab.ISSUE_STATE_CLOSED, + }; + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, []); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); + }); + + it('does not add comment', () => { + expect(addCommentScope.isDone()).to.be.false; + }); + + it('does not attempt to close the issue', () => { + expect(closeIssueScope.isDone()).to.be.false; + }); + }); + + context('when the issue does not exist', () => { + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const TITLE = 'Non-existent Issue'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(TITLE)}&per_page=100`) + .reply(200, []); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(/\/projects\/\d+\/issues\/\d+\/notes/, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(/\/projects\/\d+\/issues\/\d+/, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: TITLE, comment: COMMENT }); + }); + + it('does not attempt to add comment', () => { + expect(addCommentScope.isDone()).to.be.false; + }); + + it('does not attempt to close the issue', () => { + expect(closeIssueScope.isDone()).to.be.false; + }); + }); + }); + + describe('#Gitlab_createOrUpdateIssue', () => { + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${encodeURIComponent('owner/repo')}`) + .reply(200, { id: 4 }); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, MANAGED_LABELS); + + await gitlab.initialize(); + }); + + context('when the issue does not exist', () => { + let createIssueScope; + const ISSUE_TO_CREATE = { + title: 'New Issue', + description: 'Description of the new issue', + label: 'bug', + }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE_TO_CREATE.title)}&per_page=100`) + .reply(200, []); // Simulate that there is no issues on the repository + + createIssueScope = nock(gitlabApiUrl, reqHeaders) + .post( + `/projects/${projectId}/issues`, + { + title: ISSUE_TO_CREATE.title, + description: ISSUE_TO_CREATE.description, + labels: [ISSUE_TO_CREATE.label], + }, + ) + .reply(200, { iid: 123, web_url: 'https://example.com/test/test' }); + + await gitlab.createOrUpdateIssue(ISSUE_TO_CREATE); + }); + + it('creates the issue', () => { + expect(createIssueScope.isDone()).to.be.true; + }); + }); + + context('when the issue already exists', () => { + const ISSUE = { + title: 'Existing Issue', + description: 'New comment', + label: 'location', + }; + + context('when issue is closed', () => { + let setIssueLabelsScope; + let addCommentScope; + let openIssueScope; + + const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = { + iid: 123, + title: ISSUE.title, + description: ISSUE.description, + labels: [{ name: 'selectors' }], + state: GitLab.ISSUE_STATE_CLOSED, + }; + + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const responseIssuereopened = { iid: 123 }; + const responseSetLabels = { + iid: 123, + labels: ['location'], + }; + const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description }; + const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); + + openIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + .reply(200, responseIssuereopened); + + setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + .reply(200, responseSetLabels); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + .reply(200, responseAddcomment); + + await gitlab.createOrUpdateIssue(ISSUE); + }); + + it('reopens the issue', () => { + expect(openIssueScope.isDone()).to.be.true; + }); + + it("updates the issue's label", () => { + expect(setIssueLabelsScope.isDone()).to.be.true; + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + }); + + context('when issue is already opened', () => { + let setIssueLabelsScope; + let addCommentScope; + let openIssueScope; + + const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = { + number: 123, + title: ISSUE.title, + description: ISSUE.description, + labels: [{ name: 'selectors' }], + state: GitLab.ISSUE_STATE_OPEN, + }; + + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const responseIssuereopened = { iid: 123 }; + const responseSetLabels = { + iid: 123, + labels: ['location'], + }; + const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description }; + const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); + + openIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + .reply(200, responseIssuereopened); + + setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + .reply(200, responseSetLabels); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + .reply(200, responseAddcomment); + + await gitlab.createOrUpdateIssue(ISSUE); + }); + + it('does not change the issue state', () => { + expect(openIssueScope.isDone()).to.be.false; + }); + + it("updates the issue's label", () => { + expect(setIssueLabelsScope.isDone()).to.be.true; + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + }); + }); + }); +}); diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js index d0917607..be516118 100644 --- a/src/reporterGitlab/index.js +++ b/src/reporterGitlab/index.js @@ -6,7 +6,7 @@ import GitLab from './gitlab.js'; const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/'; const DOC_URL = 'https://docs.opentermsarchive.org'; -const REPO_URL = 'https://gitlab.com/'; +const REPO_URL = process.env.OTA_ENGINE_GITLAB_BASE_URL; const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = { 'has no match': 'selectors', diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index 3a0c7876..a1e61f2d 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -20,7 +20,7 @@ describe('Reporter GitLab labels', () => { expect(descriptionLength).to.be.lessThan(GITLAB_LABEL_DESCRIPTION_MAX_LENGTH); }); - it('complies with the GitHub constraints for color', () => { + it('complies with the GitLab constraints for color', () => { const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` expect(validHexColorRegex.test(label.color)).to.be.true; From b67e3614d747696c75ad7a652b960db79a446358 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Thu, 12 Sep 2024 11:15:13 +0200 Subject: [PATCH 04/26] Remove existing GitLab repository in the config Remove the GitLab releases repository with a new one used as an example --- config/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.json b/config/default.json index 9009d52a..8102f222 100644 --- a/config/default.json +++ b/config/default.json @@ -59,7 +59,7 @@ "dataset": { "title": "sandbox", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox", - "versionsRepositoryURLGitLab": "https://gitlab.com/p2b/contrib-versions", + "versionsRepositoryURLGitLab": "https://gitlab.com/ota-sandbox-example/sandbox", "publishingSchedule": "30 8 * * MON" } } From 46567ed3ee3d764d81a988fed20d06c6f249df67 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 8 Oct 2024 14:33:44 +0200 Subject: [PATCH 05/26] Add EUPL-1.2 copyright --- scripts/dataset/assets/README.templateGitLab.js | 16 ++++++++++++++++ scripts/dataset/publishGitLab/index.js | 16 ++++++++++++++++ src/reporterGitlab/gitlab.js | 16 ++++++++++++++++ src/reporterGitlab/gitlab.test.js | 16 ++++++++++++++++ src/reporterGitlab/index.js | 16 ++++++++++++++++ src/reporterGitlab/labels.test.js | 16 ++++++++++++++++ 6 files changed, 96 insertions(+) diff --git a/scripts/dataset/assets/README.templateGitLab.js b/scripts/dataset/assets/README.templateGitLab.js index c9f0bb8d..ef8361e6 100644 --- a/scripts/dataset/assets/README.templateGitLab.js +++ b/scripts/dataset/assets/README.templateGitLab.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import config from 'config'; const LOCALE = 'en-EN'; diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index c752c1ab..2f2f7f9b 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import fsApi from 'fs'; import path from 'path'; import url from 'url'; diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index 392b5b48..68a63810 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import { createRequire } from 'module'; import HttpProxyAgent from 'http-proxy-agent'; diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js index ede97cd1..093384ce 100644 --- a/src/reporterGitlab/gitlab.test.js +++ b/src/reporterGitlab/gitlab.test.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import { createRequire } from 'module'; import { expect } from 'chai'; diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js index be516118..923d0011 100644 --- a/src/reporterGitlab/index.js +++ b/src/reporterGitlab/index.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import mime from 'mime'; import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index a1e61f2d..dd6e8fe5 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import { createRequire } from 'module'; import chai from 'chai'; From 3a64bf0811de41c96fd0c5ca7577780198ba01f6 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Wed, 9 Oct 2024 17:22:41 +0200 Subject: [PATCH 06/26] Use only a single template for README Remove duplicate readme for gitlab and handle the new input parameter --- scripts/dataset/assets/README.template.js | 8 +- .../dataset/assets/README.templateGitLab.js | 81 ------------------- scripts/dataset/export/index.js | 3 +- scripts/dataset/index.js | 16 +++- scripts/dataset/publishGitLab/index.js | 2 +- 5 files changed, 19 insertions(+), 91 deletions(-) delete mode 100644 scripts/dataset/assets/README.templateGitLab.js diff --git a/scripts/dataset/assets/README.template.js b/scripts/dataset/assets/README.template.js index 1c63d117..1cda3428 100644 --- a/scripts/dataset/assets/README.template.js +++ b/scripts/dataset/assets/README.template.js @@ -3,10 +3,10 @@ import config from 'config'; const LOCALE = 'en-EN'; const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; -export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { +export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { return `# Open Terms Archive — ${title({ releaseDate })} -${body({ servicesCount, firstVersionDate, lastVersionDate })}`; +${body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL })}`; } export function title({ releaseDate }) { @@ -17,12 +17,10 @@ export function title({ releaseDate }) { return `${title} — ${releaseDate} dataset`; } -export function body({ servicesCount, firstVersionDate, lastVersionDate }) { +export function body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - const versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); - return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURL}](${versionsRepositoryURL}). diff --git a/scripts/dataset/assets/README.templateGitLab.js b/scripts/dataset/assets/README.templateGitLab.js deleted file mode 100644 index ef8361e6..00000000 --- a/scripts/dataset/assets/README.templateGitLab.js +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - -import config from 'config'; - -const LOCALE = 'en-EN'; -const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; - -export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { - return `# Open Terms Archive — ${title({ releaseDate })} - -${body({ servicesCount, firstVersionDate, lastVersionDate })}`; -} - -export function title({ releaseDate }) { - releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - - const title = config.get('@opentermsarchive/engine.dataset.title'); - - return `${title} — ${releaseDate} dataset`; -} - -export function body({ servicesCount, firstVersionDate, lastVersionDate }) { - firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - - const versionsRepositoryURLGitLab = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); - - return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. - -This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURLGitLab}](${versionsRepositoryURLGitLab}). - -It has been generated with [Open Terms Archive](https://opentermsarchive.org). - -### Dataset format - -This dataset represents each version of a document as a separate [Markdown](https://spec.commonmark.org/0.30/) file, nested in a directory with the name of the service provider and in a directory with the name of the terms type. The filesystem layout will look like below. - -\`\`\` -├ README.md -├┬ Service provider 1 (e.g. Facebook) -│├┬ Terms type 1 (e.g. Terms of Service) -││├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-08-01T01-03-12Z.md) -┆┆┆ -││└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-10-03T08-12-25Z.md) -┆┆ -│└┬ Terms type X (e.g. Privacy Policy) -│ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) -┆ ┆ -│ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) -┆ -└┬ Service provider Y (e.g. Google) - ├┬ Terms type 1 (e.g. Developer Terms) - │├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2019-03-12T04-18-22Z.md) - ┆┆ - │└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-12-04T22-47-05Z.md) - └┬ Terms type Z (e.g. Privacy Policy) - ┆ - ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) - ┆ - └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) -\`\`\` - -### License - -This dataset is made available under an [Open Database (OdBL) License](https://opendatacommons.org/licenses/odbl/1.0/) by Open Terms Archive Contributors. -`; -} diff --git a/scripts/dataset/export/index.js b/scripts/dataset/export/index.js index bc0a028a..3215768c 100644 --- a/scripts/dataset/export/index.js +++ b/scripts/dataset/export/index.js @@ -16,7 +16,7 @@ const fs = fsApi.promises; const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats -export default async function generate({ archivePath, releaseDate }) { +export default async function generate({ archivePath, releaseDate, versionsRepositoryURL }) { const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize(); const archive = await initializeArchive(archivePath); @@ -61,6 +61,7 @@ export default async function generate({ archivePath, releaseDate }) { releaseDate, firstVersionDate, lastVersionDate, + versionsRepositoryURL, }), { name: `${archive.basename}/README.md` }, ); diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 735ad2d3..8041813c 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -15,7 +15,17 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start exporting dataset…'); - const stats = await generateRelease({ archivePath, releaseDate }); + const usesGitHub = (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined'); + const usesGitLab = (typeof process.env.OTA_ENGINE_GITLAB_TOKEN !== 'undefined'); + + let versionsRepositoryURL = ''; + + if (usesGitHub) + versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); + if (usesGitLab) + versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + const stats = await generateRelease({ archivePath, releaseDate, versionsRepositoryURL }); logger.info(`Dataset exported in ${archivePath}`); @@ -25,7 +35,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start publishing dataset…'); - if (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined') { + if (usesGitHub) { const releaseUrl = await publishRelease({ archivePath, releaseDate, @@ -35,7 +45,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info(`Dataset published to ${releaseUrl}`); } - if (typeof process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN !== 'undefined') { + if (usesGitLab) { const releaseUrl = await publishReleaseGitLab({ archivePath, releaseDate, diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 2f2f7f9b..e2a46b65 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -24,7 +24,7 @@ import FormData from 'form-data'; import nodeFetch from 'node-fetch'; import GitLab from '../../../src/reporterGitlab/gitlab.js'; -import * as readme from '../assets/README.templateGitLab.js'; +import * as readme from '../assets/README.template.js'; import logger from '../logger/index.js'; dotenv.config(); From d46c68c47ca8e2073914a715febe2660d3376e8a Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Mon, 21 Oct 2024 09:35:12 +0200 Subject: [PATCH 07/26] Fix release zip upload not working Fix the release zip workflow using GitLab packages to upload the file --- scripts/dataset/index.js | 8 +++--- scripts/dataset/publishGitLab/index.js | 34 ++++++++++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 8041813c..66ea0a1f 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -19,11 +19,9 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } const usesGitLab = (typeof process.env.OTA_ENGINE_GITLAB_TOKEN !== 'undefined'); let versionsRepositoryURL = ''; - - if (usesGitHub) - versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); - if (usesGitLab) - versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + if (usesGitHub) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); + if (usesGitLab) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); const stats = await generateRelease({ archivePath, releaseDate, versionsRepositoryURL }); diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index e2a46b65..b8614b07 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -30,7 +30,6 @@ import logger from '../logger/index.js'; dotenv.config(); const gitlabAPIUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; -const gitlabUrl = process.env.OTA_ENGINE_GITLAB_BASE_URL; export default async function publishReleaseGitLab({ archivePath, @@ -97,14 +96,36 @@ export default async function publishReleaseGitLab({ logger.info(`Created release with releaseId: ${releaseId}`); - // Then, upload the ZIP file as an asset to the release + // Upload the package + options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + options.method = 'PUT'; + options.body = fsApi.createReadStream(archivePath); + + // restrict characters to the ones allowed by GitLab APIs + const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-'); + const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-'); + const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-'); + + logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`); + + const packageResponse = await nodeFetch( + `${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`, + options, + ); + const packageRes = await packageResponse.json(); + + const packageFilesId = packageRes.id; + + logger.debug(`package file id: ${packageFilesId}`); + + // use the package id to build the download url for the release + const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')}/-/package_files/${packageFilesId}/download`; + + // Create the release and link the package const formData = new FormData(); formData.append('name', archivePath); - formData.append( - 'url', - `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`, - ); + formData.append('url', publishedPackageUrl); formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); @@ -120,7 +141,6 @@ export default async function publishReleaseGitLab({ options, ); const uploadRes = await uploadResponse.json(); - const releaseUrl = uploadRes.direct_asset_url; return releaseUrl; From e423dd3c7657b8cfca0d8e321bae4b121af83fb8 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Mon, 21 Oct 2024 10:07:23 +0200 Subject: [PATCH 08/26] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 242c9466..919be8c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [minor] + +### Added + +- Add GtiLab functionalities + + ## 2.2.1 - 2024-06-07 _Full changeset and discussions: [#1088](https://github.com/OpenTermsArchive/engine/pull/1088)._ From 3b964ca5760f2e6571fe9b1259c5578b452af0b2 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 22 Oct 2024 09:33:51 +0200 Subject: [PATCH 09/26] Fix typo on changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e39d24b9..99740d25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All changes that impact users of this module are documented in this file, in the ### Added -- Add GtiLab functionalities +- Add GitLab functionalities ## 2.3.0 - 2024-10-21 From 07aac6f5af953080ca1bafd6961efa2d91112e2e Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 29 Oct 2024 11:12:43 +0100 Subject: [PATCH 10/26] Remove explicit EUPL-1.2 copyright in files --- scripts/dataset/publishGitLab/index.js | 16 ---------------- src/reporterGitlab/gitlab.js | 16 ---------------- src/reporterGitlab/gitlab.test.js | 16 ---------------- src/reporterGitlab/index.js | 16 ---------------- src/reporterGitlab/labels.test.js | 16 ---------------- 5 files changed, 80 deletions(-) diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index b8614b07..16f08bfc 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import fsApi from 'fs'; import path from 'path'; import url from 'url'; diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index 68a63810..392b5b48 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import { createRequire } from 'module'; import HttpProxyAgent from 'http-proxy-agent'; diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js index 093384ce..ede97cd1 100644 --- a/src/reporterGitlab/gitlab.test.js +++ b/src/reporterGitlab/gitlab.test.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import { createRequire } from 'module'; import { expect } from 'chai'; diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js index 923d0011..be516118 100644 --- a/src/reporterGitlab/index.js +++ b/src/reporterGitlab/index.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import mime from 'mime'; import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index dd6e8fe5..a1e61f2d 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import { createRequire } from 'module'; import chai from 'chai'; From bbf5e58ae1eb5a1bc84391c33a042fda43bc26cf Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 29 Oct 2024 11:34:12 +0100 Subject: [PATCH 11/26] Align logging messages format with latest releases Aligned format and minor wording fixes --- scripts/dataset/publishGitLab/index.js | 2 +- src/reporterGitlab/gitlab.js | 36 +++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 16f08bfc..e41c3e1c 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -47,7 +47,7 @@ export default async function publishReleaseGitLab({ projectId = res.id; } catch (error) { - logger.error(`🤖 Error while obtaining projectId: ${error}`); + logger.error(`Error while obtaining projectId: ${error}`); projectId = null; } diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index 392b5b48..cf4133a9 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -40,11 +40,11 @@ export default class GitLab { if (response.ok) { this.projectId = res.id; } else { - logger.error(`🤖 Error while obtaining projectId: ${JSON.strinfigy(res)}`); + logger.error(`Error while obtaining projectId: ${JSON.strinfigy(res)}`); this.projectId = null; } } catch (error) { - logger.error(`🤖 Error while obtaining projectId: ${error}`); + logger.error(`Error while obtaining projectId: ${error}`); this.projectId = null; } this.MANAGED_LABELS = require('./labels.json'); @@ -54,7 +54,7 @@ export default class GitLab { const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name)); if (missingLabels.length) { - logger.info(`🤖 Following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); + logger.info(`The following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); for (const label of missingLabels) { await this.createLabel({ /* eslint-disable-line no-await-in-loop */ @@ -79,11 +79,11 @@ export default class GitLab { return res; } - logger.error(`🤖 Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); + logger.error(`Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); return null; } catch (error) { - logger.error(`🤖 Could get labels: ${error}`); + logger.error(`Could not get labels: ${error}`); } } @@ -112,12 +112,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 New label created: ${res.name} , color: ${res.color}`); + logger.info(`New label created: ${res.name} , color: ${res.color}`); } else { logger.error(`createLabel response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Failed to create label: ${error}`); + logger.error(`Failed to create label: ${error}`); } } @@ -146,14 +146,14 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); + logger.info(`Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); return res; } logger.error(`createIssue response: ${JSON.stringify(res)}`); } catch (error) { - logger.error(`🤖 Could not create GitLab issue "${title}": ${error}`); + logger.error(`Could not create GitLab issue "${title}": ${error}`); } } @@ -177,12 +177,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + logger.info(`Updated labels to GitLab issue #${issue.iid}`); } else { logger.error(`setIssueLabels response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -205,12 +205,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Opened GitLab issue #${res.iid}`); + logger.info(`Opened GitLab issue #${res.iid}`); } else { logger.error(`openIssue response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -234,12 +234,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + logger.info(`Closed GitLab issue #${issue.iid}`); } else { logger.error(`closeIssue response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -268,7 +268,7 @@ export default class GitLab { logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { - logger.error(`🤖 Could not find GitLab issue "${title}": ${error}`); + logger.error(`Could not find GitLab issue "${title}": ${error}`); } } @@ -292,14 +292,14 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); + logger.info(`Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); return res.body; } logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { - logger.error(`🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } From 3b9251796c826c9b83f625fe37fa7dc4e84fb67a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 5 Nov 2024 11:25:58 +0100 Subject: [PATCH 12/26] Factorize code between Gitlab and GitHub reporters --- .env.example | 2 - src/index.js | 26 +-- src/reporter/factory.js | 13 ++ src/reporter/{github.js => github/index.js} | 16 +- .../{github.test.js => github/index.test.js} | 2 +- src/reporter/{ => github}/labels.json | 0 src/reporter/{ => github}/labels.test.js | 2 +- .../gitlab.js => reporter/gitlab/index.js} | 48 ++++-- .../gitlab/index.test.js} | 153 +++++++++--------- .../gitlab}/labels.json | 0 .../gitlab}/labels.test.js | 4 +- src/reporter/index.js | 24 +-- src/reporterGitlab/index.js | 147 ----------------- 13 files changed, 153 insertions(+), 284 deletions(-) create mode 100644 src/reporter/factory.js rename src/reporter/{github.js => github/index.js} (89%) rename src/reporter/{github.test.js => github/index.test.js} (99%) rename src/reporter/{ => github}/labels.json (100%) rename src/reporter/{ => github}/labels.test.js (94%) rename src/{reporterGitlab/gitlab.js => reporter/gitlab/index.js} (83%) rename src/{reporterGitlab/gitlab.test.js => reporter/gitlab/index.test.js} (70%) rename src/{reporterGitlab => reporter/gitlab}/labels.json (100%) rename src/{reporterGitlab => reporter/gitlab}/labels.test.js (81%) delete mode 100644 src/reporterGitlab/index.js diff --git a/.env.example b/.env.example index bae1061c..08813d69 100644 --- a/.env.example +++ b/.env.example @@ -3,5 +3,3 @@ OTA_ENGINE_SMTP_PASSWORD='password' OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX -OTA_ENGINE_GITLAB_API_BASE_URL=https://gitlab.com/api/v4 -OTA_ENGINE_GITLAB_BASE_URL=https://gitlab.com \ No newline at end of file diff --git a/src/index.js b/src/index.js index 517512e0..142c63ef 100644 --- a/src/index.js +++ b/src/index.js @@ -8,7 +8,6 @@ import Archivist from './archivist/index.js'; import logger from './logger/index.js'; import Notifier from './notifier/index.js'; import Reporter from './reporter/index.js'; -import ReporterGitlab from './reporterGitlab/index.js'; const require = createRequire(import.meta.url); @@ -56,8 +55,8 @@ export default async function track({ services, types, extractOnly, schedule }) logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored'); } - if (process.env.OTA_ENGINE_GITHUB_TOKEN) { - if (config.has('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations')) { + if (process.env.OTA_ENGINE_GITHUB_TOKEN || process.env.OTA_ENGINE_GITLAB_TOKEN) { + if (config.has('@opentermsarchive/engine.reporter.repositories.declarations')) { try { const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter')); @@ -67,27 +66,10 @@ export default async function track({ services, types, extractOnly, schedule }) logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); } } else { - logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + logger.warn('Configuration key "reporter.repositories.declarations" was not found; issues on the declarations repository cannot be created'); } } else { - logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored'); - } - - if (process.env.OTA_ENGINE_GITLAB_TOKEN) { - if (config.has('@opentermsarchive/engine.reporter.gitlabIssues.repositories.declarations')) { - try { - const reporter = new ReporterGitlab(config.get('@opentermsarchive/engine.reporter')); - - await reporter.initialize(); - archivist.attach(reporter); - } catch (error) { - logger.error('Cannot instantiate the ReporterGitlab module; it will be ignored:', error); - } - } else { - logger.warn('Configuration key "reporter.gitlabIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); - } - } else { - logger.warn('Environment variable "OTA_ENGINE_GITLAB_TOKEN" was not found; the ReporterGitlab module will be ignored'); + logger.warn('Environment variable with token for GitHub or GitLab was not found; the Reporter module will be ignored'); } if (!schedule) { diff --git a/src/reporter/factory.js b/src/reporter/factory.js new file mode 100644 index 00000000..f8619277 --- /dev/null +++ b/src/reporter/factory.js @@ -0,0 +1,13 @@ +import GitHub from './github/index.js'; +import GitLab from './gitlab/index.js'; + +export function createReporter(config) { + switch (config.type) { + case 'github': + return new GitHub(config.repositories.declarations); + case 'gitlab': + return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL); + default: + throw new Error(`Unsupported reporter type: ${config.type}`); + } +} diff --git a/src/reporter/github.js b/src/reporter/github/index.js similarity index 89% rename from src/reporter/github.js rename to src/reporter/github/index.js index 535d008a..28d262de 100644 --- a/src/reporter/github.js +++ b/src/reporter/github/index.js @@ -2,7 +2,7 @@ import { createRequire } from 'module'; import { Octokit } from 'octokit'; -import logger from '../logger/index.js'; +import logger from '../../logger/index.js'; const require = createRequire(import.meta.url); @@ -14,7 +14,7 @@ export default class GitHub { static ISSUE_STATE_ALL = 'all'; constructor(repository) { - const { version } = require('../../package.json'); + const { version } = require('../../../package.json'); this.octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN, @@ -198,4 +198,16 @@ export default class GitHub { logger.error(`Failed to update issue "${title}": ${error.stack}`); } } + + generateDeclarationURL(serviceName) { + return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`; + } + + generateVersionURL(serviceName, termsType) { + return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`; + } + + generateSnapshotsBaseUrl(serviceName, termsType) { + return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`; + } } diff --git a/src/reporter/github.test.js b/src/reporter/github/index.test.js similarity index 99% rename from src/reporter/github.test.js rename to src/reporter/github/index.test.js index 5aa21f18..7a126413 100644 --- a/src/reporter/github.test.js +++ b/src/reporter/github/index.test.js @@ -3,7 +3,7 @@ import { createRequire } from 'module'; import { expect } from 'chai'; import nock from 'nock'; -import GitHub from './github.js'; +import GitHub from './index.js'; const require = createRequire(import.meta.url); diff --git a/src/reporter/labels.json b/src/reporter/github/labels.json similarity index 100% rename from src/reporter/labels.json rename to src/reporter/github/labels.json diff --git a/src/reporter/labels.test.js b/src/reporter/github/labels.test.js similarity index 94% rename from src/reporter/labels.test.js rename to src/reporter/github/labels.test.js index c7fc9fd4..a817a806 100644 --- a/src/reporter/labels.test.js +++ b/src/reporter/github/labels.test.js @@ -2,7 +2,7 @@ import { createRequire } from 'module'; import chai from 'chai'; -import { MANAGED_BY_OTA_MARKER } from './github.js'; +import { MANAGED_BY_OTA_MARKER } from './index.js'; const require = createRequire(import.meta.url); diff --git a/src/reporterGitlab/gitlab.js b/src/reporter/gitlab/index.js similarity index 83% rename from src/reporterGitlab/gitlab.js rename to src/reporter/gitlab/index.js index cf4133a9..2999f0b1 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporter/gitlab/index.js @@ -4,25 +4,27 @@ import HttpProxyAgent from 'http-proxy-agent'; import HttpsProxyAgent from 'https-proxy-agent'; import nodeFetch from 'node-fetch'; -import logger from '../logger/index.js'; +import logger from '../../logger/index.js'; const require = createRequire(import.meta.url); export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; +const BASE_URL = 'https://gitlab.com'; +const API_BASE_URL = 'https://gitlab.com/api/v4'; export default class GitLab { static ISSUE_STATE_CLOSED = 'closed'; static ISSUE_STATE_OPEN = 'opened'; static ISSUE_STATE_ALL = 'all'; - constructor(repository) { + constructor(repository, baseURL = BASE_URL, apiBaseURL = API_BASE_URL) { const [ owner, repo ] = repository.split('/'); this.commonParams = { owner, repo }; this.projectId = null; - const gitlabUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; - - this.gitlabUrl = gitlabUrl; + this.baseURL = baseURL; + console.log('this.baseURL', this.baseURL); + this.apiBaseURL = apiBaseURL; } async initialize() { @@ -31,7 +33,7 @@ export default class GitLab { try { const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; const response = await nodeFetch( - `${this.gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, + `${this.apiBaseURL}/projects/${encodeURIComponent(repositoryPath)}`, options, ); @@ -47,6 +49,7 @@ export default class GitLab { logger.error(`Error while obtaining projectId: ${error}`); this.projectId = null; } + this.MANAGED_LABELS = require('./labels.json'); const existingLabels = await this.getRepositoryLabels(); @@ -70,9 +73,10 @@ export default class GitLab { try { const options = GitLab.baseOptionsHttpReq(); const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + `${this.apiBaseURL}/projects/${this.projectId}/labels?with_counts=true`, options, ); + const res = await response.json(); if (response.ok) { @@ -105,7 +109,7 @@ export default class GitLab { }; const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/labels`, + `${this.apiBaseURL}/projects/${this.projectId}/labels`, options, ); @@ -139,7 +143,7 @@ export default class GitLab { }; const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/issues`, + `${this.apiBaseURL}/projects/${this.projectId}/issues`, options, ); @@ -170,7 +174,7 @@ export default class GitLab { try { const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + `${this.apiBaseURL}/projects/${this.projectId}/issues/${issue.iid}`, options, ); @@ -199,7 +203,7 @@ export default class GitLab { try { const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + `${this.apiBaseURL}/projects/${this.projectId}/issues/${issue.iid}`, options, ); const res = await response.json(); @@ -228,7 +232,7 @@ export default class GitLab { try { const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + `${this.apiBaseURL}/projects/${this.projectId}/issues/${issue.iid}`, options, ); const res = await response.json(); @@ -245,10 +249,10 @@ export default class GitLab { async getIssue({ title, ...searchParams }) { try { - let apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + let apiUrl = `${this.apiBaseURL}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; - if (searchParams.state == 'all') apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; - apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; + if (searchParams.state == 'all') apiUrl = `${this.apiBaseURL}/projects/${this.projectId}/issues?per_page=100`; + apiUrl = `${this.apiBaseURL}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; const options = GitLab.baseOptionsHttpReq(); @@ -286,7 +290,7 @@ export default class GitLab { try { const response = await nodeFetch( - `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, + `${this.apiBaseURL}/projects/${this.projectId}/issues/${issue.iid}/notes`, options, ); const res = await response.json(); @@ -363,4 +367,16 @@ export default class GitLab { return options; } + + generateDeclarationURL(serviceName) { + return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/declarations/${encodeURIComponent(serviceName)}.json`; + } + + generateVersionURL(serviceName, termsType) { + return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(serviceName, termsType)}.md`; + } + + generateSnapshotsBaseUrl(serviceName, termsType) { + return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`; + } } diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporter/gitlab/index.test.js similarity index 70% rename from src/reporterGitlab/gitlab.test.js rename to src/reporter/gitlab/index.test.js index ede97cd1..dfc5cd60 100644 --- a/src/reporterGitlab/gitlab.test.js +++ b/src/reporter/gitlab/index.test.js @@ -3,7 +3,7 @@ import { createRequire } from 'module'; import { expect } from 'chai'; import nock from 'nock'; -import GitLab from './gitlab.js'; +import GitLab from './index.js'; const require = createRequire(import.meta.url); @@ -12,37 +12,32 @@ describe('GitLab', function () { let MANAGED_LABELS; let gitlab; - let gitlabApiUrl = ''; - let reqHeaders; - const projectId = '4'; + const PROJECT_ID = '4'; before(() => { MANAGED_LABELS = require('./labels.json'); gitlab = new GitLab('owner/repo'); - gitlab.projectId = projectId; - gitlabApiUrl = gitlab.gitlabUrl; - reqHeaders = { reqheaders: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }; }); - describe('#Gitlab_initialize', () => { + describe('#initialize', () => { const scopes = []; before(async () => { const existingLabels = MANAGED_LABELS.slice(0, -2); - nock(gitlabApiUrl, reqHeaders) + nock(gitlab.apiBaseURL) .get(`/projects/${encodeURIComponent('owner/repo')}`) - .reply(200, { id: 4 }); + .reply(200, { id: PROJECT_ID }); - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/labels?with_counts=true`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/labels?with_counts=true`) .reply(200, existingLabels); const missingLabels = MANAGED_LABELS.slice(-2); for (const label of missingLabels) { - scopes.push(nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/labels`) + scopes.push(nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/labels`) .reply(200, { name: label.name })); } @@ -56,14 +51,14 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_getRepositoryLabels', () => { + describe('#getRepositoryLabels', () => { let scope; let result; const LABELS = [{ name: 'bug' }, { name: 'enhancement' }]; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/labels?with_counts=true`) + scope = nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/labels?with_counts=true`) .reply(200, LABELS); result = await gitlab.getRepositoryLabels(); @@ -80,13 +75,13 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_createLabel', () => { + describe('#createLabel', () => { let scope; const LABEL = { name: 'new_label', color: 'ffffff' }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/labels`, body => body.name === LABEL.name) + scope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/labels`, body => body.name === LABEL.name) .reply(200, LABEL); await gitlab.createLabel(LABEL); @@ -99,7 +94,7 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_createIssue', () => { + describe('#createIssue', () => { let scope; let result; @@ -117,8 +112,8 @@ describe('GitLab', function () { }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/issues`) + scope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/issues`) .reply(200, CREATED_ISSUE); result = await gitlab.createIssue(ISSUE); @@ -135,7 +130,7 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_setIssueLabels', () => { + describe('#setIssueLabels', () => { let scope; const issue = { iid: 123, @@ -149,8 +144,8 @@ describe('GitLab', function () { }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${issue.iid}`, { labels }) + scope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${issue.iid}`, { labels }) .reply(200, response); await gitlab.setIssueLabels({ issue, labels }); @@ -163,15 +158,15 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_openIssue', () => { + describe('#openIssue', () => { let scope; const ISSUE = { iid: 123, title: 'issue reopened' }; const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; const response = { iid: 123 }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + scope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) .reply(200, response); await gitlab.openIssue(ISSUE); @@ -184,15 +179,15 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_closeIssue', () => { + describe('#closeIssue', () => { let scope; const ISSUE = { iid: 123, title: 'close issue' }; const EXPECTED_REQUEST_BODY = { state_event: 'close' }; const response = { iid: 123 }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + scope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) .reply(200, response); await gitlab.closeIssue(ISSUE); @@ -205,7 +200,7 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_getIssue', () => { + describe('#getIssue', () => { let scope; let result; @@ -213,8 +208,8 @@ describe('GitLab', function () { const ANOTHER_ISSUE = { number: 124, title: 'Test Issue 2' }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + scope = nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) .reply(200, [ ISSUE, ANOTHER_ISSUE ]); result = await gitlab.getIssue({ title: ISSUE.title }); @@ -231,15 +226,15 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_addCommentToIssue', () => { + describe('#addCommentToIssue', () => { let scope; const ISSUE = { iid: 123, title: 'Test Issue' }; const COMMENT = 'Test comment'; const response = { iid: 123, id: 23, body: 'Test comment' }; before(async () => { - scope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + scope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) .reply(200, response); await gitlab.addCommentToIssue({ issue: ISSUE, comment: COMMENT }); @@ -252,7 +247,7 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_closeIssueWithCommentIfExists', () => { + describe('#closeIssueWithCommentIfExists', () => { after(nock.cleanAll); context('when the issue exists and is open', () => { @@ -269,16 +264,16 @@ describe('GitLab', function () { const responseCloseissue = { iid: 123 }; before(async () => { - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) .reply(200, [ISSUE]); - addCommentScope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + addCommentScope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) .reply(200, responseAddcomment); - closeIssueScope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + closeIssueScope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}`, closeissueBody) .reply(200, responseCloseissue); await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); @@ -307,16 +302,16 @@ describe('GitLab', function () { const responseCloseissue = { iid: 123 }; before(async () => { - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) .reply(200, []); - addCommentScope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + addCommentScope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) .reply(200, responseAddcomment); - closeIssueScope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + closeIssueScope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${ISSUE.iid}`, closeissueBody) .reply(200, responseCloseissue); await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); @@ -341,15 +336,15 @@ describe('GitLab', function () { const responseCloseissue = { iid: 123 }; before(async () => { - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(TITLE)}&per_page=100`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(TITLE)}&per_page=100`) .reply(200, []); - addCommentScope = nock(gitlabApiUrl, reqHeaders) + addCommentScope = nock(gitlab.apiBaseURL) .post(/\/projects\/\d+\/issues\/\d+\/notes/, { body: COMMENT }) .reply(200, responseAddcomment); - closeIssueScope = nock(gitlabApiUrl, reqHeaders) + closeIssueScope = nock(gitlab.apiBaseURL) .put(/\/projects\/\d+\/issues\/\d+/, closeissueBody) .reply(200, responseCloseissue); @@ -366,14 +361,14 @@ describe('GitLab', function () { }); }); - describe('#Gitlab_createOrUpdateIssue', () => { + describe('#createOrUpdateIssue', () => { before(async () => { - nock(gitlabApiUrl, reqHeaders) + nock(gitlab.apiBaseURL) .get(`/projects/${encodeURIComponent('owner/repo')}`) .reply(200, { id: 4 }); - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/labels?with_counts=true`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/labels?with_counts=true`) .reply(200, MANAGED_LABELS); await gitlab.initialize(); @@ -388,13 +383,13 @@ describe('GitLab', function () { }; before(async () => { - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE_TO_CREATE.title)}&per_page=100`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE_TO_CREATE.title)}&per_page=100`) .reply(200, []); // Simulate that there is no issues on the repository - createIssueScope = nock(gitlabApiUrl, reqHeaders) + createIssueScope = nock(gitlab.apiBaseURL) .post( - `/projects/${projectId}/issues`, + `/projects/${PROJECT_ID}/issues`, { title: ISSUE_TO_CREATE.title, description: ISSUE_TO_CREATE.description, @@ -441,20 +436,20 @@ describe('GitLab', function () { const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; before(async () => { - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); - openIssueScope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + openIssueScope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${iid}`, EXPECTED_REQUEST_BODY) .reply(200, responseIssuereopened); - setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + setIssueLabelsScope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: ['location'] }) .reply(200, responseSetLabels); - addCommentScope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + addCommentScope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/issues/${iid}/notes`, { body: ISSUE.description }) .reply(200, responseAddcomment); await gitlab.createOrUpdateIssue(ISSUE); @@ -496,20 +491,20 @@ describe('GitLab', function () { const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; before(async () => { - nock(gitlabApiUrl, reqHeaders) - .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + nock(gitlab.apiBaseURL) + .get(`/projects/${PROJECT_ID}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); - openIssueScope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + openIssueScope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${iid}`, EXPECTED_REQUEST_BODY) .reply(200, responseIssuereopened); - setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) - .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + setIssueLabelsScope = nock(gitlab.apiBaseURL) + .put(`/projects/${PROJECT_ID}/issues/${iid}`, { labels: ['location'] }) .reply(200, responseSetLabels); - addCommentScope = nock(gitlabApiUrl, reqHeaders) - .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + addCommentScope = nock(gitlab.apiBaseURL) + .post(`/projects/${PROJECT_ID}/issues/${iid}/notes`, { body: ISSUE.description }) .reply(200, responseAddcomment); await gitlab.createOrUpdateIssue(ISSUE); diff --git a/src/reporterGitlab/labels.json b/src/reporter/gitlab/labels.json similarity index 100% rename from src/reporterGitlab/labels.json rename to src/reporter/gitlab/labels.json diff --git a/src/reporterGitlab/labels.test.js b/src/reporter/gitlab/labels.test.js similarity index 81% rename from src/reporterGitlab/labels.test.js rename to src/reporter/gitlab/labels.test.js index a1e61f2d..1c50a892 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporter/gitlab/labels.test.js @@ -2,7 +2,7 @@ import { createRequire } from 'module'; import chai from 'chai'; -import { MANAGED_BY_OTA_MARKER } from './gitlab.js'; +import { MANAGED_BY_OTA_MARKER } from './index.js'; const require = createRequire(import.meta.url); @@ -21,7 +21,7 @@ describe('Reporter GitLab labels', () => { }); it('complies with the GitLab constraints for color', () => { - const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` + const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; expect(validHexColorRegex.test(label.color)).to.be.true; }); diff --git a/src/reporter/index.js b/src/reporter/index.js index 23b678de..93568369 100644 --- a/src/reporter/index.js +++ b/src/reporter/index.js @@ -2,7 +2,7 @@ import mime from 'mime'; import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; -import GitHub from './github.js'; +import { createReporter } from './factory.js'; const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/en/service'; const DOC_URL = 'https://docs.opentermsarchive.org'; @@ -33,28 +33,28 @@ function getLabelNameFromError(error) { // In the following class, it is assumed that each issue is managed using its title as a unique identifier export default class Reporter { constructor(config) { - const { repositories } = config.githubIssues; + const { repositories } = config; for (const repositoryType of Object.keys(repositories)) { if (!repositories[repositoryType].includes('/') || repositories[repositoryType].includes('https://')) { - throw new Error(`Configuration entry "reporter.githubIssues.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); + throw new Error(`Configuration entry "reporter.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); } } - this.github = new GitHub(repositories.declarations); + this.reporter = createReporter(config); this.repositories = repositories; } initialize() { - return this.github.initialize(); + return this.reporter.initialize(); } onTrackingStarted() { - return this.github.clearCache(); + return this.reporter.clearCache(); } async onVersionRecorded(version) { - await this.github.closeIssueWithCommentIfExists({ + await this.reporter.closeIssueWithCommentIfExists({ title: Reporter.generateTitleID(version.serviceId, version.termsType), comment: `### Tracking resumed @@ -63,7 +63,7 @@ A new version has been recorded.`, } async onVersionNotChanged(version) { - await this.github.closeIssueWithCommentIfExists({ + await this.reporter.closeIssueWithCommentIfExists({ title: Reporter.generateTitleID(version.serviceId, version.termsType), comment: `### Tracking resumed @@ -76,7 +76,7 @@ No changes were found in the last run, so no new version has been recorded.`, } async onInaccessibleContent(error, terms) { - await this.github.createOrUpdateIssue({ + await this.reporter.createOrUpdateIssue({ title: Reporter.generateTitleID(terms.service.id, terms.type), description: this.generateDescription({ error, terms }), label: getLabelNameFromError(error), @@ -97,9 +97,9 @@ No changes were found in the last run, so no new version has been recorded.`, }); const contributionToolUrl = `${CONTRIBUTION_TOOL_URL}?${contributionToolParams}`; - const latestDeclarationLink = `[Latest declaration](https://github.com/${this.repositories.declarations}/blob/main/declarations/${encodeURIComponent(terms.service.name)}.json)`; - const latestVersionLink = `[Latest version](https://github.com/${this.repositories.versions}/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}.md)`; - const snapshotsBaseUrl = `https://github.com/${this.repositories.snapshots}/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}`; + const latestDeclarationLink = `[Latest declaration](${this.reporter.generateDeclarationURL(terms.service.name)})`; + const latestVersionLink = `[Latest version](${this.reporter.generateVersionURL(terms.service.name, terms.type)})`; + const snapshotsBaseUrl = this.reporter.generateSnapshotsBaseUrl(terms.service.name, terms.type); const latestSnapshotsLink = terms.hasMultipleSourceDocuments ? `Latest snapshots:\n - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n - ')}` : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`; diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js deleted file mode 100644 index be516118..00000000 --- a/src/reporterGitlab/index.js +++ /dev/null @@ -1,147 +0,0 @@ -import mime from 'mime'; - -import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; - -import GitLab from './gitlab.js'; - -const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/'; -const DOC_URL = 'https://docs.opentermsarchive.org'; -const REPO_URL = process.env.OTA_ENGINE_GITLAB_BASE_URL; - -const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = { - 'has no match': 'selectors', - 'HTTP code 404': 'location', - 'HTTP code 403': '403', - 'HTTP code 429': '429', - 'HTTP code 500': '500', - 'HTTP code 502': '502', - 'HTTP code 503': '503', - 'Timed out after': 'timeout', - 'getaddrinfo EAI_AGAIN': 'EAI_AGAIN', - 'getaddrinfo ENOTFOUND': 'ENOTFOUND', - 'Response is empty': 'empty response', - 'unable to verify the first certificate': 'first certificate', - 'certificate has expired': 'certificate expired', - 'maximum redirect reached': 'redirects', -}; - -function getLabelNameFromError(error) { - return ERROR_MESSAGE_TO_ISSUE_LABEL_MAP[Object.keys(ERROR_MESSAGE_TO_ISSUE_LABEL_MAP).find(substring => error.toString().includes(substring))] || 'to clarify'; -} - -// In the following class, it is assumed that each issue is managed using its title as a unique identifier -export default class Reporter { - constructor(config) { - const { repositories } = config.gitlabIssues; - - for (const repositoryType of Object.keys(repositories)) { - if (!repositories[repositoryType].includes('/') || repositories[repositoryType].includes('https://')) { - throw new Error(`Configuration entry "reporter.gitlabIssues.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); - } - } - - this.gitlab = new GitLab(repositories.declarations); - this.repositories = repositories; - } - - initialize() { - return this.gitlab.initialize(); - } - - async onVersionRecorded(version) { - await this.gitlab.closeIssueWithCommentIfExists({ - title: Reporter.generateTitleID(version.serviceId, version.termsType), - comment: `### Tracking resumed - -A new version has been recorded.`, - }); - } - - async onVersionNotChanged(version) { - await this.gitlab.closeIssueWithCommentIfExists({ - title: Reporter.generateTitleID(version.serviceId, version.termsType), - comment: `### Tracking resumed - -No changes were found in the last run, so no new version has been recorded.`, - }); - } - - async onFirstVersionRecorded(version) { - return this.onVersionRecorded(version); - } - - async onInaccessibleContent(error, terms) { - await this.gitlab.createOrUpdateIssue({ - title: Reporter.generateTitleID(terms.service.id, terms.type), - description: this.generateDescription({ error, terms }), - label: getLabelNameFromError(error), - }); - } - - generateDescription({ error, terms }) { - const date = new Date(); - const currentFormattedDate = date.toLocaleDateString('en-GB', { year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric', timeZoneName: 'short', timeZone: 'UTC' }); - const validUntil = toISODateWithoutMilliseconds(date); - - const hasSnapshots = terms.sourceDocuments.every(sourceDocument => sourceDocument.snapshotId); - - const contributionToolParams = new URLSearchParams({ - json: JSON.stringify(terms.toPersistence()), - destination: this.repositories.declarations, - step: '2', - }); - const contributionToolUrl = `${CONTRIBUTION_TOOL_URL}?${contributionToolParams}`; - - const latestDeclarationLink = `[Latest declaration](${REPO_URL}/${this.repositories.declarations}/-/blob/main/declarations/${encodeURIComponent(terms.service.name)}.json)`; - const latestVersionLink = `[Latest version](${REPO_URL}/${this.repositories.versions}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}.md)`; - const snapshotsBaseUrl = `${REPO_URL}/${this.repositories.snapshots}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}`; - const latestSnapshotsLink = terms.hasMultipleSourceDocuments - ? `Latest snapshots:\n - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n - ')}` - : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`; - - /* eslint-disable no-irregular-whitespace */ - return ` -### No version of the \`${terms.type}\` of service \`${terms.service.name}\` is recorded anymore since ${currentFormattedDate} - -The source document${terms.hasMultipleSourceDocuments ? 's have' : ' has'}${hasSnapshots ? ' ' : ' not '}been recorded in ${terms.hasMultipleSourceDocuments ? 'snapshots' : 'a snapshot'}, ${hasSnapshots ? 'but ' : 'thus '} no version can be [extracted](${DOC_URL}/#tracking-terms). -${hasSnapshots ? 'After correction, it might still be possible to recover the missed versions.' : ''} - -### What went wrong - -- ${error.reasons.join('\n- ')} - -### How to resume tracking - -First of all, check if the source documents are accessible through a web browser: - -- [ ] ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.location}](${sourceDocument.location})`).join('\n- [ ] ')} - -#### If the source documents are accessible through a web browser - -[Edit the declaration](${contributionToolUrl}): -- Try updating the selectors. -- Try switching client scripts on with expert mode. - -#### If the source documents are not accessible anymore - -- If the source documents have moved, find their new location and [update it](${contributionToolUrl}). -- If these terms have been removed, move them from the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. -- If the service has closed, move the entire contents of the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. - -#### If none of the above works - -If the source documents are accessible in a browser but fetching them always fails from the Open Terms Archive server, this is most likely because the service provider has blocked the Open Terms Archive robots from accessing its content. In this case, updating the declaration will not enable resuming tracking. Only an agreement with the service provider, an engine upgrade, or some technical workarounds provided by the administrator of this collection’s server might resume tracking. - -### References - -- ${latestDeclarationLink} -${this.repositories.versions ? `- ${latestVersionLink}` : ''} -${this.repositories.snapshots ? `- ${latestSnapshotsLink}` : ''} -`; - /* eslint-enable no-irregular-whitespace */ - } - - static generateTitleID(serviceId, type) { - return `\`${serviceId}\` ‧ \`${type}\` ‧ not tracked anymore`; - } -} From 48ad0933c70a2329d61f0a0ea9c86c7321a8c73f Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 5 Nov 2024 15:07:12 +0100 Subject: [PATCH 13/26] Factorize code between Gitlab and GitHub publisher --- config/default.json | 1 - scripts/dataset/assets/README.template.js | 8 ++-- scripts/dataset/export/index.js | 3 +- scripts/dataset/index.js | 35 ++++------------ scripts/dataset/publish/github/index.js | 36 ++++++++++++++++ .../gitlab}/index.js | 11 +++-- scripts/dataset/publish/index.js | 42 +++++-------------- 7 files changed, 64 insertions(+), 72 deletions(-) create mode 100644 scripts/dataset/publish/github/index.js rename scripts/dataset/{publishGitLab => publish/gitlab}/index.js (93%) diff --git a/config/default.json b/config/default.json index 8102f222..0d3d82ea 100644 --- a/config/default.json +++ b/config/default.json @@ -59,7 +59,6 @@ "dataset": { "title": "sandbox", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox", - "versionsRepositoryURLGitLab": "https://gitlab.com/ota-sandbox-example/sandbox", "publishingSchedule": "30 8 * * MON" } } diff --git a/scripts/dataset/assets/README.template.js b/scripts/dataset/assets/README.template.js index 1cda3428..1c63d117 100644 --- a/scripts/dataset/assets/README.template.js +++ b/scripts/dataset/assets/README.template.js @@ -3,10 +3,10 @@ import config from 'config'; const LOCALE = 'en-EN'; const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; -export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { +export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { return `# Open Terms Archive — ${title({ releaseDate })} -${body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL })}`; +${body({ servicesCount, firstVersionDate, lastVersionDate })}`; } export function title({ releaseDate }) { @@ -17,10 +17,12 @@ export function title({ releaseDate }) { return `${title} — ${releaseDate} dataset`; } -export function body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { +export function body({ servicesCount, firstVersionDate, lastVersionDate }) { firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + const versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); + return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURL}](${versionsRepositoryURL}). diff --git a/scripts/dataset/export/index.js b/scripts/dataset/export/index.js index 3215768c..bc0a028a 100644 --- a/scripts/dataset/export/index.js +++ b/scripts/dataset/export/index.js @@ -16,7 +16,7 @@ const fs = fsApi.promises; const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats -export default async function generate({ archivePath, releaseDate, versionsRepositoryURL }) { +export default async function generate({ archivePath, releaseDate }) { const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize(); const archive = await initializeArchive(archivePath); @@ -61,7 +61,6 @@ export default async function generate({ archivePath, releaseDate, versionsRepos releaseDate, firstVersionDate, lastVersionDate, - versionsRepositoryURL, }), { name: `${archive.basename}/README.md` }, ); diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 66ea0a1f..4c739686 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -6,7 +6,6 @@ import config from 'config'; import generateRelease from './export/index.js'; import logger from './logger/index.js'; import publishRelease from './publish/index.js'; -import publishReleaseGitLab from './publishGitLab/index.js'; export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) { const releaseDate = new Date(); @@ -15,15 +14,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start exporting dataset…'); - const usesGitHub = (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined'); - const usesGitLab = (typeof process.env.OTA_ENGINE_GITLAB_TOKEN !== 'undefined'); - - let versionsRepositoryURL = ''; - - if (usesGitHub) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); - if (usesGitLab) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); - - const stats = await generateRelease({ archivePath, releaseDate, versionsRepositoryURL }); + const stats = await generateRelease({ archivePath, releaseDate }); logger.info(`Dataset exported in ${archivePath}`); @@ -33,25 +24,13 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start publishing dataset…'); - if (usesGitHub) { - const releaseUrl = await publishRelease({ - archivePath, - releaseDate, - stats, - }); + const releaseUrl = await publishRelease({ + archivePath, + releaseDate, + stats, + }); - logger.info(`Dataset published to ${releaseUrl}`); - } - - if (usesGitLab) { - const releaseUrl = await publishReleaseGitLab({ - archivePath, - releaseDate, - stats, - }); - - logger.info(`Dataset published to ${releaseUrl}`); - } + logger.info(`Dataset published to ${releaseUrl}`); if (!shouldRemoveLocalCopy) { return; diff --git a/scripts/dataset/publish/github/index.js b/scripts/dataset/publish/github/index.js new file mode 100644 index 00000000..54d54b44 --- /dev/null +++ b/scripts/dataset/publish/github/index.js @@ -0,0 +1,36 @@ +import fsApi from 'fs'; +import path from 'path'; +import url from 'url'; + +import config from 'config'; +import { Octokit } from 'octokit'; + +import * as readme from '../../assets/README.template.js'; + +export default async function publish({ archivePath, releaseDate, stats }) { + const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN }); + + const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component); + + const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag + + const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({ + owner, + repo, + tag_name: tagName, + name: readme.title({ releaseDate }), + body: readme.body(stats), + }); + + await octokit.rest.repos.uploadReleaseAsset({ + data: fsApi.readFileSync(archivePath), + headers: { + 'content-type': 'application/zip', + 'content-length': fsApi.statSync(archivePath).size, + }, + name: path.basename(archivePath), + url: uploadUrl, + }); + + return releaseUrl; +} diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publish/gitlab/index.js similarity index 93% rename from scripts/dataset/publishGitLab/index.js rename to scripts/dataset/publish/gitlab/index.js index e41c3e1c..bf8ecdfd 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publish/gitlab/index.js @@ -7,20 +7,19 @@ import dotenv from 'dotenv'; import FormData from 'form-data'; import nodeFetch from 'node-fetch'; -import GitLab from '../../../src/reporterGitlab/gitlab.js'; -import * as readme from '../assets/README.template.js'; -import logger from '../logger/index.js'; +import GitLab from '../../../../src/reporter/gitlab/index.js'; +import * as readme from '../../assets/README.template.js'; +import logger from '../../logger/index.js'; dotenv.config(); -const gitlabAPIUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; - -export default async function publishReleaseGitLab({ +export default async function publish({ archivePath, releaseDate, stats, }) { let projectId = null; + const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL'); const [ owner, repo ] = url .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) diff --git a/scripts/dataset/publish/index.js b/scripts/dataset/publish/index.js index 89e0f7fe..9d4ae05f 100644 --- a/scripts/dataset/publish/index.js +++ b/scripts/dataset/publish/index.js @@ -1,36 +1,14 @@ -import fsApi from 'fs'; -import path from 'path'; -import url from 'url'; +import publishGitHub from './github/index.js'; +import publishGitLab from './gitlab/index.js'; -import config from 'config'; -import { Octokit } from 'octokit'; +export default function publishRelease({ archivePath, releaseDate, stats }) { + if (process.env.OTA_ENGINE_GITHUB_TOKEN) { + return publishGitHub({ archivePath, releaseDate, stats }); + } -import * as readme from '../assets/README.template.js'; + if (process.env.OTA_ENGINE_GITLAB_TOKEN) { + return publishGitLab({ archivePath, releaseDate, stats }); + } -export default async function publish({ archivePath, releaseDate, stats }) { - const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN }); - - const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component); - - const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag - - const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({ - owner, - repo, - tag_name: tagName, - name: readme.title({ releaseDate }), - body: readme.body(stats), - }); - - await octokit.rest.repos.uploadReleaseAsset({ - data: fsApi.readFileSync(archivePath), - headers: { - 'content-type': 'application/zip', - 'content-length': fsApi.statSync(archivePath).size, - }, - name: path.basename(archivePath), - url: uploadUrl, - }); - - return releaseUrl; + throw new Error('No GitHub or GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.'); } From ca58f974e81c66d47b5b62ec8393da28c0a83d11 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 5 Nov 2024 15:07:18 +0100 Subject: [PATCH 14/26] Fix deprecated method --- scripts/dataset/publish/gitlab/index.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/dataset/publish/gitlab/index.js b/scripts/dataset/publish/gitlab/index.js index bf8ecdfd..d789c640 100644 --- a/scripts/dataset/publish/gitlab/index.js +++ b/scripts/dataset/publish/gitlab/index.js @@ -1,6 +1,5 @@ import fsApi from 'fs'; import path from 'path'; -import url from 'url'; import config from 'config'; import dotenv from 'dotenv'; @@ -21,10 +20,10 @@ export default async function publish({ let projectId = null; const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL'); - const [ owner, repo ] = url - .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) - .pathname.split('/') - .filter(component => component); + const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) + .pathname + .split('/') + .filter(Boolean); const commonParams = { owner, repo }; try { From a6d8d298d807605081871decfe066a588ca22015 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 5 Nov 2024 15:20:10 +0100 Subject: [PATCH 15/26] Improve changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72440274..9867cc3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ All changes that impact users of this module are documented in this file, in the ### Added -- Add GitLab functionalities +- Add support for GitLab as an alternative platform to GitHub for issue reporting +- Add support for GitLab as an alternative platform to GitHub for publishing datasets ## 2.5.0 - 2024-10-29 From 64d106025d27ae338b61fec2d2f029b788dbc10a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Thu, 7 Nov 2024 14:03:58 +0100 Subject: [PATCH 16/26] Improve changelog entry Co-authored-by: Matti Schneider --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9867cc3d..2d10fabd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,8 @@ All changes that impact users of this module are documented in this file, in the ### Added -- Add support for GitLab as an alternative platform to GitHub for issue reporting -- Add support for GitLab as an alternative platform to GitHub for publishing datasets +- Add support for GitLab for issue reporting +- Add support for GitLab Releases for publishing datasets ## 2.5.0 - 2024-10-29 From 8769d0fe7459c4d89f5f6ab706c851c4022633a8 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Thu, 7 Nov 2024 14:08:34 +0100 Subject: [PATCH 17/26] Add release funders --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d10fabd..3bb04759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ All changes that impact users of this module are documented in this file, in the ## Unreleased [minor] +> Development of this release was supported by the European Commission’s [Joint Research Centre](https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/joint-research-centre_en) and the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs. + ### Added - Add support for GitLab for issue reporting From aed17f341523d3686aa819e2c7e1dd7947114d67 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Thu, 7 Nov 2024 14:12:34 +0100 Subject: [PATCH 18/26] Clarify token precedence between GitHub and GitLab --- .env.example | 3 +++ scripts/dataset/publish/index.js | 1 + 2 files changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 08813d69..f55dfdf1 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,8 @@ OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…' OTA_ENGINE_SMTP_PASSWORD='password' + +# If both GitHub and GitLab tokens are defined, GitHub takes precedence for dataset publishing OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX + OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX diff --git a/scripts/dataset/publish/index.js b/scripts/dataset/publish/index.js index 9d4ae05f..08aa0c2d 100644 --- a/scripts/dataset/publish/index.js +++ b/scripts/dataset/publish/index.js @@ -2,6 +2,7 @@ import publishGitHub from './github/index.js'; import publishGitLab from './gitlab/index.js'; export default function publishRelease({ archivePath, releaseDate, stats }) { + // If both GitHub and GitLab tokens are defined, GitHub takes precedence if (process.env.OTA_ENGINE_GITHUB_TOKEN) { return publishGitHub({ archivePath, releaseDate, stats }); } From cd411eb219f432296c63de3a9b26ba187598ab8a Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 12 Nov 2024 15:39:16 +0100 Subject: [PATCH 19/26] Update changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb04759..d76a1acf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All changes that impact users of this module are documented in this file, in the ## Unreleased [minor] -> Development of this release was supported by the European Commission’s [Joint Research Centre](https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/joint-research-centre_en) and the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs. +> Development of this release was supported by the [European Union](https://commission.europa.eu/) and the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs. ### Added From c9fa05e1667c674b03f22e5b95d26710a8eacc70 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 12 Nov 2024 16:41:17 +0100 Subject: [PATCH 20/26] Add backward compatibility for legacy config --- src/index.js | 18 +++++------ src/reporter/index.js | 52 ++++++++++++++++++++++++++----- src/reporter/index.test.js | 63 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 src/reporter/index.test.js diff --git a/src/index.js b/src/index.js index 142c63ef..8b011133 100644 --- a/src/index.js +++ b/src/index.js @@ -56,17 +56,13 @@ export default async function track({ services, types, extractOnly, schedule }) } if (process.env.OTA_ENGINE_GITHUB_TOKEN || process.env.OTA_ENGINE_GITLAB_TOKEN) { - if (config.has('@opentermsarchive/engine.reporter.repositories.declarations')) { - try { - const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter')); - - await reporter.initialize(); - archivist.attach(reporter); - } catch (error) { - logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); - } - } else { - logger.warn('Configuration key "reporter.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + try { + const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter')); + + await reporter.initialize(); + archivist.attach(reporter); + } catch (error) { + logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); } } else { logger.warn('Environment variable with token for GitHub or GitLab was not found; the Reporter module will be ignored'); diff --git a/src/reporter/index.js b/src/reporter/index.js index 93568369..97542c9c 100644 --- a/src/reporter/index.js +++ b/src/reporter/index.js @@ -1,6 +1,7 @@ import mime from 'mime'; import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; +import logger from '../logger/index.js'; import { createReporter } from './factory.js'; @@ -33,16 +34,53 @@ function getLabelNameFromError(error) { // In the following class, it is assumed that each issue is managed using its title as a unique identifier export default class Reporter { constructor(config) { - const { repositories } = config; + const normalizedConfig = Reporter.normalizeConfig(config); - for (const repositoryType of Object.keys(repositories)) { - if (!repositories[repositoryType].includes('/') || repositories[repositoryType].includes('https://')) { - throw new Error(`Configuration entry "reporter.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); - } + Reporter.validateConfiguration(normalizedConfig.repositories); + + this.reporter = createReporter(normalizedConfig); + this.repositories = normalizedConfig.repositories; + } + + /** + * Support for legacy config format where reporter configuration was nested under "githubIssues" + * Example: + * + * ```json + * { + * "githubIssues": { + * "repositories": { + * "declarations": "OpenTermsArchive/sandbox-declarations" + * } + * } + * } + * ``` + * + * @deprecated + */ + static normalizeConfig(config) { + if (config.githubIssues) { + logger.warn('The "reporter.githubIssues" key is deprecated; please see configuration documentation for the new format: https://docs.opentermsarchive.org/#configuring'); + + return { + type: 'github', + repositories: config.githubIssues.repositories, + }; + } + + return config; + } + + static validateConfiguration(repositories) { + if (!repositories?.declarations) { + throw new Error('Required configuration key "reporter.repositories.declarations" was not found; issues on the declarations repository cannot be created'); } - this.reporter = createReporter(config); - this.repositories = repositories; + for (const [ type, repo ] of Object.entries(repositories)) { + if (!repo.includes('/') || repo.includes('https://')) { + throw new Error(`Configuration entry "reporter.repositories.${type}" is expected to be a string in the format /, but received: "${repo}"`); + } + } } initialize() { diff --git a/src/reporter/index.test.js b/src/reporter/index.test.js new file mode 100644 index 00000000..b0fd8412 --- /dev/null +++ b/src/reporter/index.test.js @@ -0,0 +1,63 @@ +import { expect } from 'chai'; + +import Reporter from './index.js'; + +describe('Reporter', () => { + describe('#normalizeConfig', () => { + context('with current config format', () => { + it('returns the config as is', () => { + const config = { repositories: { declarations: 'owner/repo' } }; + const normalizedConfig = Reporter.normalizeConfig(config); + + expect(normalizedConfig).to.deep.equal(config); + }); + }); + + context('with old config format where githubIssues is nested under reporter', () => { + it('returns a normalized config', () => { + const config = { githubIssues: { repositories: { declarations: 'owner/repo' } } }; + const expectedConfig = { + type: 'github', + repositories: { declarations: 'owner/repo' }, + }; + const normalizedConfig = Reporter.normalizeConfig(config); + + expect(normalizedConfig).to.deep.equal(expectedConfig); + }); + }); + }); + + describe('#validateConfiguration', () => { + context('with valid configuration', () => { + it('does not throw an error', () => { + const repositories = { declarations: 'owner/repo' }; + + expect(() => { + Reporter.validateConfiguration(repositories); + }).not.to.throw(); + }); + }); + + context('with invalid configuration', () => { + context('when declarations key is missing', () => { + it('throws an error', () => { + const repositories = {}; + + expect(() => { + Reporter.validateConfiguration(repositories); + }).to.throw('Required configuration key "reporter.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + }); + }); + + context('when repository format is incorrect', () => { + it('throws an error', () => { + const repositories = { declarations: 'invalidFormat' }; + + expect(() => { + Reporter.validateConfiguration(repositories); + }).to.throw('Configuration entry "reporter.repositories.declarations" is expected to be a string in the format /, but received: "invalidFormat"'); + }); + }); + }); + }); +}); From 4496f52cecb3cf3cf74a725f3bc13c8724b69c35 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 12 Nov 2024 17:47:51 +0100 Subject: [PATCH 21/26] Use proper configuration key --- scripts/dataset/publish/gitlab/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/dataset/publish/gitlab/index.js b/scripts/dataset/publish/gitlab/index.js index d789c640..1b3c798f 100644 --- a/scripts/dataset/publish/gitlab/index.js +++ b/scripts/dataset/publish/gitlab/index.js @@ -20,7 +20,7 @@ export default async function publish({ let projectId = null; const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL'); - const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) + const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')) .pathname .split('/') .filter(Boolean); @@ -101,7 +101,7 @@ export default async function publish({ logger.debug(`package file id: ${packageFilesId}`); // use the package id to build the download url for the release - const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')}/-/package_files/${packageFilesId}/download`; + const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')}/-/package_files/${packageFilesId}/download`; // Create the release and link the package const formData = new FormData(); From 24875eb8f48b3946a2a07d8256ce0f86fc8a8b5c Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 13 Nov 2024 10:02:54 +0100 Subject: [PATCH 22/26] Improve test maintainability --- src/reporter/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reporter/index.test.js b/src/reporter/index.test.js index b0fd8412..14fb03f5 100644 --- a/src/reporter/index.test.js +++ b/src/reporter/index.test.js @@ -45,7 +45,7 @@ describe('Reporter', () => { expect(() => { Reporter.validateConfiguration(repositories); - }).to.throw('Required configuration key "reporter.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + }).to.throw().and.have.property('message').that.match(/Required configuration key.*was not found/); }); }); From 0bdd637170edd5b130f8ae0beb24999767142eec Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 13 Nov 2024 10:03:23 +0100 Subject: [PATCH 23/26] Improve comment --- src/reporter/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reporter/index.js b/src/reporter/index.js index 97542c9c..d3014b08 100644 --- a/src/reporter/index.js +++ b/src/reporter/index.js @@ -43,7 +43,7 @@ export default class Reporter { } /** - * Support for legacy config format where reporter configuration was nested under "githubIssues" + * Support for legacy config format where reporter configuration was nested under `githubIssues` * Example: * * ```json From 356a8b81654015dffb6fcaf196fa489ce86275c3 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 13 Nov 2024 17:31:03 +0100 Subject: [PATCH 24/26] Implement no-op clearCache in GitLab reporter --- src/reporter/gitlab/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/reporter/gitlab/index.js b/src/reporter/gitlab/index.js index 2999f0b1..a40168b4 100644 --- a/src/reporter/gitlab/index.js +++ b/src/reporter/gitlab/index.js @@ -379,4 +379,9 @@ export default class GitLab { generateSnapshotsBaseUrl(serviceName, termsType) { return `${this.baseURL}/${this.commonParams.owner}/${this.commonParams.repo}/-/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`; } + + // GitLab API responses are not cached unlike GitHub, so this method only exists to satisfy the Reporter interface contract + clearCache() { /* eslint-disable-line class-methods-use-this */ + logger.debug('Cache clearing not implemented for GitLab reporter as it is not needed'); + } } From 2dda758522704fb5b9d774c777bde007b0c375a6 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Wed, 13 Nov 2024 17:31:13 +0100 Subject: [PATCH 25/26] Remove obsolete log --- src/reporter/gitlab/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/reporter/gitlab/index.js b/src/reporter/gitlab/index.js index a40168b4..e427fbc8 100644 --- a/src/reporter/gitlab/index.js +++ b/src/reporter/gitlab/index.js @@ -262,7 +262,6 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.debug(`response data: ${JSON.stringify(res)}`); const issues = res; const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one From 7df61b67276807827d1a025d5d76eed7425fe525 Mon Sep 17 00:00:00 2001 From: Nicolas Dupont Date: Tue, 19 Nov 2024 09:28:48 +0100 Subject: [PATCH 26/26] Improve wording Co-authored-by: Matti Schneider --- scripts/dataset/publish/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dataset/publish/index.js b/scripts/dataset/publish/index.js index 08aa0c2d..6ed8ead0 100644 --- a/scripts/dataset/publish/index.js +++ b/scripts/dataset/publish/index.js @@ -11,5 +11,5 @@ export default function publishRelease({ archivePath, releaseDate, stats }) { return publishGitLab({ archivePath, releaseDate, stats }); } - throw new Error('No GitHub or GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.'); + throw new Error('No GitHub nor GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.'); }