diff --git a/.env.example b/.env.example index 892ff5b3..bae1061c 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,7 @@ OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…' OTA_ENGINE_SMTP_PASSWORD='password' OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX +OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX +OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX +OTA_ENGINE_GITLAB_API_BASE_URL=https://gitlab.com/api/v4 +OTA_ENGINE_GITLAB_BASE_URL=https://gitlab.com \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index a6dc587c..72440274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [minor] + +### Added + +- Add GitLab functionalities + ## 2.5.0 - 2024-10-29 _Full changeset and discussions: [#1115](https://github.com/OpenTermsArchive/engine/pull/1115)._ diff --git a/config/default.json b/config/default.json index 0d3d82ea..8102f222 100644 --- a/config/default.json +++ b/config/default.json @@ -59,6 +59,7 @@ "dataset": { "title": "sandbox", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox", + "versionsRepositoryURLGitLab": "https://gitlab.com/ota-sandbox-example/sandbox", "publishingSchedule": "30 8 * * MON" } } diff --git a/package-lock.json b/package-lock.json index 5d7c06bc..37184856 100644 --- a/package-lock.json +++ b/package-lock.json @@ -171,6 +171,14 @@ "npm": ">=6" } }, + "node_modules/@accordproject/concerto-util/node_modules/axios": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", + "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@accordproject/markdown-cicero": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/@accordproject/markdown-cicero/-/markdown-cicero-0.15.2.tgz", @@ -324,6 +332,14 @@ "npm": ">=6" } }, + "node_modules/@accordproject/markdown-pdf/node_modules/axios": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", + "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@accordproject/markdown-pdf/node_modules/pdfjs-dist": { "version": "2.13.216", "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.13.216.tgz", @@ -4753,14 +4769,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/axios": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", - "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", - "dependencies": { - "follow-redirects": "^1.14.4" - } - }, "node_modules/b4a": { "version": "1.6.7", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", @@ -7296,9 +7304,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", diff --git a/scripts/dataset/assets/README.template.js b/scripts/dataset/assets/README.template.js index 1c63d117..1cda3428 100644 --- a/scripts/dataset/assets/README.template.js +++ b/scripts/dataset/assets/README.template.js @@ -3,10 +3,10 @@ import config from 'config'; const LOCALE = 'en-EN'; const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; -export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { +export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { return `# Open Terms Archive — ${title({ releaseDate })} -${body({ servicesCount, firstVersionDate, lastVersionDate })}`; +${body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL })}`; } export function title({ releaseDate }) { @@ -17,12 +17,10 @@ export function title({ releaseDate }) { return `${title} — ${releaseDate} dataset`; } -export function body({ servicesCount, firstVersionDate, lastVersionDate }) { +export function body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - const versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); - return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURL}](${versionsRepositoryURL}). diff --git a/scripts/dataset/export/index.js b/scripts/dataset/export/index.js index bc0a028a..3215768c 100644 --- a/scripts/dataset/export/index.js +++ b/scripts/dataset/export/index.js @@ -16,7 +16,7 @@ const fs = fsApi.promises; const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats -export default async function generate({ archivePath, releaseDate }) { +export default async function generate({ archivePath, releaseDate, versionsRepositoryURL }) { const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize(); const archive = await initializeArchive(archivePath); @@ -61,6 +61,7 @@ export default async function generate({ archivePath, releaseDate }) { releaseDate, firstVersionDate, lastVersionDate, + versionsRepositoryURL, }), { name: `${archive.basename}/README.md` }, ); diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 4c739686..66ea0a1f 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -6,6 +6,7 @@ import config from 'config'; import generateRelease from './export/index.js'; import logger from './logger/index.js'; import publishRelease from './publish/index.js'; +import publishReleaseGitLab from './publishGitLab/index.js'; export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) { const releaseDate = new Date(); @@ -14,7 +15,15 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start exporting dataset…'); - const stats = await generateRelease({ archivePath, releaseDate }); + const usesGitHub = (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined'); + const usesGitLab = (typeof process.env.OTA_ENGINE_GITLAB_TOKEN !== 'undefined'); + + let versionsRepositoryURL = ''; + + if (usesGitHub) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); + if (usesGitLab) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + const stats = await generateRelease({ archivePath, releaseDate, versionsRepositoryURL }); logger.info(`Dataset exported in ${archivePath}`); @@ -24,13 +33,25 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start publishing dataset…'); - const releaseUrl = await publishRelease({ - archivePath, - releaseDate, - stats, - }); + if (usesGitHub) { + const releaseUrl = await publishRelease({ + archivePath, + releaseDate, + stats, + }); - logger.info(`Dataset published to ${releaseUrl}`); + logger.info(`Dataset published to ${releaseUrl}`); + } + + if (usesGitLab) { + const releaseUrl = await publishReleaseGitLab({ + archivePath, + releaseDate, + stats, + }); + + logger.info(`Dataset published to ${releaseUrl}`); + } if (!shouldRemoveLocalCopy) { return; diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js new file mode 100644 index 00000000..e41c3e1c --- /dev/null +++ b/scripts/dataset/publishGitLab/index.js @@ -0,0 +1,135 @@ +import fsApi from 'fs'; +import path from 'path'; +import url from 'url'; + +import config from 'config'; +import dotenv from 'dotenv'; +import FormData from 'form-data'; +import nodeFetch from 'node-fetch'; + +import GitLab from '../../../src/reporterGitlab/gitlab.js'; +import * as readme from '../assets/README.template.js'; +import logger from '../logger/index.js'; + +dotenv.config(); + +const gitlabAPIUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; + +export default async function publishReleaseGitLab({ + archivePath, + releaseDate, + stats, +}) { + let projectId = null; + + const [ owner, repo ] = url + .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) + .pathname.split('/') + .filter(component => component); + const commonParams = { owner, repo }; + + try { + const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; + + const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + + options.method = 'GET'; + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, + options, + ); + const res = await response.json(); + + projectId = res.id; + } catch (error) { + logger.error(`Error while obtaining projectId: ${error}`); + projectId = null; + } + + const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag + + try { + let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + + options.method = 'POST'; + options.body = { + ref: 'main', + tag_name: tagName, + name: readme.title({ releaseDate }), + description: readme.body(stats), + }; + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + options.body = JSON.stringify(options.body); + + const releaseResponse = await nodeFetch( + `${gitlabAPIUrl}/projects/${projectId}/releases`, + options, + ); + const releaseRes = await releaseResponse.json(); + + const releaseId = releaseRes.commit.id; + + logger.info(`Created release with releaseId: ${releaseId}`); + + // Upload the package + options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + options.method = 'PUT'; + options.body = fsApi.createReadStream(archivePath); + + // restrict characters to the ones allowed by GitLab APIs + const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-'); + const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-'); + const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-'); + + logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`); + + const packageResponse = await nodeFetch( + `${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`, + options, + ); + const packageRes = await packageResponse.json(); + + const packageFilesId = packageRes.id; + + logger.debug(`package file id: ${packageFilesId}`); + + // use the package id to build the download url for the release + const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')}/-/package_files/${packageFilesId}/download`; + + // Create the release and link the package + const formData = new FormData(); + + formData.append('name', archivePath); + formData.append('url', publishedPackageUrl); + formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); + + options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + options.method = 'POST'; + options.headers = { + ...formData.getHeaders(), + ...options.headers, + }; + options.body = formData; + + const uploadResponse = await nodeFetch( + `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, + options, + ); + const uploadRes = await uploadResponse.json(); + const releaseUrl = uploadRes.direct_asset_url; + + return releaseUrl; + } catch (error) { + logger.error('Failed to create release or upload ZIP file:', error); + throw error; + } +} diff --git a/src/index.js b/src/index.js index 2083e180..517512e0 100644 --- a/src/index.js +++ b/src/index.js @@ -8,6 +8,7 @@ import Archivist from './archivist/index.js'; import logger from './logger/index.js'; import Notifier from './notifier/index.js'; import Reporter from './reporter/index.js'; +import ReporterGitlab from './reporterGitlab/index.js'; const require = createRequire(import.meta.url); @@ -72,6 +73,23 @@ export default async function track({ services, types, extractOnly, schedule }) logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored'); } + if (process.env.OTA_ENGINE_GITLAB_TOKEN) { + if (config.has('@opentermsarchive/engine.reporter.gitlabIssues.repositories.declarations')) { + try { + const reporter = new ReporterGitlab(config.get('@opentermsarchive/engine.reporter')); + + await reporter.initialize(); + archivist.attach(reporter); + } catch (error) { + logger.error('Cannot instantiate the ReporterGitlab module; it will be ignored:', error); + } + } else { + logger.warn('Configuration key "reporter.gitlabIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + } + } else { + logger.warn('Environment variable "OTA_ENGINE_GITLAB_TOKEN" was not found; the ReporterGitlab module will be ignored'); + } + if (!schedule) { await archivist.track({ services, types }); diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js new file mode 100644 index 00000000..cf4133a9 --- /dev/null +++ b/src/reporterGitlab/gitlab.js @@ -0,0 +1,366 @@ +import { createRequire } from 'module'; + +import HttpProxyAgent from 'http-proxy-agent'; +import HttpsProxyAgent from 'https-proxy-agent'; +import nodeFetch from 'node-fetch'; + +import logger from '../logger/index.js'; + +const require = createRequire(import.meta.url); + +export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; + +export default class GitLab { + static ISSUE_STATE_CLOSED = 'closed'; + static ISSUE_STATE_OPEN = 'opened'; + static ISSUE_STATE_ALL = 'all'; + + constructor(repository) { + const [ owner, repo ] = repository.split('/'); + + this.commonParams = { owner, repo }; + this.projectId = null; + const gitlabUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; + + this.gitlabUrl = gitlabUrl; + } + + async initialize() { + const options = GitLab.baseOptionsHttpReq(); + + try { + const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, + options, + ); + + const res = await response.json(); + + if (response.ok) { + this.projectId = res.id; + } else { + logger.error(`Error while obtaining projectId: ${JSON.strinfigy(res)}`); + this.projectId = null; + } + } catch (error) { + logger.error(`Error while obtaining projectId: ${error}`); + this.projectId = null; + } + this.MANAGED_LABELS = require('./labels.json'); + + const existingLabels = await this.getRepositoryLabels(); + const existingLabelsNames = existingLabels.map(label => label.name); + const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name)); + + if (missingLabels.length) { + logger.info(`The following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); + + for (const label of missingLabels) { + await this.createLabel({ /* eslint-disable-line no-await-in-loop */ + name: label.name, + color: label.color, + description: `${label.description} ${MANAGED_BY_OTA_MARKER}`, + }); + } + } + } + + async getRepositoryLabels() { + try { + const options = GitLab.baseOptionsHttpReq(); + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + options, + ); + const res = await response.json(); + + if (response.ok) { + return res; + } + + logger.error(`Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); + + return null; + } catch (error) { + logger.error(`Could not get labels: ${error}`); + } + } + + async createLabel({ name, color, description }) { + try { + const label = { + name, + color, + description, + }; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(label); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/labels`, + options, + ); + + const res = await response.json(); + + if (response.ok) { + logger.info(`New label created: ${res.name} , color: ${res.color}`); + } else { + logger.error(`createLabel response: ${JSON.stringify(res)}`); + } + } catch (error) { + logger.error(`Failed to create label: ${error}`); + } + } + + async createIssue({ title, description, labels }) { + try { + const issue = { + title, + labels, + description, + }; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(issue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues`, + options, + ); + + const res = await response.json(); + + if (response.ok) { + logger.info(`Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); + + return res; + } + + logger.error(`createIssue response: ${JSON.stringify(res)}`); + } catch (error) { + logger.error(`Could not create GitLab issue "${title}": ${error}`); + } + } + + async setIssueLabels({ issue, labels }) { + const newLabels = { labels }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(newLabels); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + try { + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, + ); + + const res = await response.json(); + + if (response.ok) { + logger.info(`Updated labels to GitLab issue #${issue.iid}`); + } else { + logger.error(`setIssueLabels response: ${JSON.stringify(res)}`); + } + } catch (error) { + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + } + } + + async openIssue(issue) { + const updateIssue = { state_event: 'reopen' }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(updateIssue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + try { + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, + ); + const res = await response.json(); + + if (response.ok) { + logger.info(`Opened GitLab issue #${res.iid}`); + } else { + logger.error(`openIssue response: ${JSON.stringify(res)}`); + } + } catch (error) { + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + } + } + + async closeIssue(issue) { + const updateIssue = { state_event: 'close' }; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(updateIssue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + try { + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, + ); + const res = await response.json(); + + if (response.ok) { + logger.info(`Closed GitLab issue #${issue.iid}`); + } else { + logger.error(`closeIssue response: ${JSON.stringify(res)}`); + } + } catch (error) { + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + } + } + + async getIssue({ title, ...searchParams }) { + try { + let apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + + if (searchParams.state == 'all') apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'GET'; + + const response = await nodeFetch(apiUrl, options); + const res = await response.json(); + + if (response.ok) { + logger.debug(`response data: ${JSON.stringify(res)}`); + const issues = res; + + const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one + + return issue; + } + + logger.error(`openIssue response: ${JSON.stringify(res)}`); + } catch (error) { + logger.error(`Could not find GitLab issue "${title}": ${error}`); + } + } + + async addCommentToIssue({ issue, comment }) { + const body = { body: comment }; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(body); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + try { + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, + options, + ); + const res = await response.json(); + + if (response.ok) { + logger.info(`Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); + + return res.body; + } + + logger.error(`openIssue response: ${JSON.stringify(res)}`); + } catch (error) { + logger.error(`Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); + } + } + + async closeIssueWithCommentIfExists({ title, comment }) { + const issue = await this.getIssue({ + title, + state: GitLab.ISSUE_STATE_OPEN, + }); + + // if issue does not exist in the "opened" state + if (!issue) { + return; + } + + await this.addCommentToIssue({ issue, comment }); + + return this.closeIssue(issue); + } + + async createOrUpdateIssue({ title, description, label }) { + const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_ALL }); + + if (!issue) { + return this.createIssue({ title, description, labels: [label] }); + } + + if (issue.state == GitLab.ISSUE_STATE_CLOSED) { + await this.openIssue(issue); + } + + const managedLabelsNames = this.MANAGED_LABELS.map(label => label.name); + const [managedLabel] = issue.labels.filter(label => + managedLabelsNames.includes(label.name)); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive + + if (managedLabel?.name == label) { + // if the label is already assigned to the issue, the error is redundant with the one already reported and no further action is necessary + return; + } + + const labelsNotManagedToKeep = issue.labels + .map(label => label.name) + .filter(label => !managedLabelsNames.includes(label)); + + await this.setIssueLabels({ + issue, + labels: [ label, ...labelsNotManagedToKeep ], + }); + await this.addCommentToIssue({ issue, comment: description }); + } + + static baseOptionsHttpReq(token = process.env.OTA_ENGINE_GITLAB_TOKEN) { + const options = {}; + + if (process.env.HTTPS_PROXY) { + options.agent = new HttpsProxyAgent(process.env.HTTPS_PROXY); + } else if (process.env.HTTP_PROXY) { + options.agent = new HttpProxyAgent(process.env.HTTP_PROXY); + } + + options.headers = { Authorization: `Bearer ${token}` }; + + return options; + } +} diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js new file mode 100644 index 00000000..ede97cd1 --- /dev/null +++ b/src/reporterGitlab/gitlab.test.js @@ -0,0 +1,532 @@ +import { createRequire } from 'module'; + +import { expect } from 'chai'; +import nock from 'nock'; + +import GitLab from './gitlab.js'; + +const require = createRequire(import.meta.url); + +describe('GitLab', function () { + this.timeout(5000); + + let MANAGED_LABELS; + let gitlab; + let gitlabApiUrl = ''; + let reqHeaders; + const projectId = '4'; + + before(() => { + MANAGED_LABELS = require('./labels.json'); + gitlab = new GitLab('owner/repo'); + gitlab.projectId = projectId; + gitlabApiUrl = gitlab.gitlabUrl; + reqHeaders = { reqheaders: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }; + }); + + describe('#Gitlab_initialize', () => { + const scopes = []; + + before(async () => { + const existingLabels = MANAGED_LABELS.slice(0, -2); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${encodeURIComponent('owner/repo')}`) + .reply(200, { id: 4 }); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, existingLabels); + + const missingLabels = MANAGED_LABELS.slice(-2); + + for (const label of missingLabels) { + scopes.push(nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/labels`) + .reply(200, { name: label.name })); + } + + await gitlab.initialize(); + }); + + after(nock.cleanAll); + + it('should create missing labels', () => { + scopes.forEach(scope => expect(scope.isDone()).to.be.true); + }); + }); + + describe('#Gitlab_getRepositoryLabels', () => { + let scope; + let result; + const LABELS = [{ name: 'bug' }, { name: 'enhancement' }]; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, LABELS); + + result = await gitlab.getRepositoryLabels(); + }); + + after(nock.cleanAll); + + it('fetches repository labels', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the repository labels', () => { + expect(result).to.deep.equal(LABELS); + }); + }); + + describe('#Gitlab_createLabel', () => { + let scope; + const LABEL = { name: 'new_label', color: 'ffffff' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/labels`, body => body.name === LABEL.name) + .reply(200, LABEL); + + await gitlab.createLabel(LABEL); + }); + + after(nock.cleanAll); + + it('creates the new label', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_createIssue', () => { + let scope; + let result; + + const ISSUE = { + title: 'New Issue', + description: 'Description of the new issue', + labels: ['bug'], + }; + const CREATED_ISSUE = { + title: 'New Issue', + description: 'Description of the new issue', + labels: ['bug'], + iid: 555, + web_url: 'https://example.com/test/test', + }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues`) + .reply(200, CREATED_ISSUE); + + result = await gitlab.createIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('creates the new issue', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the created issue', () => { + expect(result).to.deep.equal(CREATED_ISSUE); + }); + }); + + describe('#Gitlab_setIssueLabels', () => { + let scope; + const issue = { + iid: 123, + title: 'test issue', + }; + const labels = [ 'bug', 'enhancement' ]; + + const response = { + iid: 123, + labels, + }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${issue.iid}`, { labels }) + .reply(200, response); + + await gitlab.setIssueLabels({ issue, labels }); + }); + + after(nock.cleanAll); + + it('sets labels on the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_openIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'issue reopened' }; + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const response = { iid: 123 }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + .reply(200, response); + + await gitlab.openIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('opens the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_closeIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'close issue' }; + const EXPECTED_REQUEST_BODY = { state_event: 'close' }; + const response = { iid: 123 }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + .reply(200, response); + + await gitlab.closeIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('closes the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_getIssue', () => { + let scope; + let result; + + const ISSUE = { number: 123, title: 'Test Issue' }; + const ANOTHER_ISSUE = { number: 124, title: 'Test Issue 2' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [ ISSUE, ANOTHER_ISSUE ]); + + result = await gitlab.getIssue({ title: ISSUE.title }); + }); + + after(nock.cleanAll); + + it('searches for the issue', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the expected issue', () => { + expect(result).to.deep.equal(ISSUE); + }); + }); + + describe('#Gitlab_addCommentToIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'Test Issue' }; + const COMMENT = 'Test comment'; + const response = { iid: 123, id: 23, body: 'Test comment' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, response); + + await gitlab.addCommentToIssue({ issue: ISSUE, comment: COMMENT }); + }); + + after(nock.cleanAll); + + it('adds the comment to the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_closeIssueWithCommentIfExists', () => { + after(nock.cleanAll); + + context('when the issue exists and is open', () => { + const ISSUE = { + iid: 123, + title: 'Open Issue', + state: GitLab.ISSUE_STATE_OPEN, + }; + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [ISSUE]); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + + it('closes the issue', () => { + expect(closeIssueScope.isDone()).to.be.true; + }); + }); + + context('when the issue exists and is closed', () => { + const ISSUE = { + number: 123, + title: 'Closed Issue', + state: GitLab.ISSUE_STATE_CLOSED, + }; + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, []); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); + }); + + it('does not add comment', () => { + expect(addCommentScope.isDone()).to.be.false; + }); + + it('does not attempt to close the issue', () => { + expect(closeIssueScope.isDone()).to.be.false; + }); + }); + + context('when the issue does not exist', () => { + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const TITLE = 'Non-existent Issue'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(TITLE)}&per_page=100`) + .reply(200, []); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(/\/projects\/\d+\/issues\/\d+\/notes/, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(/\/projects\/\d+\/issues\/\d+/, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: TITLE, comment: COMMENT }); + }); + + it('does not attempt to add comment', () => { + expect(addCommentScope.isDone()).to.be.false; + }); + + it('does not attempt to close the issue', () => { + expect(closeIssueScope.isDone()).to.be.false; + }); + }); + }); + + describe('#Gitlab_createOrUpdateIssue', () => { + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${encodeURIComponent('owner/repo')}`) + .reply(200, { id: 4 }); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, MANAGED_LABELS); + + await gitlab.initialize(); + }); + + context('when the issue does not exist', () => { + let createIssueScope; + const ISSUE_TO_CREATE = { + title: 'New Issue', + description: 'Description of the new issue', + label: 'bug', + }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE_TO_CREATE.title)}&per_page=100`) + .reply(200, []); // Simulate that there is no issues on the repository + + createIssueScope = nock(gitlabApiUrl, reqHeaders) + .post( + `/projects/${projectId}/issues`, + { + title: ISSUE_TO_CREATE.title, + description: ISSUE_TO_CREATE.description, + labels: [ISSUE_TO_CREATE.label], + }, + ) + .reply(200, { iid: 123, web_url: 'https://example.com/test/test' }); + + await gitlab.createOrUpdateIssue(ISSUE_TO_CREATE); + }); + + it('creates the issue', () => { + expect(createIssueScope.isDone()).to.be.true; + }); + }); + + context('when the issue already exists', () => { + const ISSUE = { + title: 'Existing Issue', + description: 'New comment', + label: 'location', + }; + + context('when issue is closed', () => { + let setIssueLabelsScope; + let addCommentScope; + let openIssueScope; + + const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = { + iid: 123, + title: ISSUE.title, + description: ISSUE.description, + labels: [{ name: 'selectors' }], + state: GitLab.ISSUE_STATE_CLOSED, + }; + + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const responseIssuereopened = { iid: 123 }; + const responseSetLabels = { + iid: 123, + labels: ['location'], + }; + const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description }; + const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); + + openIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + .reply(200, responseIssuereopened); + + setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + .reply(200, responseSetLabels); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + .reply(200, responseAddcomment); + + await gitlab.createOrUpdateIssue(ISSUE); + }); + + it('reopens the issue', () => { + expect(openIssueScope.isDone()).to.be.true; + }); + + it("updates the issue's label", () => { + expect(setIssueLabelsScope.isDone()).to.be.true; + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + }); + + context('when issue is already opened', () => { + let setIssueLabelsScope; + let addCommentScope; + let openIssueScope; + + const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = { + number: 123, + title: ISSUE.title, + description: ISSUE.description, + labels: [{ name: 'selectors' }], + state: GitLab.ISSUE_STATE_OPEN, + }; + + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const responseIssuereopened = { iid: 123 }; + const responseSetLabels = { + iid: 123, + labels: ['location'], + }; + const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description }; + const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); + + openIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + .reply(200, responseIssuereopened); + + setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + .reply(200, responseSetLabels); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + .reply(200, responseAddcomment); + + await gitlab.createOrUpdateIssue(ISSUE); + }); + + it('does not change the issue state', () => { + expect(openIssueScope.isDone()).to.be.false; + }); + + it("updates the issue's label", () => { + expect(setIssueLabelsScope.isDone()).to.be.true; + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + }); + }); + }); +}); diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js new file mode 100644 index 00000000..be516118 --- /dev/null +++ b/src/reporterGitlab/index.js @@ -0,0 +1,147 @@ +import mime from 'mime'; + +import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; + +import GitLab from './gitlab.js'; + +const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/'; +const DOC_URL = 'https://docs.opentermsarchive.org'; +const REPO_URL = process.env.OTA_ENGINE_GITLAB_BASE_URL; + +const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = { + 'has no match': 'selectors', + 'HTTP code 404': 'location', + 'HTTP code 403': '403', + 'HTTP code 429': '429', + 'HTTP code 500': '500', + 'HTTP code 502': '502', + 'HTTP code 503': '503', + 'Timed out after': 'timeout', + 'getaddrinfo EAI_AGAIN': 'EAI_AGAIN', + 'getaddrinfo ENOTFOUND': 'ENOTFOUND', + 'Response is empty': 'empty response', + 'unable to verify the first certificate': 'first certificate', + 'certificate has expired': 'certificate expired', + 'maximum redirect reached': 'redirects', +}; + +function getLabelNameFromError(error) { + return ERROR_MESSAGE_TO_ISSUE_LABEL_MAP[Object.keys(ERROR_MESSAGE_TO_ISSUE_LABEL_MAP).find(substring => error.toString().includes(substring))] || 'to clarify'; +} + +// In the following class, it is assumed that each issue is managed using its title as a unique identifier +export default class Reporter { + constructor(config) { + const { repositories } = config.gitlabIssues; + + for (const repositoryType of Object.keys(repositories)) { + if (!repositories[repositoryType].includes('/') || repositories[repositoryType].includes('https://')) { + throw new Error(`Configuration entry "reporter.gitlabIssues.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); + } + } + + this.gitlab = new GitLab(repositories.declarations); + this.repositories = repositories; + } + + initialize() { + return this.gitlab.initialize(); + } + + async onVersionRecorded(version) { + await this.gitlab.closeIssueWithCommentIfExists({ + title: Reporter.generateTitleID(version.serviceId, version.termsType), + comment: `### Tracking resumed + +A new version has been recorded.`, + }); + } + + async onVersionNotChanged(version) { + await this.gitlab.closeIssueWithCommentIfExists({ + title: Reporter.generateTitleID(version.serviceId, version.termsType), + comment: `### Tracking resumed + +No changes were found in the last run, so no new version has been recorded.`, + }); + } + + async onFirstVersionRecorded(version) { + return this.onVersionRecorded(version); + } + + async onInaccessibleContent(error, terms) { + await this.gitlab.createOrUpdateIssue({ + title: Reporter.generateTitleID(terms.service.id, terms.type), + description: this.generateDescription({ error, terms }), + label: getLabelNameFromError(error), + }); + } + + generateDescription({ error, terms }) { + const date = new Date(); + const currentFormattedDate = date.toLocaleDateString('en-GB', { year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric', timeZoneName: 'short', timeZone: 'UTC' }); + const validUntil = toISODateWithoutMilliseconds(date); + + const hasSnapshots = terms.sourceDocuments.every(sourceDocument => sourceDocument.snapshotId); + + const contributionToolParams = new URLSearchParams({ + json: JSON.stringify(terms.toPersistence()), + destination: this.repositories.declarations, + step: '2', + }); + const contributionToolUrl = `${CONTRIBUTION_TOOL_URL}?${contributionToolParams}`; + + const latestDeclarationLink = `[Latest declaration](${REPO_URL}/${this.repositories.declarations}/-/blob/main/declarations/${encodeURIComponent(terms.service.name)}.json)`; + const latestVersionLink = `[Latest version](${REPO_URL}/${this.repositories.versions}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}.md)`; + const snapshotsBaseUrl = `${REPO_URL}/${this.repositories.snapshots}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}`; + const latestSnapshotsLink = terms.hasMultipleSourceDocuments + ? `Latest snapshots:\n - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n - ')}` + : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`; + + /* eslint-disable no-irregular-whitespace */ + return ` +### No version of the \`${terms.type}\` of service \`${terms.service.name}\` is recorded anymore since ${currentFormattedDate} + +The source document${terms.hasMultipleSourceDocuments ? 's have' : ' has'}${hasSnapshots ? ' ' : ' not '}been recorded in ${terms.hasMultipleSourceDocuments ? 'snapshots' : 'a snapshot'}, ${hasSnapshots ? 'but ' : 'thus '} no version can be [extracted](${DOC_URL}/#tracking-terms). +${hasSnapshots ? 'After correction, it might still be possible to recover the missed versions.' : ''} + +### What went wrong + +- ${error.reasons.join('\n- ')} + +### How to resume tracking + +First of all, check if the source documents are accessible through a web browser: + +- [ ] ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.location}](${sourceDocument.location})`).join('\n- [ ] ')} + +#### If the source documents are accessible through a web browser + +[Edit the declaration](${contributionToolUrl}): +- Try updating the selectors. +- Try switching client scripts on with expert mode. + +#### If the source documents are not accessible anymore + +- If the source documents have moved, find their new location and [update it](${contributionToolUrl}). +- If these terms have been removed, move them from the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. +- If the service has closed, move the entire contents of the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. + +#### If none of the above works + +If the source documents are accessible in a browser but fetching them always fails from the Open Terms Archive server, this is most likely because the service provider has blocked the Open Terms Archive robots from accessing its content. In this case, updating the declaration will not enable resuming tracking. Only an agreement with the service provider, an engine upgrade, or some technical workarounds provided by the administrator of this collection’s server might resume tracking. + +### References + +- ${latestDeclarationLink} +${this.repositories.versions ? `- ${latestVersionLink}` : ''} +${this.repositories.snapshots ? `- ${latestSnapshotsLink}` : ''} +`; + /* eslint-enable no-irregular-whitespace */ + } + + static generateTitleID(serviceId, type) { + return `\`${serviceId}\` ‧ \`${type}\` ‧ not tracked anymore`; + } +} diff --git a/src/reporterGitlab/labels.json b/src/reporterGitlab/labels.json new file mode 100644 index 00000000..8d73f4e8 --- /dev/null +++ b/src/reporterGitlab/labels.json @@ -0,0 +1,77 @@ +[ + { + "name": "403", + "color": "#0b08a0", + "description": "Fetching fails with a 403 (forbidden) HTTP code" + }, + { + "name": "429", + "color": "#0b08a0", + "description": "Fetching fails with a 429 (too many requests) HTTP code" + }, + { + "name": "500", + "color": "#0b08a0", + "description": "Fetching fails with a 500 (internal server error) HTTP code" + }, + { + "name": "502", + "color": "#0b08a0", + "description": "Fetching fails with a 502 (bad gateway) HTTP code" + }, + { + "name": "503", + "color": "#0b08a0", + "description": "Fetching fails with a 503 (service unavailable) HTTP code" + }, + { + "name": "certificate expired", + "color": "#0b08a0", + "description": "Fetching fails because the domain SSL certificate has expired" + }, + { + "name": "EAI_AGAIN", + "color": "#0b08a0", + "description": "Fetching fails because the domain fails to resolve on DNS" + }, + { + "name": "ENOTFOUND", + "color": "#0b08a0", + "description": "Fetching fails because the domain fails to resolve on DNS" + }, + { + "name": "empty response", + "color": "#0b08a0", + "description": "Fetching fails with a “response is empty” error" + }, + { + "name": "first certificate", + "color": "#0b08a0", + "description": "Fetching fails with an “unable to verify the first certificate” error" + }, + { + "name": "redirects", + "color": "#0b08a0", + "description": "Fetching fails with a “too many redirects” error" + }, + { + "name": "timeout", + "color": "#0b08a0", + "description": "Fetching fails with a timeout error" + }, + { + "name": "to clarify", + "color": "#0496ff", + "description": "Default failure label" + }, + { + "name": "selectors", + "color": "#FBCA04", + "description": "Extraction selectors are outdated" + }, + { + "name": "location", + "color": "#FBCA04", + "description": "Fetch location is outdated" + } +] diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js new file mode 100644 index 00000000..a1e61f2d --- /dev/null +++ b/src/reporterGitlab/labels.test.js @@ -0,0 +1,30 @@ +import { createRequire } from 'module'; + +import chai from 'chai'; + +import { MANAGED_BY_OTA_MARKER } from './gitlab.js'; + +const require = createRequire(import.meta.url); + +const { expect } = chai; +const labels = require('./labels.json'); + +const GITLAB_LABEL_DESCRIPTION_MAX_LENGTH = 255; + +describe('Reporter GitLab labels', () => { + labels.forEach(label => { + describe(`"${label.name}"`, () => { + it('complies with the GitLab character limit for descriptions', () => { + const descriptionLength = label.description.length + MANAGED_BY_OTA_MARKER.length; + + expect(descriptionLength).to.be.lessThan(GITLAB_LABEL_DESCRIPTION_MAX_LENGTH); + }); + + it('complies with the GitLab constraints for color', () => { + const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` + + expect(validHexColorRegex.test(label.color)).to.be.true; + }); + }); + }); +});