Skip to content

Commit 7c65a48

Browse files
NdpntNdpnt
authored
Refactor GitLab integration (#1116)
2 parents 4cdde4b + 7df61b6 commit 7c65a48

File tree

18 files changed

+1388
-76
lines changed

18 files changed

+1388
-76
lines changed

‎.env.example‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
11
OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…'
22
OTA_ENGINE_SMTP_PASSWORD='password'
3+
4+
# If both GitHub and GitLab tokens are defined, GitHub takes precedence for dataset publishing
35
OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX
6+
7+
OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX
8+
OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX

‎CHANGELOG.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
44

5+
## Unreleased [minor]
6+
7+
> Development of this release was supported by the [European Union](https://commission.europa.eu/) and the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
8+
9+
### Added
10+
11+
- Add support for GitLab for issue reporting
12+
- Add support for GitLab Releases for publishing datasets
13+
514
## 2.5.0 - 2024-10-29
615

716
_Full changeset and discussions: [#1115](https://github.com/OpenTermsArchive/engine/pull/1115)._

‎package-lock.json‎

Lines changed: 19 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/dataset/publish/github/index.js‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import fsApi from 'fs';
2+
import path from 'path';
3+
import url from 'url';
4+
5+
import config from 'config';
6+
import { Octokit } from 'octokit';
7+
8+
import * as readme from '../../assets/README.template.js';
9+
10+
export default async function publish({ archivePath, releaseDate, stats }) {
11+
const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
12+
13+
const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
14+
15+
const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
16+
17+
const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
18+
owner,
19+
repo,
20+
tag_name: tagName,
21+
name: readme.title({ releaseDate }),
22+
body: readme.body(stats),
23+
});
24+
25+
await octokit.rest.repos.uploadReleaseAsset({
26+
data: fsApi.readFileSync(archivePath),
27+
headers: {
28+
'content-type': 'application/zip',
29+
'content-length': fsApi.statSync(archivePath).size,
30+
},
31+
name: path.basename(archivePath),
32+
url: uploadUrl,
33+
});
34+
35+
return releaseUrl;
36+
}

‎scripts/dataset/publish/gitlab/index.js‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import fsApi from 'fs';
2+
import path from 'path';
3+
4+
import config from 'config';
5+
import dotenv from 'dotenv';
6+
import FormData from 'form-data';
7+
import nodeFetch from 'node-fetch';
8+
9+
import GitLab from '../../../../src/reporter/gitlab/index.js';
10+
import * as readme from '../../assets/README.template.js';
11+
import logger from '../../logger/index.js';
12+
13+
dotenv.config();
14+
15+
export default async function publish({
16+
archivePath,
17+
releaseDate,
18+
stats,
19+
}) {
20+
let projectId = null;
21+
const gitlabAPIUrl = config.get('@opentermsarchive/engine.dataset.apiBaseURL');
22+
23+
const [ owner, repo ] = new URL(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'))
24+
.pathname
25+
.split('/')
26+
.filter(Boolean);
27+
const commonParams = { owner, repo };
28+
29+
try {
30+
const repositoryPath = `${commonParams.owner}/${commonParams.repo}`;
31+
32+
const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
33+
34+
options.method = 'GET';
35+
options.headers = {
36+
'Content-Type': 'application/json',
37+
...options.headers,
38+
};
39+
40+
const response = await nodeFetch(
41+
`${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`,
42+
options,
43+
);
44+
const res = await response.json();
45+
46+
projectId = res.id;
47+
} catch (error) {
48+
logger.error(`Error while obtaining projectId: ${error}`);
49+
projectId = null;
50+
}
51+
52+
const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
53+
54+
try {
55+
let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
56+
57+
options.method = 'POST';
58+
options.body = {
59+
ref: 'main',
60+
tag_name: tagName,
61+
name: readme.title({ releaseDate }),
62+
description: readme.body(stats),
63+
};
64+
options.headers = {
65+
'Content-Type': 'application/json',
66+
...options.headers,
67+
};
68+
69+
options.body = JSON.stringify(options.body);
70+
71+
const releaseResponse = await nodeFetch(
72+
`${gitlabAPIUrl}/projects/${projectId}/releases`,
73+
options,
74+
);
75+
const releaseRes = await releaseResponse.json();
76+
77+
const releaseId = releaseRes.commit.id;
78+
79+
logger.info(`Created release with releaseId: ${releaseId}`);
80+
81+
// Upload the package
82+
options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
83+
options.method = 'PUT';
84+
options.body = fsApi.createReadStream(archivePath);
85+
86+
// restrict characters to the ones allowed by GitLab APIs
87+
const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-');
88+
const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-');
89+
const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-');
90+
91+
logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`);
92+
93+
const packageResponse = await nodeFetch(
94+
`${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`,
95+
options,
96+
);
97+
const packageRes = await packageResponse.json();
98+
99+
const packageFilesId = packageRes.id;
100+
101+
logger.debug(`package file id: ${packageFilesId}`);
102+
103+
// use the package id to build the download url for the release
104+
const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')}/-/package_files/${packageFilesId}/download`;
105+
106+
// Create the release and link the package
107+
const formData = new FormData();
108+
109+
formData.append('name', archivePath);
110+
formData.append('url', publishedPackageUrl);
111+
formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) });
112+
113+
options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN);
114+
options.method = 'POST';
115+
options.headers = {
116+
...formData.getHeaders(),
117+
...options.headers,
118+
};
119+
options.body = formData;
120+
121+
const uploadResponse = await nodeFetch(
122+
`${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`,
123+
options,
124+
);
125+
const uploadRes = await uploadResponse.json();
126+
const releaseUrl = uploadRes.direct_asset_url;
127+
128+
return releaseUrl;
129+
} catch (error) {
130+
logger.error('Failed to create release or upload ZIP file:', error);
131+
throw error;
132+
}
133+
}

‎scripts/dataset/publish/index.js‎

Lines changed: 11 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,15 @@
1-
import fsApi from 'fs';
2-
import path from 'path';
3-
import url from 'url';
1+
import publishGitHub from './github/index.js';
2+
import publishGitLab from './gitlab/index.js';
43

5-
import config from 'config';
6-
import { Octokit } from 'octokit';
4+
export default function publishRelease({ archivePath, releaseDate, stats }) {
5+
// If both GitHub and GitLab tokens are defined, GitHub takes precedence
6+
if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
7+
return publishGitHub({ archivePath, releaseDate, stats });
8+
}
79

8-
import * as readme from '../assets/README.template.js';
10+
if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
11+
return publishGitLab({ archivePath, releaseDate, stats });
12+
}
913

10-
export default async function publish({ archivePath, releaseDate, stats }) {
11-
const octokit = new Octokit({ auth: process.env.OTA_ENGINE_GITHUB_TOKEN });
12-
13-
const [ owner, repo ] = url.parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL')).pathname.split('/').filter(component => component);
14-
15-
const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
16-
17-
const { data: { upload_url: uploadUrl, html_url: releaseUrl } } = await octokit.rest.repos.createRelease({
18-
owner,
19-
repo,
20-
tag_name: tagName,
21-
name: readme.title({ releaseDate }),
22-
body: readme.body(stats),
23-
});
24-
25-
await octokit.rest.repos.uploadReleaseAsset({
26-
data: fsApi.readFileSync(archivePath),
27-
headers: {
28-
'content-type': 'application/zip',
29-
'content-length': fsApi.statSync(archivePath).size,
30-
},
31-
name: path.basename(archivePath),
32-
url: uploadUrl,
33-
});
34-
35-
return releaseUrl;
14+
throw new Error('No GitHub nor GitLab token found in environment variables (OTA_ENGINE_GITHUB_TOKEN or OTA_ENGINE_GITLAB_TOKEN). Cannot publish the dataset without authentication.');
3615
}

‎src/index.js‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -55,21 +55,17 @@ export default async function track({ services, types, extractOnly, schedule })
5555
logger.warn('Environment variable "OTA_ENGINE_SENDINBLUE_API_KEY" was not found; the Notifier module will be ignored');
5656
}
5757

58-
if (process.env.OTA_ENGINE_GITHUB_TOKEN) {
59-
if (config.has('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations')) {
60-
try {
61-
const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
62-
63-
await reporter.initialize();
64-
archivist.attach(reporter);
65-
} catch (error) {
66-
logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
67-
}
68-
} else {
69-
logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
58+
if (process.env.OTA_ENGINE_GITHUB_TOKEN || process.env.OTA_ENGINE_GITLAB_TOKEN) {
59+
try {
60+
const reporter = new Reporter(config.get('@opentermsarchive/engine.reporter'));
61+
62+
await reporter.initialize();
63+
archivist.attach(reporter);
64+
} catch (error) {
65+
logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
7066
}
7167
} else {
72-
logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
68+
logger.warn('Environment variable with token for GitHub or GitLab was not found; the Reporter module will be ignored');
7369
}
7470

7571
if (!schedule) {

‎src/reporter/factory.js‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import GitHub from './github/index.js';
2+
import GitLab from './gitlab/index.js';
3+
4+
export function createReporter(config) {
5+
switch (config.type) {
6+
case 'github':
7+
return new GitHub(config.repositories.declarations);
8+
case 'gitlab':
9+
return new GitLab(config.repositories.declarations, config.baseURL, config.apiBaseURL);
10+
default:
11+
throw new Error(`Unsupported reporter type: ${config.type}`);
12+
}
13+
}

‎src/reporter/github.js‎ renamed to ‎src/reporter/github/index.js‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { createRequire } from 'module';
22

33
import { Octokit } from 'octokit';
44

5-
import logger from '../logger/index.js';
5+
import logger from '../../logger/index.js';
66

77
const require = createRequire(import.meta.url);
88

@@ -14,7 +14,7 @@ export default class GitHub {
1414
static ISSUE_STATE_ALL = 'all';
1515

1616
constructor(repository) {
17-
const { version } = require('../../package.json');
17+
const { version } = require('../../../package.json');
1818

1919
this.octokit = new Octokit({
2020
auth: process.env.OTA_ENGINE_GITHUB_TOKEN,
@@ -198,4 +198,16 @@ export default class GitHub {
198198
logger.error(`Failed to update issue "${title}": ${error.stack}`);
199199
}
200200
}
201+
202+
generateDeclarationURL(serviceName) {
203+
return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/declarations/${encodeURIComponent(serviceName)}.json`;
204+
}
205+
206+
generateVersionURL(serviceName, termsType) {
207+
return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}.md`;
208+
}
209+
210+
generateSnapshotsBaseUrl(serviceName, termsType) {
211+
return `https://github.com/${this.commonParams.owner}/${this.commonParams.repo}/blob/main/${encodeURIComponent(serviceName)}/${encodeURIComponent(termsType)}`;
212+
}
201213
}

‎src/reporter/github.test.js‎ renamed to ‎src/reporter/github/index.test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { createRequire } from 'module';
33
import { expect } from 'chai';
44
import nock from 'nock';
55

6-
import GitHub from './github.js';
6+
import GitHub from './index.js';
77

88
const require = createRequire(import.meta.url);
99

0 commit comments

Comments
 (0)