Skip to content

Commit 3303593

Browse files
NdpntNdpnt
committed
Improve step name
1 parent 4924940 commit 3303593

File tree

11 files changed

+73
-63
lines changed

11 files changed

+73
-63
lines changed

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,16 @@
22

33
All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
44

5+
## Unreleased [major]
6+
7+
> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
8+
9+
### Changed
10+
11+
- **Breaking:** Rename `extractOnly` parameter to `technicalUpgradeOnly` in `track()` function to clarify that only the technical upgrade pass will run; ifyou are using the `track()` function directly with the `extractOnly` parameter, rename it to `technicalUpgradeOnly`
12+
- **Breaking:** Rename CLI flag from `--extract-only` to `--technical-upgrade-only` for consistency and clarity; if you are using the CLI with the `--extract-only` flag, rename it to `--technical-upgrade-only`
13+
- Enhance technical upgrade pass to fetch missing snapshots for newly added source documents in combined terms declarations
14+
515
## 9.2.0 - 2025-11-05
616

717
_Full changeset and discussions: [#1173](https://github.com/OpenTermsArchive/engine/pull/1173)._

‎bin/ota-track.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ program
1515
.description('Retrieve declared documents, record snapshots, extract versions and publish the resulting records')
1616
.option('-s, --services [serviceId...]', 'service IDs of services to track')
1717
.option('-t, --types [termsType...]', 'terms types to track')
18-
.option('-e, --extract-only', 'extract versions from existing snapshots with latest declarations and engine, without recording new snapshots')
18+
.option('-u, --technical-upgrade-only', 'only apply technical upgrades: regenerate versions from existing snapshots with updated declarations/engine, and fetch missing snapshots for newly added source documents; skip regular tracking')
1919
.option('--schedule', 'track automatically at a regular interval');
2020

2121
track(program.parse(process.argv).opts());

‎src/archivist/index.js‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,22 +128,22 @@ export default class Archivist extends events.EventEmitter {
128128
});
129129
}
130130

131-
async track({ services: servicesIds = this.servicesIds, types: termsTypes = [], extractOnly = false } = {}) {
131+
async track({ services: servicesIds = this.servicesIds, types: termsTypes = [], technicalUpgradeOnly = false } = {}) {
132132
const numberOfTerms = Service.getNumberOfTerms(this.services, servicesIds, termsTypes);
133133

134-
this.emit('trackingStarted', servicesIds.length, numberOfTerms, extractOnly);
134+
this.emit('trackingStarted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
135135

136136
await Promise.all([ launchHeadlessBrowser(), this.recorder.initialize() ]);
137137

138-
this.trackingQueue.concurrency = extractOnly ? MAX_PARALLEL_EXTRACTING : MAX_PARALLEL_TRACKING;
138+
this.trackingQueue.concurrency = technicalUpgradeOnly ? MAX_PARALLEL_EXTRACTING : MAX_PARALLEL_TRACKING;
139139

140140
servicesIds.forEach(serviceId => {
141141
this.services[serviceId].getTermsTypes().forEach(termsType => {
142142
if (termsTypes.length && !termsTypes.includes(termsType)) {
143143
return;
144144
}
145145

146-
this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), extractOnly });
146+
this.trackingQueue.push({ terms: this.services[serviceId].getTerms({ type: termsType }), technicalUpgradeOnly });
147147
});
148148
});
149149

@@ -153,14 +153,14 @@ export default class Archivist extends events.EventEmitter {
153153

154154
await Promise.all([ stopHeadlessBrowser(), this.recorder.finalize() ]);
155155

156-
this.emit('trackingCompleted', servicesIds.length, numberOfTerms, extractOnly);
156+
this.emit('trackingCompleted', servicesIds.length, numberOfTerms, technicalUpgradeOnly);
157157
}
158158

159-
async trackTermsChanges({ terms, extractOnly = false }) {
160-
if (!extractOnly) {
159+
async trackTermsChanges({ terms, technicalUpgradeOnly = false }) {
160+
if (!technicalUpgradeOnly) {
161161
await this.fetchAndRecordSnapshots(terms);
162162
} else {
163-
// In extractOnly mode (technical upgrade pass), fetch and record snapshots only for new source documents
163+
// In technical upgrade mode, fetch and record snapshots only for new source documents
164164
// that don't have existing snapshots yet (e.g., when a declaration is updated to add a new source document)
165165
await this.fetchAndRecordMissingSnapshots(terms);
166166
}
@@ -171,7 +171,7 @@ export default class Archivist extends events.EventEmitter {
171171
return;
172172
}
173173

174-
await this.recordVersion(terms, contents.join(Version.SOURCE_DOCUMENTS_SEPARATOR), extractOnly);
174+
await this.recordVersion(terms, contents.join(Version.SOURCE_DOCUMENTS_SEPARATOR), technicalUpgradeOnly);
175175
}
176176

177177
async fetchAndRecordSnapshots(terms) {
@@ -297,14 +297,14 @@ export default class Archivist extends events.EventEmitter {
297297
return contents;
298298
}
299299

300-
async recordVersion(terms, content, extractOnly) {
300+
async recordVersion(terms, content, technicalUpgradeOnly) {
301301
const record = new Version({
302302
content,
303303
snapshotIds: terms.sourceDocuments.map(sourceDocuments => sourceDocuments.snapshotId),
304304
serviceId: terms.service.id,
305305
termsType: terms.type,
306306
fetchDate: terms.fetchDate,
307-
isExtractOnly: extractOnly,
307+
isTechnicalUpgrade: technicalUpgradeOnly,
308308
metadata: { 'x-engine-version': PACKAGE_VERSION },
309309
});
310310

‎src/archivist/index.test.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ describe('Archivist', function () {
167167

168168
app.services[SERVICE_A_ID].getTerms({ type: SERVICE_A_TYPE }).sourceDocuments[0].contentSelectors = 'h1';
169169

170-
await app.track({ services: [ 'service·A', 'Service B!' ], extractOnly: true });
170+
await app.track({ services: [ 'service·A', 'Service B!' ], technicalUpgradeOnly: true });
171171

172172
const [reExtractedVersionCommit] = await gitVersion.log({ file: SERVICE_A_EXPECTED_VERSION_FILE_PATH });
173173

@@ -229,7 +229,7 @@ describe('Archivist', function () {
229229
}
230230
versionNotChangedSpy(record);
231231
});
232-
await app.track({ services, extractOnly: true });
232+
await app.track({ services, technicalUpgradeOnly: true });
233233
});
234234

235235
after(resetGitRepositories);

‎src/archivist/recorder/index.test.js‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -258,8 +258,8 @@ describe('Recorder', () => {
258258
expect(await record.content).to.equal(UPDATED_CONTENT);
259259
});
260260

261-
it('records in the version that it is not an extracted only version', () => {
262-
expect(record.isExtractOnly).to.equal(false);
261+
it('records in the version that it is not a technical upgrade version', () => {
262+
expect(record.isTechnicalUpgrade).to.equal(false);
263263
});
264264

265265
it('returns the record id', () => {
@@ -315,7 +315,7 @@ describe('Recorder', () => {
315315
content: CONTENT,
316316
snapshotIds: [SNAPSHOT_ID],
317317
fetchDate: FETCH_DATE,
318-
isExtractOnly: true,
318+
isTechnicalUpgrade: true,
319319
})));
320320

321321
record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
@@ -354,7 +354,7 @@ describe('Recorder', () => {
354354
content: UPDATED_CONTENT,
355355
snapshotIds: [SNAPSHOT_ID],
356356
fetchDate: FETCH_DATE_LATER,
357-
isExtractOnly: true,
357+
isTechnicalUpgrade: true,
358358
})));
359359

360360
record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);
@@ -366,8 +366,8 @@ describe('Recorder', () => {
366366
expect(await record.content).to.equal(UPDATED_CONTENT);
367367
});
368368

369-
it('records in the version that it is an extracted only version', () => {
370-
expect(record.isExtractOnly).to.equal(true);
369+
it('records in the version that it is an technical upgrade version', () => {
370+
expect(record.isTechnicalUpgrade).to.equal(true);
371371
});
372372

373373
it('returns the record id', () => {
@@ -395,7 +395,7 @@ describe('Recorder', () => {
395395
content: CONTENT,
396396
snapshotIds: [SNAPSHOT_ID],
397397
fetchDate: FETCH_DATE_LATER,
398-
isExtractOnly: true,
398+
isTechnicalUpgrade: true,
399399
})));
400400

401401
record = await recorder.versionsRepository.findLatest(SERVICE_ID, TYPE);

‎src/archivist/recorder/repositories/git/dataMapper.js‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import Version from '../../version.js';
77

88
export const COMMIT_MESSAGE_PREFIXES = {
99
startTracking: 'First record of',
10-
extractOnly: 'Apply technical or declaration upgrade on',
10+
technicalUpgrade: 'Apply technical or declaration upgrade on',
1111
update: 'Record new changes of',
1212
deprecated_startTracking: 'Start tracking',
1313
deprecated_refilter: 'Refilter',
@@ -22,9 +22,9 @@ const MULTIPLE_SOURCE_DOCUMENTS_PREFIX = 'This version was recorded after extrac
2222
export const COMMIT_MESSAGE_PREFIXES_REGEXP = new RegExp(`^(${Object.values(COMMIT_MESSAGE_PREFIXES).join('|')})`);
2323

2424
export function toPersistence(record, snapshotIdentiferTemplate) {
25-
const { serviceId, termsType, documentId, isExtractOnly, snapshotIds = [], mimeType, isFirstRecord, metadata } = record;
25+
const { serviceId, termsType, documentId, isTechnicalUpgrade, snapshotIds = [], mimeType, isFirstRecord, metadata } = record;
2626

27-
let prefix = isExtractOnly ? COMMIT_MESSAGE_PREFIXES.extractOnly : COMMIT_MESSAGE_PREFIXES.update;
27+
let prefix = isTechnicalUpgrade ? COMMIT_MESSAGE_PREFIXES.technicalUpgrade : COMMIT_MESSAGE_PREFIXES.update;
2828

2929
prefix = isFirstRecord ? COMMIT_MESSAGE_PREFIXES.startTracking : prefix;
3030

@@ -75,7 +75,7 @@ export function toDomain(commit) {
7575
const mimeTypeValue = mime.getType(relativeFilePath);
7676

7777
if (mimeTypeValue == mime.getType('markdown')) {
78-
attributes.isExtractOnly = message.startsWith(COMMIT_MESSAGE_PREFIXES.extractOnly) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
78+
attributes.isTechnicalUpgrade = message.startsWith(COMMIT_MESSAGE_PREFIXES.technicalUpgrade) || message.startsWith(COMMIT_MESSAGE_PREFIXES.deprecated_refilter);
7979
attributes.snapshotIds = snapshotIdsMatch;
8080

8181
return new Version(attributes);

‎src/archivist/recorder/repositories/git/index.test.js‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ describe('GitRepository', () => {
208208
});
209209
});
210210

211-
context('when it is an extracted only version', () => {
211+
context('when it is an technical upgrade version', () => {
212212
const EXTRACTED_ONLY_CONTENT = `${CONTENT} extracted only`;
213213

214214
before(async () => {
@@ -217,7 +217,7 @@ describe('GitRepository', () => {
217217
termsType: TERMS_TYPE,
218218
content: CONTENT,
219219
fetchDate: FETCH_DATE_EARLIER,
220-
})); // An extracted only version cannot be the first record
220+
})); // An technical upgrade version cannot be the first record
221221

222222
numberOfRecordsBefore = (await git.log()).length;
223223

@@ -226,7 +226,7 @@ describe('GitRepository', () => {
226226
termsType: TERMS_TYPE,
227227
content: EXTRACTED_ONLY_CONTENT,
228228
fetchDate: FETCH_DATE,
229-
isExtractOnly: true,
229+
isTechnicalUpgrade: true,
230230
snapshotIds: [SNAPSHOT_ID],
231231
})));
232232

@@ -245,8 +245,8 @@ describe('GitRepository', () => {
245245
expect(commit.hash).to.include(id);
246246
});
247247

248-
it('stores information that it is an extracted only version', () => {
249-
expect(commit.message).to.include(COMMIT_MESSAGE_PREFIXES.extractOnly);
248+
it('stores information that it is an technical upgrade version', () => {
249+
expect(commit.message).to.include(COMMIT_MESSAGE_PREFIXES.technicalUpgrade);
250250
});
251251
});
252252

@@ -518,7 +518,7 @@ describe('GitRepository', () => {
518518
serviceId: SERVICE_PROVIDER_ID,
519519
termsType: TERMS_TYPE,
520520
content: `${CONTENT} - updated 2`,
521-
isExtractOnly: true,
521+
isTechnicalUpgrade: true,
522522
fetchDate: FETCH_DATE_EARLIER,
523523
snapshotIds: [SNAPSHOT_ID],
524524
}));
@@ -569,7 +569,7 @@ describe('GitRepository', () => {
569569
serviceId: SERVICE_PROVIDER_ID,
570570
termsType: TERMS_TYPE,
571571
content: `${CONTENT} - updated 2`,
572-
isExtractOnly: true,
572+
isTechnicalUpgrade: true,
573573
fetchDate: FETCH_DATE_EARLIER,
574574
snapshotIds: [SNAPSHOT_ID],
575575
}));
@@ -678,7 +678,7 @@ describe('GitRepository', () => {
678678
serviceId: SERVICE_PROVIDER_ID,
679679
termsType: TERMS_TYPE,
680680
content: `${CONTENT} - updated 2`,
681-
isExtractOnly: true,
681+
isTechnicalUpgrade: true,
682682
fetchDate: FETCH_DATE_EARLIER,
683683
snapshotIds: [SNAPSHOT_ID],
684684
mimeType: HTML_MIME_TYPE,
@@ -1079,7 +1079,7 @@ describe('GitRepository', () => {
10791079
serviceId: SERVICE_PROVIDER_ID,
10801080
termsType: TERMS_TYPE,
10811081
content: `${CONTENT} - updated 2`,
1082-
isExtractOnly: true,
1082+
isTechnicalUpgrade: true,
10831083
fetchDate: FETCH_DATE_EARLIER,
10841084
mimeType: HTML_MIME_TYPE,
10851085
}));
@@ -1130,7 +1130,7 @@ describe('GitRepository', () => {
11301130
serviceId: SERVICE_PROVIDER_ID,
11311131
termsType: TERMS_TYPE,
11321132
content: `${CONTENT} - updated 2`,
1133-
isExtractOnly: true,
1133+
isTechnicalUpgrade: true,
11341134
fetchDate: FETCH_DATE_EARLIER,
11351135
mimeType: HTML_MIME_TYPE,
11361136
}));
@@ -1269,7 +1269,7 @@ describe('GitRepository', () => {
12691269
serviceId: SERVICE_PROVIDER_ID,
12701270
termsType: TERMS_TYPE,
12711271
content: `${CONTENT} - updated 2`,
1272-
isExtractOnly: true,
1272+
isTechnicalUpgrade: true,
12731273
fetchDate: FETCH_DATE_EARLIER,
12741274
mimeType: HTML_MIME_TYPE,
12751275
}));
@@ -1398,24 +1398,24 @@ describe('GitRepository', () => {
13981398
after(() => subject.removeAll());
13991399

14001400
describe('Records attributes', () => {
1401-
describe('#isExtractOnly', () => {
1401+
describe('#isTechnicalUpgrade', () => {
14021402
context('records with deprecated message', () => {
14031403
it('returns the proper value', async () => {
1404-
expect((await subject.findById(commits.deprecatedRefilter.id)).isExtractOnly).to.be.true;
1404+
expect((await subject.findById(commits.deprecatedRefilter.id)).isTechnicalUpgrade).to.be.true;
14051405
});
14061406

14071407
it('returns the proper value', async () => {
1408-
expect((await subject.findById(commits.deprecatedFirstRecord.id)).isExtractOnly).to.be.false;
1408+
expect((await subject.findById(commits.deprecatedFirstRecord.id)).isTechnicalUpgrade).to.be.false;
14091409
});
14101410
});
14111411

14121412
context('record with current message', () => {
14131413
it('returns the proper value', async () => {
1414-
expect((await subject.findById(commits.currentExtractOnly.id)).isExtractOnly).to.be.true;
1414+
expect((await subject.findById(commits.currentExtractOnly.id)).isTechnicalUpgrade).to.be.true;
14151415
});
14161416

14171417
it('returns the proper value', async () => {
1418-
expect((await subject.findById(commits.currentFirstRecord.id)).isExtractOnly).to.be.false;
1418+
expect((await subject.findById(commits.currentFirstRecord.id)).isTechnicalUpgrade).to.be.false;
14191419
});
14201420
});
14211421
});

‎src/archivist/recorder/repositories/mongo/dataMapper.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export function toPersistence(record) {
1717
}
1818

1919
export function toDomain(mongoDocument) {
20-
const { _id, serviceId, termsType, documentId, fetchDate, mimeType, isExtractOnly, isRefilter, isFirstRecord, snapshotIds, metadata } = mongoDocument;
20+
const { _id, serviceId, termsType, documentId, fetchDate, mimeType, isTechnicalUpgrade, isExtractOnly, isRefilter, isFirstRecord, snapshotIds, metadata } = mongoDocument;
2121

2222
const attributes = {
2323
id: _id.toString(),
@@ -27,7 +27,7 @@ export function toDomain(mongoDocument) {
2727
mimeType,
2828
fetchDate: new Date(fetchDate),
2929
isFirstRecord: Boolean(isFirstRecord),
30-
isExtractOnly: Boolean(isExtractOnly) || Boolean(isRefilter),
30+
isTechnicalUpgrade: Boolean(isTechnicalUpgrade) || Boolean(isExtractOnly) || Boolean(isRefilter),
3131
snapshotIds: snapshotIds?.map(snapshotId => snapshotId.toString()) || [],
3232
metadata,
3333
};

0 commit comments

Comments
 (0)