Skip to content

Commit c19d3c8

Browse files
akunyakuny
committed
1745: add new snapshot and make language consistent
1 parent c788428 commit c19d3c8

File tree

4 files changed

+121
-30
lines changed

4 files changed

+121
-30
lines changed

‎libs/database/src/websites/websites.service.ts‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,27 @@ export class WebsiteService {
3131
}
3232

3333
async findLiveSnapshotResults(): Promise<Website[]> {
34+
const queryBuilder = this.website
35+
.createQueryBuilder('website')
36+
.innerJoinAndSelect('website.coreResult', 'coreResult')
37+
.andWhere('coreResult.finalUrlIsLive = :isLive', { isLive: true })
38+
.andWhere('coreResult.finalUrlMIMEType NOT IN (:...mimeTypes)', {
39+
mimeTypes: [
40+
'image/jpeg',
41+
'application/xml',
42+
'application/json',
43+
'text/xml',
44+
],
45+
})
46+
.orderBy({
47+
'coreResult.targetUrlBaseDomain': 'ASC',
48+
'website.url': 'ASC',
49+
});
50+
51+
return await queryBuilder.getMany();
52+
}
53+
54+
async findLiveFilteredSnapshotResults(): Promise<Website[]> {
3455
const queryBuilder = this.website
3556
.createQueryBuilder('website')
3657
.innerJoinAndSelect('website.coreResult', 'coreResult')
@@ -52,12 +73,14 @@ export class WebsiteService {
5273
return await queryBuilder.getMany();
5374
}
5475

55-
async findUniqueSnapshotResults(): Promise<Website[]> {
76+
async findLiveFilteredUniqueSnapshotResults(): Promise<Website[]> {
5677
const queryBuilder = this.website
5778
.createQueryBuilder('website')
5879
.innerJoinAndSelect('website.coreResult', 'coreResult')
5980
.andWhere('coreResult.filter = :isFilter', { isFilter: false })
60-
.andWhere('coreResult.targetUrlRedirects = :isRedirect', { isRedirect: false })
81+
.andWhere('coreResult.targetUrlRedirects = :isRedirect', {
82+
isRedirect: false,
83+
})
6184
.andWhere('coreResult.finalUrlIsLive = :isLive', { isLive: true })
6285
.andWhere('coreResult.finalUrlMIMEType NOT IN (:...mimeTypes)', {
6386
mimeTypes: [
@@ -110,7 +133,6 @@ export class WebsiteService {
110133
}
111134

112135
async findWebsitesWithPrimaryTimedOutCoreResults(): Promise<Website[]> {
113-
114136
const queryBuilder = this.website
115137
.createQueryBuilder('website')
116138
.innerJoinAndSelect('website.coreResult', 'coreResult')

‎libs/snapshot/src/config/snapshot.config.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,20 @@
44
export default () => {
55
if (process.env.NODE_ENV === 'production') {
66
return {
7-
fileNameAccessibility: 'weekly-snapshot-accessibility-details',
8-
fileNameDailyLive: 'site-scanning-live-filtered-latest',
9-
fileNameDailyUnique: 'site-scanning-live-filtered-unique-latest',
7+
fileNameDailyLive: 'site-scanning-live-latest',
8+
fileNameDailyLiveFiltered: 'site-scanning-live-filtered-latest',
9+
fileNameDailyLiveFilteredUnique:
10+
'site-scanning-live-filtered-unique-latest',
1011
fileNameDailyAll: 'site-scanning-latest',
12+
fileNameAccessibility: 'weekly-snapshot-accessibility-details',
1113
};
1214
} else {
1315
return {
14-
fileNameAccessibility: `weekly-snapshot-accessibility-details-${process.env.NODE_ENV}`,
15-
fileNameDailyLive: `site-scanning-live-filtered-latest-${process.env.NODE_ENV}`,
16-
fileNameDailyUnique: `site-scanning-live-filtered-unique-latest-${process.env.NODE_ENV}`,
16+
fileNameDailyLive: `site-scanning-live-latest-${process.env.NODE_ENV}`,
17+
fileNameDailyLiveFiltered: `site-scanning-live-filtered-latest-${process.env.NODE_ENV}`,
18+
fileNameDailyLiveFilteredUnique: `site-scanning-live-filtered-unique-latest-${process.env.NODE_ENV}`,
1719
fileNameDailyAll: `site-scanning-latest-${process.env.NODE_ENV}`,
20+
fileNameAccessibility: `weekly-snapshot-accessibility-details-${process.env.NODE_ENV}`,
1821
};
1922
}
2023
};

‎libs/snapshot/src/snapshot.service.spec.ts‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,12 @@ describe('SnapshotService', () => {
4040
useValue: {
4141
get: jest.fn((key: string) => {
4242
if (key === 'fileNameDailyLive') {
43+
return 'site-scanning-live-latest';
44+
}
45+
if (key === 'fileNameDailyLiveFiltered') {
4346
return 'site-scanning-live-filtered-latest';
4447
}
45-
if (key === 'fileNameDailyUnique') {
48+
if (key === 'fileNameDailyLiveFilteredUnique') {
4649
return 'site-scanning-live-filtered-unique-latest';
4750
}
4851
if (key === 'fileNameDailyAll') {
@@ -79,7 +82,12 @@ describe('SnapshotService', () => {
7982
website.url = 'supremecourt.gov';
8083

8184
mockWebsiteService.findLiveSnapshotResults.mockResolvedValue([website]);
82-
mockWebsiteService.findUniqueSnapshotResults.mockResolvedValue([website]);
85+
mockWebsiteService.findLiveFilteredSnapshotResults.mockResolvedValue([
86+
website,
87+
]);
88+
mockWebsiteService.findLiveFilteredUniqueSnapshotResults.mockResolvedValue([
89+
website,
90+
]);
8391
mockWebsiteService.findAllSnapshotResults.mockResolvedValue([website]);
8492

8593
mockStorageService.exists.mockResolvedValue(true);
@@ -94,6 +102,24 @@ describe('SnapshotService', () => {
94102
await service.dailySnapshot();
95103

96104
const expectedCopyCalls = [
105+
// live (JSON)
106+
[
107+
'site-scanning-live-previous.json',
108+
`archive/json/site-scanning-live-${expectedDate}.json`,
109+
],
110+
[
111+
'site-scanning-live-filtered-latest.json',
112+
'site-scanning-live-filtered-previous.json',
113+
],
114+
// live (CSV)
115+
[
116+
'site-scanning-live-previous.csv',
117+
`archive/csv/site-scanning-live-${expectedDate}.csv`,
118+
],
119+
[
120+
'site-scanning-live-filtered-latest.csv',
121+
'site-scanning-live-filtered-previous.csv',
122+
],
97123
// live filtered (JSON)
98124
[
99125
'site-scanning-live-filtered-previous.json',
@@ -145,6 +171,8 @@ describe('SnapshotService', () => {
145171
];
146172

147173
const expectedUploadCalls = [
174+
['site-scanning-live-latest.json', expect.anything()],
175+
['site-scanning-live-latest.csv', expect.anything()],
148176
['site-scanning-live-filtered-latest.json', expect.anything()],
149177
['site-scanning-live-filtered-latest.csv', expect.anything()],
150178
['site-scanning-live-filtered-unique-latest.json', expect.anything()],

‎libs/snapshot/src/snapshot.service.ts‎

Lines changed: 57 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,18 @@ export class SnapshotService {
1919
private configService: ConfigService,
2020
) {}
2121

22-
private fileNameAccessibility = this.configService.get<string>(
23-
'fileNameAccessibility',
24-
);
2522
private fileNameDailyLive =
2623
this.configService.get<string>('fileNameDailyLive');
27-
private fileNameDailyUnique = this.configService.get<string>(
28-
'fileNameDailyUnique',
24+
private fileNameDailyLiveFiltered = this.configService.get<string>(
25+
'fileNameDailyLiveFiltered',
26+
);
27+
private fileNameDailyLiveFilteredUnique = this.configService.get<string>(
28+
'fileNameDailyLiveFilteredUnique',
2929
);
3030
private fileNameDailyAll = this.configService.get<string>('fileNameDailyAll');
31+
private fileNameAccessibility = this.configService.get<string>(
32+
'fileNameAccessibility',
33+
);
3134

3235
async dailySnapshot() {
3336
const date = this.datetimeService.now();
@@ -39,10 +42,15 @@ export class SnapshotService {
3942
CoreResult.snapshotColumnOrder,
4043
this.fileNameDailyLive,
4144
);
42-
await this.uniqueSnapshot(
45+
await this.liveFilteredSnapshot(
4346
yesterday,
4447
CoreResult.snapshotColumnOrder,
45-
this.fileNameDailyUnique,
48+
this.fileNameDailyLiveFiltered,
49+
);
50+
await this.liveFilteredUniqueSnapshot(
51+
yesterday,
52+
CoreResult.snapshotColumnOrder,
53+
this.fileNameDailyLiveFilteredUnique,
4654
);
4755
await this.allSnapshot(
4856
yesterday,
@@ -58,7 +66,7 @@ export class SnapshotService {
5866
): Promise<void> {
5967
let liveWebsites = await this.websiteService.findLiveSnapshotResults();
6068
this.logger.log(
61-
`Total number of live websites retrieved for snapshot: ${liveWebsites.length}`,
69+
`Total number of live websites retrieved for live snapshot: ${liveWebsites.length}`,
6270
);
6371

6472
let liveSnapshot = new Snapshot(
@@ -79,32 +87,62 @@ export class SnapshotService {
7987
liveSnapshot = null;
8088
}
8189

82-
async uniqueSnapshot(
90+
async liveFilteredSnapshot(
91+
date: string,
92+
columns: string[],
93+
fileName: string,
94+
): Promise<void> {
95+
let liveFilteredWebsites =
96+
await this.websiteService.findLiveFilteredSnapshotResults();
97+
this.logger.log(
98+
`Total number of live websites retrieved for live filtered snapshot: ${liveFilteredWebsites.length}`,
99+
);
100+
101+
let liveFilteredSnapshot = new Snapshot(
102+
this.storageService,
103+
[new JsonSerializer(columns), new CsvSerializer(columns)],
104+
liveFilteredWebsites,
105+
date,
106+
fileName,
107+
);
108+
109+
await liveFilteredSnapshot.archiveDaily();
110+
this.logger.log('Live filtered snapshot archived.');
111+
112+
await liveFilteredSnapshot.saveNew();
113+
this.logger.log('Live filtered snapshot saved.');
114+
115+
liveFilteredWebsites = null;
116+
liveFilteredSnapshot = null;
117+
}
118+
119+
async liveFilteredUniqueSnapshot(
83120
date: string,
84121
columns: string[],
85122
filename: string,
86123
): Promise<void> {
87-
let uniqueWebsites = await this.websiteService.findUniqueSnapshotResults();
124+
let liveFilteredUniqueWebsites =
125+
await this.websiteService.findLiveFilteredUniqueSnapshotResults();
88126
this.logger.log(
89-
`Total number of unique websites retrieved for snapshot: ${uniqueWebsites.length}`,
127+
`Total number of live filtered unique websites retrieved for snapshot: ${liveFilteredUniqueWebsites.length}`,
90128
);
91129

92-
let uniqueSnapshot = new Snapshot(
130+
let liveFilteredUniqueSnapshot = new Snapshot(
93131
this.storageService,
94132
[new JsonSerializer(columns), new CsvSerializer(columns)],
95-
uniqueWebsites,
133+
liveFilteredUniqueWebsites,
96134
date,
97135
filename,
98136
);
99137

100-
await uniqueSnapshot.archiveDaily();
101-
this.logger.log('Unique snapshot archived.');
138+
await liveFilteredUniqueSnapshot.archiveDaily();
139+
this.logger.log('Live filtered unique snapshot archived.');
102140

103-
await uniqueSnapshot.saveNew();
104-
this.logger.log('Unique snapshot saved.');
141+
await liveFilteredUniqueSnapshot.saveNew();
142+
this.logger.log('Live filtered unique snapshot saved.');
105143

106-
uniqueWebsites = null;
107-
uniqueSnapshot = null;
144+
liveFilteredUniqueWebsites = null;
145+
liveFilteredUniqueSnapshot = null;
108146
}
109147

110148
async allSnapshot(date: string, columns: string[], filename: string) {

0 commit comments

Comments
 (0)