Skip to content

Commit 87b084f

Browse files
mschlicht-ttmschlicht-tt
authored
Support for pagination for commit and pull details (#28)
1 parent 9b40887 commit 87b084f

File tree

1 file changed

+122
-73
lines changed

1 file changed

+122
-73
lines changed

‎src/scm.ts‎

Lines changed: 122 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,33 @@ class Github extends BaseScmAdapter {
331331
return null;
332332
}
333333

334+
private async fetchPaginated<T>(
335+
url: string,
336+
token: string,
337+
perPage = 100,
338+
): Promise<T[]> {
339+
let page = 1;
340+
const allItems: T[] = [];
341+
let itemsOnPage: T[];
342+
343+
do {
344+
const pageUrl = `${url}?per_page=${perPage}&page=${page}`;
345+
const response = await fetch(pageUrl, {
346+
headers: this.createHeaders(token),
347+
});
348+
if (!response.ok) {
349+
throw new Error(
350+
`Failed to fetch paginated data (page ${page}): ${response.statusText}`,
351+
);
352+
}
353+
itemsOnPage = await response.json();
354+
allItems.push(...itemsOnPage);
355+
page++;
356+
} while (itemsOnPage.length === perPage);
357+
358+
return allItems;
359+
}
360+
334361
private async getCommitDetails(
335362
commitInfo: CommitInfo,
336363
token: string,
@@ -366,29 +393,29 @@ class Github extends BaseScmAdapter {
366393
commitInfo.repo
367394
}`;
368395
return commitData.files
369-
.filter(f => this.isSupportedFile(f.filename))
370-
.map(file => {
371-
const filenameOld = file.previous_filename ?? file.filename;
372-
const shaOld = commitData.parents[0]?.sha;
373-
return {
374-
filename: file.filename,
375-
filenameOld,
376-
new: file.status == 'added',
377-
renamed: file.status == 'renamed',
378-
deleted: file.status == 'removed',
379-
additions: file.additions,
380-
deletions: file.deletions,
381-
shaOld,
382-
shaNew: commitData.sha,
383-
download: {
384-
type: 'json' as const,
385-
old: shaOld
386-
? `${baseApiUrl}/contents/${filenameOld}?ref=${shaOld}`
387-
: null,
388-
new: `${baseApiUrl}/contents/${file.filename}?ref=${commitData.sha}`,
389-
},
390-
};
391-
});
396+
.filter((f) => this.isSupportedFile(f.filename))
397+
.map((file) => {
398+
const filenameOld = file.previous_filename ?? file.filename;
399+
const shaOld = commitData.parents[0]?.sha;
400+
return {
401+
filename: file.filename,
402+
filenameOld,
403+
new: file.status == 'added',
404+
renamed: file.status == 'renamed',
405+
deleted: file.status == 'removed',
406+
additions: file.additions,
407+
deletions: file.deletions,
408+
shaOld,
409+
shaNew: commitData.sha,
410+
download: {
411+
type: 'json' as const,
412+
old: shaOld
413+
? `${baseApiUrl}/contents/${filenameOld}?ref=${shaOld}`
414+
: null,
415+
new: `${baseApiUrl}/contents/${file.filename}?ref=${commitData.sha}`,
416+
},
417+
};
418+
});
392419
}
393420

394421
private async getPullDetails(
@@ -399,7 +426,7 @@ class Github extends BaseScmAdapter {
399426
commitInfo.repo
400427
}/pulls/${commitInfo.pullNumber}`;
401428

402-
let response = await fetch(pullUrl, {
429+
const response = await fetch(pullUrl, {
403430
headers: this.createHeaders(token),
404431
});
405432

@@ -413,12 +440,10 @@ class Github extends BaseScmAdapter {
413440
const pullFilesUrl = `${this.getApiUrl()}/repos/${commitInfo.owner}/${
414441
commitInfo.repo
415442
}/pulls/${commitInfo.pullNumber}/files`;
416-
response = await fetch(pullFilesUrl, {
417-
headers: this.createHeaders(token),
418-
});
419-
const files = await response.json();
443+
const allFiles: GithubChangeFile[] =
444+
await this.fetchPaginated<GithubChangeFile>(pullFilesUrl, token);
420445

421-
return { info, files };
446+
return { info, files: allFiles };
422447
}
423448

424449
protected async handlePullRequest(
@@ -429,29 +454,29 @@ class Github extends BaseScmAdapter {
429454
const baseApiUrl = `${this.getApiUrl()}/repos/${pullInfo.owner}/${
430455
pullInfo.repo
431456
}`;
432-
457+
433458
return pullData.files
434-
.filter(f => this.isSupportedFile(f.filename))
435-
.map(file => {
436-
const filenameOld = file.previous_filename ?? file.filename;
437-
const shaOld = pullData.info.base.sha;
438-
return {
439-
filename: file.filename,
440-
filenameOld,
441-
new: file.status == 'added',
442-
renamed: file.status == 'renamed',
443-
deleted: file.status == 'removed',
444-
additions: file.additions,
445-
deletions: file.deletions,
446-
shaOld,
447-
shaNew: pullData.info.head.sha,
448-
download: {
449-
type: 'json' as const,
450-
old: `${baseApiUrl}/contents/${file.filename}?ref=${shaOld}`,
451-
new: `${baseApiUrl}/contents/${file.filename}?ref=${pullData.info.head.sha}`,
452-
},
453-
};
454-
});
459+
.filter((f) => this.isSupportedFile(f.filename))
460+
.map((file) => {
461+
const filenameOld = file.previous_filename ?? file.filename;
462+
const shaOld = pullData.info.base.sha;
463+
return {
464+
filename: file.filename,
465+
filenameOld,
466+
new: file.status == 'added',
467+
renamed: file.status == 'renamed',
468+
deleted: file.status == 'removed',
469+
additions: file.additions,
470+
deletions: file.deletions,
471+
shaOld,
472+
shaNew: pullData.info.head.sha,
473+
download: {
474+
type: 'json' as const,
475+
old: `${baseApiUrl}/contents/${file.filename}?ref=${shaOld}`,
476+
new: `${baseApiUrl}/contents/${file.filename}?ref=${pullData.info.head.sha}`,
477+
},
478+
};
479+
});
455480
}
456481
}
457482

@@ -505,6 +530,36 @@ class Gitlab extends BaseScmAdapter {
505530
});
506531
}
507532

533+
private async fetchPaginated<T>(
534+
url: string,
535+
token: string,
536+
perPage = 100,
537+
): Promise<T[]> {
538+
let page = 1;
539+
let totalPages = 1;
540+
const allItems: T[] = [];
541+
542+
do {
543+
const response = await fetch(`${url}?per_page=${perPage}&page=${page}`, {
544+
headers: this.createHeaders(token),
545+
});
546+
if (!response.ok) {
547+
throw new Error(
548+
`Failed to fetch paginated data (page ${page}): [${response.status}] ${response.statusText}`,
549+
);
550+
}
551+
if (page === 1) {
552+
const tp = response.headers.get('x-total-pages');
553+
totalPages = tp ? parseInt(tp, 10) : 1;
554+
}
555+
const batch: T[] = await response.json();
556+
allItems.push(...batch);
557+
page++;
558+
} while (page <= totalPages);
559+
560+
return allItems;
561+
}
562+
508563
private async getCommitDetails(
509564
commitInfo: CommitInfo,
510565
token: string,
@@ -516,11 +571,9 @@ class Gitlab extends BaseScmAdapter {
516571
const namespace = encodeURIComponent(
517572
`${commitInfo.owner}/${commitInfo.repo}`,
518573
);
519-
520-
// get project id
521574
const commitUrl = `${this.getApiUrl()}/projects/${namespace}/repository/commits/${commitInfo.commitHash}`;
522575

523-
let response = await fetch(commitUrl, {
576+
const response = await fetch(commitUrl, {
524577
headers: this.createHeaders(token),
525578
});
526579

@@ -532,22 +585,13 @@ class Gitlab extends BaseScmAdapter {
532585
const commitData = await response.json();
533586

534587
const diffUrl = `${this.getApiUrl()}/projects/${namespace}/repository/commits/${commitInfo.commitHash}/diff`;
588+
const allChanges: GitlabChange[] = await this.fetchPaginated(
589+
diffUrl,
590+
token,
591+
);
535592

536-
response = await fetch(diffUrl, {
537-
headers: this.createHeaders(token),
538-
});
539-
540-
if (!response.ok) {
541-
throw new Error(
542-
`Failed to retrieve commit details: [${response.status}] ${response.statusText}`,
543-
);
544-
}
545-
546-
const diffData: GitlabChange[] = await response.json();
547-
const files = this.processChanges(diffData);
548-
const parents = commitData.parent_ids.map((id: string) => {
549-
return { sha: id };
550-
});
593+
const files = this.processChanges(allChanges);
594+
const parents = commitData.parent_ids.map((id: string) => ({ sha: id }));
551595

552596
return {
553597
sha: commitInfo.commitHash,
@@ -612,8 +656,15 @@ class Gitlab extends BaseScmAdapter {
612656
files: CommonChange[];
613657
}> {
614658
const namespace = encodeURIComponent(`${pullInfo.owner}/${pullInfo.repo}`);
659+
const diffsUrl = `${this.getApiUrl()}/projects/${namespace}/merge_requests/${pullInfo.pullNumber}/diffs`;
660+
const allChanges: GitlabChange[] = await this.fetchPaginated<GitlabChange>(
661+
diffsUrl,
662+
token,
663+
);
664+
const files: CommonChange[] = this.processChanges(allChanges);
665+
615666
const response = await fetch(
616-
`${this.getApiUrl()}/projects/${namespace}/merge_requests/${pullInfo.pullNumber}/changes`,
667+
`${this.getApiUrl()}/projects/${namespace}/merge_requests/${pullInfo.pullNumber}`,
617668
{ headers: this.createHeaders(token) },
618669
);
619670
if (!response.ok) {
@@ -622,11 +673,10 @@ class Gitlab extends BaseScmAdapter {
622673
);
623674
}
624675
const pullData = await response.json();
625-
const files = this.processChanges(pullData.changes);
626676

627677
return {
628678
info: {
629-
head: { sha: pullData.sha },
679+
head: { sha: pullData.diff_refs.head_sha },
630680
base: { sha: pullData.diff_refs.base_sha },
631681
},
632682
files,
@@ -641,7 +691,6 @@ class Gitlab extends BaseScmAdapter {
641691
const namespace = encodeURIComponent(`${pullInfo.owner}/${pullInfo.repo}`);
642692
const baseApiUrl = `${this.getApiUrl()}/projects/${namespace}/repository/files`;
643693

644-
645694
// pullData.info.base.sha is probably not set if target branch has no commit yet
646695
const shaOld = pullData.info.base.sha || pullData.info.head.sha;
647696
const shaNew = pullData.info.head.sha;

0 commit comments

Comments
 (0)