Add more delays to processing

Author: aegilops

src/componentSubmission.ts

@@ -23,10 +23,14 @@ export interface SubmitOpts {
     quiet?: boolean;
     languages?: string[];
     componentDetectionBinPath?: string; // optional path to component-detection executable
+    lightDelayMs?: number;
 }
 
-export async function getLanguageIntersection(octokit: Octokit, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false): Promise<string[]> {
+export async function getLanguageIntersection(octokit: Octokit, owner: string, repo: string, languages: string[] | undefined, quiet: boolean = false, lightDelayMs: number = 0): Promise<string[]> {
     const langResp = await octokit.request('GET /repos/{owner}/{repo}/languages', { owner, repo });
+
+    await new Promise(r => setTimeout(r, lightDelayMs));
+
     const repoLangs = Object.keys(langResp.data || {});
     const wanted = languages;
     const intersect = wanted ? repoLangs.filter(l => wanted.some(w => w.toLowerCase() === l.toLowerCase())) : repoLangs;
@@ -37,7 +41,7 @@ export async function getLanguageIntersection(octokit: Octokit, owner: string, r
     return intersect;
 }
 
-export async function sparseCheckout(owner: string, repo: string, branch: string, destDir: string, intersect: string[], baseUrl?: string) {
+export async function sparseCheckout(owner: string, repo: string, branch: string, destDir: string, intersect: string[], baseUrl?: string, lightDelayMs?: number) {
     const cwd = destDir;
     const repoUrl = (baseUrl && baseUrl.includes('api/v3'))
         ? baseUrl.replace(/\/api\/v3$/, '') + `/${owner}/${repo}.git`
@@ -52,6 +56,8 @@ export async function sparseCheckout(owner: string, repo: string, branch: string
     await execGit(['fetch', '--depth=1', 'origin', branch], { cwd });
     await execGit(['checkout', 'FETCH_HEAD'], { cwd });
 
+    await new Promise(r => setTimeout(r, lightDelayMs));
+
     const { stdout: shaOut } = await execGit(['rev-parse', 'HEAD'], { cwd: destDir });
     const sha = shaOut.trim();
     console.debug(`Checked out ${owner}/${repo}@${branch} to ${destDir} at commit ${sha}`);
@@ -66,22 +72,22 @@ export async function submitSnapshotIfPossible(opts: SubmitOpts): Promise<boolea
     const tmp = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'cd-submission-'));
 
     try {
-        const intersect = await getLanguageIntersection(opts.octokit, opts.owner, opts.repo, opts.languages);
+        const intersect = await getLanguageIntersection(opts.octokit, opts.owner, opts.repo, opts.languages, opts.quiet, opts.lightDelayMs);
         // Create temp dir and sparse checkout only manifest files according to selected languages
         if (!intersect.length) {
             // No matching languages, skip submission
             return true;
         }
         console.debug(chalk.green(`Sparse checkout into ${tmp} for languages: ${intersect.join(', ')}`));
 
-        const sha = await sparseCheckout(opts.owner, opts.repo, opts.branch, tmp, intersect, opts.baseUrl);
+        const sha = await sparseCheckout(opts.owner, opts.repo, opts.branch, tmp, intersect, opts.baseUrl, opts.lightDelayMs);
 
         // Run the ComponentDetection module to detect components and submit snapshot
         if (!sha) {
             if (!opts.quiet) console.error(chalk.red(`Failed to determine SHA for ${opts.owner}/${opts.repo} on branch ${opts.branch}`));
             return false;
         }
-        return await run(opts.octokit, tmp, opts.owner, opts.repo, sha, opts.branch, opts.componentDetectionBinPath);
+        return await runComponentDetectionAndSubmit(opts.octokit, tmp, opts.owner, opts.repo, sha, opts.branch, opts.componentDetectionBinPath);
 
     } catch (e) {
         if (!opts.quiet) console.error(chalk.red(`Component Detection failed: ${(e as Error).message}`));
@@ -150,7 +156,7 @@ async function execGit(args: string[], opts: { cwd: string, quiet?: boolean }):
     });
 }
 
-export async function run(octokit: Octokit, tmpDir: string, owner: string, repo: string, sha: string, ref: string, componentDetectionBinPath?: string): Promise<boolean> {
+export async function runComponentDetectionAndSubmit(octokit: Octokit, tmpDir: string, owner: string, repo: string, sha: string, ref: string, componentDetectionBinPath?: string): Promise<boolean> {
 
     const componentDetection = new ComponentDetection(octokit, '', componentDetectionBinPath);
 

src/sbomCollector.ts

@@ -300,17 +300,24 @@ export class SbomCollector {
 
           try {
             const branches = await this.listBranches(org, repo.name);
+
+            if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+
             const nonDefault = branches.filter(b => b.name !== sbom.defaultBranch);
             const limited = this.opts.branchLimit && this.opts.branchLimit > 0 ? nonDefault.slice(0, this.opts.branchLimit) : nonDefault;
             const branchDiffs: Map<string, BranchDependencyDiff> = new Map();
             for (const b of limited) {
 
               // get the commits, compare to the stored diff info. If the latest commit is newer, then fetch diff, otherwise skip
               const latestCommit = await this.getLatestCommit(org, repo.name, b.name);
+
+              if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+              
               if (!latestCommit) {
                 console.error(chalk.red(`Failed to get latest commit for ${fullName} branch ${b.name}.`));
                 continue;
               }
+
               const existing = sbom.branchDiffs instanceof Map ? sbom.branchDiffs.get(b.name) : undefined;
               if (await this.isCommitNewer(latestCommit, existing) || this.opts.forceSubmission) {
                 console.debug(chalk.green(`Fetching branch diff for ${fullName} branch ${b.name}...`));
@@ -326,7 +333,6 @@ export class SbomCollector {
               const base = this.opts.branchDiffBase || sbom?.defaultBranch;
               if (!base) { console.error(chalk.red(`Cannot compute branch diff for ${fullName} branch ${b.name} because base branch is undefined.`)); continue; }
 
-              if (this.opts.lightDelayMs) await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
               // Optionally perform dependency submission up front for the branch
               if (this.opts.forceSubmission) {
                 try {
@@ -357,7 +363,7 @@ export class SbomCollector {
         if (sbom && !sbom.error && this.opts.loadFromDir && this.opts.syncSboms && this.opts.loadFromDir.length) {
           try { writeOne(sbom, { outDir: this.opts.loadFromDir }); } catch { /* ignore write errors */ }
         }
-        
+
         if (sbom) {
           newSboms.push(sbom);
         }
@@ -382,6 +388,9 @@ export class SbomCollector {
     if (!this.octokit) throw new Error("No Octokit instance");
     try {
       const resp = await this.octokit.request("GET /repos/{owner}/{repo}/commits", { owner: org, repo, sha: branch });
+
+      await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+
       const commitSha = resp.data?.[0]?.sha;
       const commitDate = resp.data?.[0]?.commit?.author?.date;
       return { sha: commitSha, commitDate };
@@ -434,6 +443,9 @@ export class SbomCollector {
     while (!done) {
       try {
         const resp = await this.octokit.request("GET /orgs/{org}/repos", { org, per_page, page, type: this.opts.includePrivate ? "all" : "public" });
+
+        await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+
         const items = resp.data as Array<{ name: string; pushed_at?: string; updated_at?: string; default_branch?: string }>;
         for (const r of items) {
           repos.push({ name: r.name, pushed_at: r.pushed_at, updated_at: r.updated_at, default_branch: r.default_branch });
@@ -453,6 +465,9 @@ export class SbomCollector {
 
     try {
       const resp = await this.octokit.request("GET /repos/{owner}/{repo}", { owner: org, repo });
+
+      await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+
       const data = resp.data as { name: string; pushed_at?: string; updated_at?: string; default_branch?: string };
       return data;
     } catch (e) {
@@ -466,7 +481,6 @@ export class SbomCollector {
 
     const fullName = `${org}/${repo}`;
     try {
-      // TODO: Ensure dependency graph is enabled before requesting SBOM
       const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/sbom", { owner: org, repo, headers: { Accept: "application/vnd.github+json" } });
       const sbomWrapper = resp.data as { sbom?: Sbom };
       const packages: SbomPackage[] = sbomWrapper?.sbom?.packages ?? [];
@@ -505,6 +519,8 @@ export class SbomCollector {
     while (!done) {
       try {
         const resp = await this.octokit.request("GET /repos/{owner}/{repo}/branches", { owner: org, repo, per_page, page });
+        await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+
         const data = resp.data as Array<{ name: string; protected?: boolean; commit?: { sha?: string } }>;
         branches.push(...data);
         if (data.length < per_page) done = true; else page++;
@@ -523,6 +539,8 @@ export class SbomCollector {
     try {
       const basehead = `${base}...${head}`;
       const resp = await this.octokit.request("GET /repos/{owner}/{repo}/dependency-graph/compare/{basehead}", { owner: org, repo, basehead, headers: { Accept: "application/vnd.github+json" } });
+      await new Promise(r => setTimeout(r, this.opts.lightDelayMs));
+
       // Response shape includes change_set array (per docs). We normalize to DependencyReviewPackageChange[]
       const raw = resp.data;