fix: redundant CLI downloads (#163)

Author: Asaf Agami

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2.1
 
 orbs:
   prodsec: snyk/[email protected]
-  
+
 jobs:
   test:
     docker:

.snyk

@@ -0,0 +1,15 @@
+# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities.
+version: v1.25.0
+# ignores vulnerabilities until expiry date; change duration by modifying expiry date
+ignore:
+  SNYK-JS-MOCKERY-3043117:
+    - '*':
+        reason: No upgrade available
+        expires: 2024-07-18T00:00:00.000Z
+        created: 2023-07-18T12:52:43.840Z
+  SNYK-JS-SEMVER-3247795:
+    - '*':
+        reason: No upgrade available
+        expires: 2024-07-18T00:00:00.000Z
+        created: 2023-07-18T12:52:32.034Z
+patch: {}

ops/deploy/run-test-pipelines.ts

@@ -95,6 +95,10 @@ async function runBuild(
 ): Promise<void> {
   let success = false;
 
+  console.log(
+    `Starting build for project: ${testProjectName} with build definition ID: ${testBuildDefinitionId}`,
+  );
+
   try {
     const launchPipelineResult = await launchBuildPipeline(
       webApi,
@@ -123,21 +127,23 @@ async function runBuild(
       }
 
       if (status === BuildStatus.Completed) {
-        console.log('build is complete');
+        console.log(`build is complete for ${testProjectName}`);
         const result = checkBuildStatusRes.result;
         console.log(`build result: ${result}`);
         if (result) {
           if (result === BuildResult.Succeeded) {
-            console.log('build succeeded');
+            console.log(`build succeeded for ${testProjectName}`);
             success = true;
           } else {
-            console.log(`build did not succeed. BuildResult code: ${result}`);
+            console.log(
+              `build did not succeed for ${testProjectName}. BuildResult code: ${result}`,
+            );
           }
         }
         break;
       } else {
         console.log(
-          `Still waiting for build ${buildId} to complete. Status: ${status}. Time: ${new Date().getTime()}`,
+          `Still waiting for build ${buildId} (${testProjectName}) to complete. Status: ${status}. Time: ${new Date().getTime()}`,
         );
         await asyncSleep(10000);
       }
@@ -150,7 +156,9 @@ async function runBuild(
       return Promise.reject();
     }
   } catch (err) {
-    console.log('failed to launching / checking build');
+    console.log(
+      `Failed to launch/check build for project: ${testProjectName} with build definition ID: ${testBuildDefinitionId}`,
+    );
     console.log(err);
     console.log('\nrejecting - not successful');
     return Promise.reject();

snykTask/src/index.ts

@@ -154,6 +154,7 @@ async function runSnykTest(
     .argIf(taskArgs.organization, `--org=${taskArgs.organization}`)
     .argIf(taskArgs.projectName, `--project-name=${projectNameArg}`)
     .arg(`--json-file-output=${jsonReportOutputPath}`)
+    .argIf(isDebugMode(), '-d')
     .line(taskArgs.additionalArguments);
 
   const options = getOptionsToExecuteSnykCLICommand(

snykTask/src/install/index.ts

@@ -53,18 +53,59 @@ export function getSnykDownloadInfo(platform: Platform): SnykDownloads {
 export async function downloadExecutable(
   targetDirectory: string,
   executable: Executable,
+  maxRetries = 5,
 ) {
-  const fileWriter = fs.createWriteStream(
-    path.join(targetDirectory, executable.filename),
-    {
-      mode: 0o766,
-    },
-  );
-  return new Promise<void>((resolve, reject) => {
-    https.get(executable.downloadUrl, (response) => {
-      response.on('end', () => resolve());
-      response.on('error', (err) => reject(err));
-      response.pipe(fileWriter);
-    });
+  const filePath = path.join(targetDirectory, executable.filename);
+
+  // Check if the file already exists
+  if (fs.existsSync(filePath)) {
+    console.log(
+      `File ${executable.filename} already exists, skipping download.`,
+    );
+    return;
+  }
+
+  const fileWriter = fs.createWriteStream(filePath, {
+    mode: 0o766,
   });
+
+  // Wrapping the download in a function for easy retrying
+  const doDownload = () =>
+    new Promise<void>((resolve, reject) => {
+      https.get(executable.downloadUrl, (response) => {
+        response.on('end', () => resolve());
+        response.on('error', (err) => {
+          console.error(
+            `Download of ${executable.filename} failed: ${err.message}`,
+          );
+          reject(err);
+        });
+        response.pipe(fileWriter);
+      });
+    });
+
+  // Try to download the file, retry up to `maxRetries` times if the attempt fails
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      await doDownload();
+      console.log(`Download successful for ${executable.filename}`);
+      break;
+    } catch (err) {
+      console.error(
+        `Download of ${executable.filename} failed: ${err.message}`,
+      );
+
+      // Don't wait before retrying the last attempt
+      if (attempt < maxRetries - 1) {
+        console.log(
+          `Retrying download of ${executable.filename} after 5 seconds...`,
+        );
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+      } else {
+        console.error(
+          `All retries failed for ${executable.filename}: ${err.message}`,
+        );
+      }
+    }
+  }
 }