Add colcon bundle retry (#22)

* Adding exponential backoff to colcon bundle

* Update action commit hash in andstor/file-existence-action test workflow

* NCC re-build

Co-authored-by: ojasjosh <[email protected]>

Author: ojasjoshi

.github/workflows/test-ros2.yml

@@ -70,7 +70,7 @@ jobs:
     # Expectation: bundle file, build files, and dependencies all exist
     - name: Check robot_ws file existence
       id: check_robot_ws_files
-      uses: andstor/file-existence-action@87d74d4
+      uses: andstor/file-existence-action@87d74d4732ddb824259d80c8a508c0124bf1c673
       with:
         files: "./robot_ws.tar, ./robot_ws/build, ./robot_ws/install, ./robot_ws/src/deps"
         allow_failure: true
@@ -86,15 +86,15 @@ jobs:
     # Expectation: bundle file, build files, and dependencies all exist        
     - name: Check simulation_ws file existence
       id: check_simulation_ws_files
-      uses: andstor/file-existence-action@87d74d4
+      uses: andstor/file-existence-action@87d74d4732ddb824259d80c8a508c0124bf1c673
       with:
         files: "./simulation_ws.tar, ./simulation_ws/build, ./simulation_ws/install, ./simulation_ws/src/deps"
         allow_failure: true
 
     # Expectation: source files exist    
     - name: Check source file existence
       id: check_source_files
-      uses: andstor/file-existence-action@87d74d4
+      uses: andstor/file-existence-action@87d74d4732ddb824259d80c8a508c0124bf1c673
       with:        
         files: "./sources.zip, ./sources.tar.gz"
 

.github/workflows/test.yml

@@ -80,7 +80,7 @@ jobs:
     # Expectation: bundle file, build files, and dependencies all exist
     - name: Check robot_ws file existence
       id: check_robot_ws_files
-      uses: andstor/file-existence-action@87d74d4
+      uses: andstor/file-existence-action@87d74d4732ddb824259d80c8a508c0124bf1c673
       with:
         files: "./robot_ws.tar, ./robot_ws/build, ./robot_ws/install, ./robot_ws/src/deps"
         allow_failure: true
@@ -96,15 +96,15 @@ jobs:
     # Expectation: bundle file, build files, and dependencies all exist        
     - name: Check simulation_ws file existence
       id: check_simulation_ws_files
-      uses: andstor/file-existence-action@87d74d4
+      uses: andstor/file-existence-action@87d74d4732ddb824259d80c8a508c0124bf1c673
       with:
         files: "./simulation_ws.tar, ./simulation_ws/build, ./simulation_ws/install, ./simulation_ws/src/deps"
         allow_failure: true
 
     # Expectation: source files exist    
     - name: Check source file existence
       id: check_source_files
-      uses: andstor/file-existence-action@87d74d4
+      uses: andstor/file-existence-action@87d74d4732ddb824259d80c8a508c0124bf1c673
       with:        
         files: "./sources.zip, ./sources.tar.gz"
 

robomaker-sample-app-ci/action.yml

@@ -13,6 +13,10 @@ inputs:
   generate-sources:
     description: 'Whether or not to generate sources.zip and sources.tar.gz files [true|false]'
     default: false
+  colcon-bundle-retries:
+    description: 'Number of times colcon bundle should be re-tried with exponential backoff in case of failure [0-9]'
+    default: 0
+
 outputs:
   ros-distro:
     description: 'Distribution of ROS to use [kinetic|melodic|dashing]'

robomaker-sample-app-ci/dist/index.js

@@ -671,6 +671,12 @@ const WORKSPACE_DIRECTORY = core.getInput('workspace-dir');
 const GENERATE_SOURCES = core.getInput('generate-sources');
 let PACKAGES = "none";
 const ROS_ENV_VARIABLES = {};
+const COLCON_BUNDLE_RETRIES = Number.parseInt(core.getInput('colcon-bundle-retries'), 10);
+const MINIMUM_BACKOFF_TIME_SECONDS = 32; // delay for the first retry in seconds
+const MAXIMUM_BACKOFF_TIME_SECONDS = 128; // maximum delay for a retry in seconds
+function delay(ms) {
+    return new Promise( resolve => setTimeout(resolve, ms) );
+}
 function loadROSEnvVariables() {
     return __awaiter(this, void 0, void 0, function* () {
         const options = {
@@ -852,14 +858,25 @@ function build() {
 }
 function bundle() {
     return __awaiter(this, void 0, void 0, function* () {
-        try {
-            const bundleFilename = path.basename(WORKSPACE_DIRECTORY);
-            yield exec.exec("colcon", ["bundle", "--build-base", "build", "--install-base", "install", "--bundle-base", "bundle"], getWorkingDirExecOptions());
-            yield exec.exec("mv", ["bundle/output.tar", `../${bundleFilename}.tar`], getWorkingDirExecOptions());
-            yield exec.exec("rm", ["-rf", "bundle"], getWorkingDirExecOptions()); // github actions have been failing with no disk space
-        }
-        catch (error) {
-            core.setFailed(error.message);
+        let delay_ms = 1000 * MINIMUM_BACKOFF_TIME_SECONDS;
+        // indexed from 0 because COLCON_BUNDLE_RETRIES is the number of retries AFTER the initial try
+        for (let i = 0; i <= COLCON_BUNDLE_RETRIES; i++) {
+            try {
+                const bundleFilename = path.basename(WORKSPACE_DIRECTORY);
+                yield exec.exec("colcon", ["bundle", "--build-base", "build", "--install-base", "install", "--bundle-base", "bundle"], getWorkingDirExecOptions());
+                yield exec.exec("mv", ["bundle/output.tar", `../${bundleFilename}.tar`], getWorkingDirExecOptions());
+                yield exec.exec("rm", ["-rf", "bundle"], getWorkingDirExecOptions());  // github actions have been failing with no disk space
+                break; // break if colcon bundle passes
+            } catch (error) {
+                yield exec.exec("rm", ["-rf", "bundle"], getWorkingDirExecOptions()); // remove erred bundle assets
+                if (i == COLCON_BUNDLE_RETRIES){
+                    core.setFailed(error.message); // set action to Failed if the colcon bundle fails even after COLCON_BUNDLE_RETRIES number of retries
+                    break;
+                }
+                console.log(`Colcon bundle failed.. retrying in ${delay_ms} milliseconds`);
+                yield delay(delay_ms); // wait for next retry per the current exponential backoff delay
+                delay_ms = Math.min(delay_ms * 2, MAXIMUM_BACKOFF_TIME_SECONDS); // double the delay for the next retry, truncate if required
+            }
         }
     });
 }
@@ -869,6 +886,11 @@ function run() {
         console.log(`GAZEBO_VERSION: ${GAZEBO_VERSION}`);
         console.log(`WORKSPACE_DIRECTORY: ${WORKSPACE_DIRECTORY}`);
         console.log(`GENERATE_SOURCES: ${GENERATE_SOURCES}`);
+        console.log(`COLCON_BUNDLE_RETRIES: ${COLCON_BUNDLE_RETRIES}`);
+        // check if COLCON_BUNDLE_RETRIES is valid (i.e. 0<) and not too large (i.e. <10)
+        if (COLCON_BUNDLE_RETRIES<0 || 9<COLCON_BUNDLE_RETRIES){
+            core.setFailed(`Invalid number of colcon bundle retries. Must be between 0-9 inclusive`);
+        }
         yield setup();
         if (ROS_DISTRO == "kinetic" && (GAZEBO_VERSION == "" || GAZEBO_VERSION == "7")) {
             GAZEBO_VERSION = "7";

robomaker-sample-app-ci/src/aws-robomaker-sample-application-ci.ts

@@ -12,6 +12,13 @@ const WORKSPACE_DIRECTORY = core.getInput('workspace-dir');
 const GENERATE_SOURCES = core.getInput('generate-sources');
 let PACKAGES = "none"
 const ROS_ENV_VARIABLES: any = {};
+const COLCON_BUNDLE_RETRIES = Number.parseInt(core.getInput('colcon-bundle-retries'), 10);
+const MINIMUM_BACKOFF_TIME_SECONDS = 32; // delay for the first retry in seconds
+const MAXIMUM_BACKOFF_TIME_SECONDS = 128; // maximum delay for a retry in seconds
+
+function delay(ms: number) {
+  return new Promise( resolve => setTimeout(resolve, ms) );
+}
 
 async function loadROSEnvVariables() {
   const options = {
@@ -184,7 +191,6 @@ async function prepare_sources() {
 async function build() {
   try {
     await exec.exec("rosdep", ["install", "--from-paths", ".", "--ignore-src", "-r", "-y", "--rosdistro", ROS_DISTRO], getWorkingDirExecOptions());
-
     console.log(`Building the following packages: ${PACKAGES}`);
     await exec.exec("colcon", ["build", "--build-base", "build", "--install-base", "install"], getWorkingDirExecOptions());
   } catch (error) {
@@ -193,13 +199,25 @@ async function build() {
 }
 
 async function bundle() {
-  try {
-    const bundleFilename = path.basename(WORKSPACE_DIRECTORY); 
-    await exec.exec("colcon", ["bundle", "--build-base", "build", "--install-base", "install", "--bundle-base", "bundle"], getWorkingDirExecOptions());
-    await exec.exec("mv", ["bundle/output.tar", `../${bundleFilename}.tar`], getWorkingDirExecOptions());
-    await exec.exec("rm", ["-rf", "bundle"], getWorkingDirExecOptions());  // github actions have been failing with no disk space
-  } catch (error) {
-    core.setFailed(error.message);
+  let delay_ms = 1000 * MINIMUM_BACKOFF_TIME_SECONDS; 
+  // indexed from 0 because COLCON_BUNDLE_RETRIES is the number of retries AFTER the initial try
+  for (let i = 0; i <= COLCON_BUNDLE_RETRIES; i++) {
+    try {
+      const bundleFilename = path.basename(WORKSPACE_DIRECTORY);
+      await exec.exec("colcon", ["bundle", "--build-base", "build", "--install-base", "install", "--bundle-base", "bundle"], getWorkingDirExecOptions());
+      await exec.exec("mv", ["bundle/output.tar", `../${bundleFilename}.tar`], getWorkingDirExecOptions());
+      await exec.exec("rm", ["-rf", "bundle"], getWorkingDirExecOptions());  // github actions have been failing with no disk space
+      break; // break if colcon bundle passes
+    } catch (error) {
+      await exec.exec("rm", ["-rf", "bundle"], getWorkingDirExecOptions()); // remove erred bundle assets
+      if (i == COLCON_BUNDLE_RETRIES){
+        core.setFailed(error.message); // set action to Failed if the colcon bundle fails even after COLCON_BUNDLE_RETRIES number of retries
+        break;
+      }
+      console.log(`Colcon bundle failed.. retrying in ${delay_ms} milliseconds`);
+      await delay(delay_ms); // wait for next retry per the current exponential backoff delay
+      delay_ms = Math.min(delay_ms * 2, MAXIMUM_BACKOFF_TIME_SECONDS); // double the delay for the next retry, truncate if required
+    }
   }
 }
 
@@ -208,6 +226,12 @@ async function run() {
   console.log(`GAZEBO_VERSION: ${GAZEBO_VERSION}`);
   console.log(`WORKSPACE_DIRECTORY: ${WORKSPACE_DIRECTORY}`);
   console.log(`GENERATE_SOURCES: ${GENERATE_SOURCES}`);
+  console.log(`COLCON_BUNDLE_RETRIES: ${COLCON_BUNDLE_RETRIES}`);
+  
+  // check if COLCON_BUNDLE_RETRIES is valid (i.e. 0<) and not too large (i.e. <10)  
+  if (COLCON_BUNDLE_RETRIES<0 || 9<COLCON_BUNDLE_RETRIES){
+    core.setFailed(`Invalid number of colcon bundle retries. Must be between 0-9 inclusive`);
+  }
 
   await setup();
   if (ROS_DISTRO == "kinetic" && (GAZEBO_VERSION == "" || GAZEBO_VERSION == "7")) {