s3: fix SIGINT handling (#1653)

* don't short-circuit uploader destruction
* allow time for uploaders to shutdown before propagating signals

Closes #1417

Author: alxndrsn

lib/external/s3.js

@@ -28,15 +28,20 @@ const init = (config) => {
   const inflight = new Set();
   function destroy() {
     destroyed = true;
-    return new Promise((resolve, reject) => {
+    return new Promise(resolve => {
       let remaining = 0;
       for (const req of inflight) {
         ++remaining; // eslint-disable-line no-plusplus
         req.once('close', () => { // eslint-disable-line no-loop-func
           if (!--remaining) resolve(); // eslint-disable-line no-plusplus
         });
-        req.once('error', reject);
-        req.destroy(new Error('Aborted by request'));
+
+        const destroyError = new Error('Aborted by request');
+        req.once('error', err => {
+          // eslint-disable-next-line no-console
+          if (err !== destroyError) console.error('Ignoring unexpected error:', err);
+        });
+        req.destroy(destroyError);
       }
     });
   }

lib/task/s3.js

@@ -65,19 +65,34 @@ const uploadPending = withContainer(({ s3, Blobs }) => async (limit) => {
 
   const signals = ['SIGINT', 'SIGTERM'];
 
+  let uploader;
+
   const shutdownListener = async signal => {
     try {
       await s3.destroy();
     } catch (err) {
       console.log('s3 threw error while shutting down; it will be ignored:', err);
     }
+
+    if (uploader) {
+      // Wait a reasonable amount of time for the uploader to complete
+      try {
+        await Promise.race([
+          uploader,
+          new Promise(resolve => { setTimeout(resolve, 10_000); }),
+        ]);
+      } catch (err) {
+        console.error('Unexpected error from s3 uploader:', err);
+      }
+    }
+
     process.kill(process.pid, signal);
   };
   signals.forEach(s => process.once(s, shutdownListener));
 
   try {
     console.log(`Uploading ${count ?? 'all'} blobs...`);
-    await Blobs.s3UploadPending(limit);
+    await (uploader = Blobs.s3UploadPending(limit));
     console.log(`[${new Date().toISOString()}]`, 'Upload completed.');
   } finally {
     signals.forEach(s => process.removeListener(s, shutdownListener));