feat: Concurrency (#89)

# Description

Enable a concurrency parameter to allow users to concurrently upload
assets

## Issue Ticket Number

Closes #57

## Type of change

<!-- Please select all options that are applicable. -->

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [X] This change requires a documentation update


# Checklist

<!-- These must all be followed and checked. -->

- [ ] I have followed the contributing guidelines of this project as
mentioned in [CONTRIBUTING.md](/CONTRIBUTING.md)
- [ ] I have created an
[issue](https://github.com/colbyfayock/netlify-plugin-cloudinary/issues)
ticket for this PR
- [ ] I have checked to ensure there aren't other open [Pull
Requests](https://github.com/colbyfayock/netlify-plugin-cloudinary/pulls)
for the same update/change?
- [ ] I have performed a self-review of my own code
- [x] I have run tests locally to ensure they all pass
- [ ] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes needed to the documentation

---------

Co-authored-by: Colby Fayock <[email protected]>

Author: matiasfha

README.md

@@ -97,7 +97,7 @@ npm install netlify-plugin-cloudinary
 | maxSize         | object  | No      | eager     | See Below. |
 | privateCdn      | boolean | No      | true      | Enables Private CDN Distribution (Advanced Plan Users) |
 | uploadPreset    | string  | No      | my-preset | Defined set of asset upload defaults in Cloudinary |
-
+| uploadConcurrency | number | No     | 10        | Maximum value of concurrent uploads |
 *Cloud Name must be set as an environment variable if not as an input
 
 #### Max Size
@@ -322,4 +322,4 @@ npm run test
 <!-- markdownlint-restore -->
 <!-- prettier-ignore-end -->
 
-<!-- ALL-CONTRIBUTORS-LIST:END -->
+<!-- ALL-CONTRIBUTORS-LIST:END -->

netlify-plugin-cloudinary/manifest.yml

@@ -30,3 +30,7 @@ inputs:
   - name: uploadPreset
     required: false
     description: "Defined set of asset upload defaults in Cloudinary"
+  - name: uploadConcurrency
+    required: false
+    description: "Maximum value of concurrent uploads"
+    default: 10

netlify-plugin-cloudinary/package.json

@@ -22,7 +22,8 @@
     "cloudinary": "^1.40.0",
     "glob": "^10.3.3",
     "jsdom": "21",
-    "node-fetch": "2"
+    "node-fetch": "2",
+    "p-limit": "3.1.0"
   },
   "devDependencies": {
     "@netlify/config": "20.8.1",

netlify-plugin-cloudinary/src/index.ts

@@ -1,6 +1,7 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { glob } from 'glob';
+import pLimit from 'p-limit';
 
 import { Inputs } from './types/integration';
 
@@ -101,6 +102,8 @@ const CLOUDINARY_ASSET_DIRECTORIES = [
   },
 ];
 
+const DEFAULT_CONCURRENCY = 10;
+
 /**
  * TODO
  * - Handle srcset
@@ -130,13 +133,14 @@ export async function onBuild({
   const {
     cname,
     deliveryType,
-    folder = process.env.SITE_NAME,
+    folder = process.env.SITE_NAME || '',
     imagesPath = CLOUDINARY_ASSET_DIRECTORIES.find(
       ({ inputKey }) => inputKey === 'imagesPath',
     )?.path,
     maxSize,
     privateCdn,
     uploadPreset,
+    uploadConcurrency = DEFAULT_CONCURRENCY,
   } = inputs;
 
   if (!folder) {
@@ -201,26 +205,31 @@ export async function onBuild({
   }
 
   try {
-    _cloudinaryAssets.images = await Promise.all(
-      imagesFiles.map(async image => {
-        const publishPath = image.replace(PUBLISH_DIR, '');
-
-        const cloudinary = await getCloudinaryUrl({
-          deliveryType,
-          folder,
-          path: publishPath,
-          localDir: PUBLISH_DIR,
-          uploadPreset,
-          remoteHost: host,
-          transformations
-        });
-
-        return {
-          publishPath,
-          ...cloudinary,
-        };
-      }),
-    );
+    const limitUploadFiles = pLimit(uploadConcurrency);
+    const uploadsQueue = imagesFiles.map(image => {
+      const publishPath = image.replace(PUBLISH_DIR, '');
+      return limitUploadFiles(() => {
+        async function uploadFile() {
+          const cloudinary = await getCloudinaryUrl({
+            deliveryType,
+            folder,
+            path: publishPath,
+            localDir: PUBLISH_DIR,
+            uploadPreset,
+            remoteHost: host,
+            transformations
+          });
+
+          return {
+            publishPath,
+            ...cloudinary,
+          };
+        }
+        return uploadFile();
+      })
+    })
+
+    _cloudinaryAssets.images = await Promise.all(uploadsQueue);
   } catch (e) {
     globalErrors.push(e)
   }

netlify-plugin-cloudinary/src/types/integration.ts

@@ -16,4 +16,5 @@ export type Inputs = {
   };
   privateCdn: boolean;
   uploadPreset: string;
-};
+  uploadConcurrency: number;
+};

netlify.toml

@@ -14,7 +14,7 @@
 
   [plugins.inputs]
   cloudName = "netlify-cloudinary"
-  # deliveryType = "upload"
+  deliveryType = "upload"
   imagesPath = [ "images", "screenshots" ]
 
   [plugins.inputs.maxSize]