[dev] [Marfuen] mariano/batch (#1432)

* chore: remove training video backfill action and related documentation

- Deleted the trigger-training-video-backfill action and its associated README documentation as they are no longer needed. This change streamlines the codebase and eliminates outdated references to backfill functionality.

* feat: enhance training video backfill process with batch processing

- Updated the backfillTrainingVideosForAllOrgs function to process organizations in batches of 500, improving efficiency and handling of large datasets.
- Added logging for batch processing to track the number of organizations processed and any errors encountered during triggering.
- Adjusted return values to reflect the total number of organizations processed and the number of batches created.

---------

Co-authored-by: Mariano Fuentes <marfuen98@gmail.com>

Author: github-actions[bot]

apps/app/src/jobs/tasks/onboarding/backfill-training-videos-for-all-orgs.ts

@@ -41,24 +41,50 @@ export const backfillTrainingVideosForAllOrgs = task({
       );
 
       // Create batch items for processing
-      const batchItems = organizations.map((organization) => ({
+      const allBatchItems = organizations.map((organization) => ({
         payload: {
           organizationId: organization.id,
         },
       }));
 
-      logger.info(`Triggering batch job for ${batchItems.length} organizations`);
+      // Split into chunks of 500 (Trigger.dev batch size limit)
+      const BATCH_SIZE = 500;
+      const batches: (typeof allBatchItems)[] = [];
 
-      // Trigger the batch job to process all organizations
-      await backfillTrainingVideosForOrg.batchTrigger(batchItems);
+      for (let i = 0; i < allBatchItems.length; i += BATCH_SIZE) {
+        batches.push(allBatchItems.slice(i, i + BATCH_SIZE));
+      }
+
+      logger.info(
+        `Splitting ${allBatchItems.length} organizations into ${batches.length} batches of max ${BATCH_SIZE} each`,
+      );
+
+      // Process each batch
+      let totalTriggered = 0;
+      for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i];
+        logger.info(
+          `Triggering batch ${i + 1}/${batches.length} with ${batch.length} organizations`,
+        );
+
+        try {
+          await backfillTrainingVideosForOrg.batchTrigger(batch);
+          totalTriggered += batch.length;
+          logger.info(`Successfully triggered batch ${i + 1}/${batches.length}`);
+        } catch (error) {
+          logger.error(`Failed to trigger batch ${i + 1}/${batches.length}: ${error}`);
+          throw error;
+        }
+      }
 
       logger.info(
-        `Successfully triggered training video backfill jobs for ${organizations.length} organizations`,
+        `Successfully triggered training video backfill jobs for ${totalTriggered} organizations across ${batches.length} batches`,
       );
 
       return {
         success: true,
-        organizationsProcessed: organizations.length,
+        organizationsProcessed: totalTriggered,
+        totalBatches: batches.length,
         totalMembers,
       };
     } catch (error) {