DO NOT MERGE: [Feature] Create script to do a one-off backload of API data for a given date, one day at a time until the present

- Allow deploy from bugfix branch and change command to the one-off backload task
- Update script to be deployed to dev
- Move backload dependencies to prod dependencies
- Convert the async child process to resolve a promise
- Make logging better by responding to child process events

Author: Michael Levin

.github/workflows/ci.yml

@@ -51,8 +51,8 @@ jobs:
     needs:
       - lint
       - test
-    if: github.ref == 'refs/heads/develop'
-    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    if: github.ref == 'refs/heads/bugfix/backload_ga4_api_data'
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@bugfix/backload_ga4_api_data
     with:
       ANALYTICS_GA4_CALL_RETRY_COUNT: ${{ vars.ANALYTICS_GA4_CALL_RETRY_COUNT_DEV }}
       ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}

deploy/api.sh

@@ -6,7 +6,7 @@ export ANALYTICS_SCRIPT_NAME=api.sh
 # Gov Wide
 $ANALYTICS_ROOT_PATH/bin/analytics --debug --write-to-database --output /tmp
 
-# Iterate over each agency config in deploy/envs
+# Iterate over each agency config
 for filename in $ANALYTICS_ROOT_PATH/deploy/envs/*.config.sh; do
   source $filename
   $ANALYTICS_ROOT_PATH/bin/analytics --debug --write-to-database --output /tmp

deploy/backload.js

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const { program } = require("commander");
+const { formatISO, parseISO, differenceInDays, subDays } = require("date-fns");
+const apiReports = require("../reports/api.json");
+const { exec } = require("child_process");
+const logger = require("../src/logger").initialize();
+
+/**
+ * Script to backload DAP API data for a particular date.
+ *
+ * Example run command:
+ * dotenv -e .env.analytics node -- ./backload.js --date 2024-01-01
+ */
+
+program
+  .option("-d, --date <string>", "date to run the backload for in ISO format")
+  .option(
+    "-e, --env <string>",
+    "dotenv file to load for the process",
+    ".env.analytics",
+  );
+
+program.parse();
+
+const options = program.opts();
+require("dotenv").config({ path: options.env });
+
+const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy`;
+
+const runScriptWithLogName = async (scriptPath, scriptLoggingName) => {
+  logger.info(`Beginning: ${scriptLoggingName}`);
+  logger.info(`File path: ${scriptPath}`);
+  const promise = new Promise((resolve) => {
+    const childProcess = exec(scriptPath);
+    childProcess.stdout.on("data", (data) => {
+      console.log(data.toString().trim());
+    });
+
+    childProcess.stderr.on("data", (data) => {
+      console.log(data.toString().trim());
+    });
+
+    childProcess.on("close", () => {
+      resolve();
+    });
+  });
+
+  await promise;
+  fs.unlinkSync("reports/api.json");
+  fs.renameSync("reports/api.original.json", "reports/api.json");
+};
+
+const api_run = () => {
+  return runScriptWithLogName(`${scriptRootPath}/api.sh`, "api.sh");
+};
+
+(async () => {
+  logger.info("===========================================");
+  logger.info("======= STARTING ANALYTICS-REPORTER =======");
+  logger.info(`Running /deploy/backload.js for: ${options.date}`);
+  logger.info("===========================================");
+
+  const scriptTargetDate = parseISO(options.date);
+  const today = new Date();
+  let totalDaysAgo = differenceInDays(today, scriptTargetDate);
+
+  while (totalDaysAgo >= 1) {
+    const iterationTargetDate = subDays(new Date(), totalDaysAgo);
+
+    const iterationDaysAgo = differenceInDays(today, iterationTargetDate) + 1;
+    const modifiedApiJsonString = JSON.stringify(apiReports)
+      .replaceAll('"yesterday"', `"${iterationDaysAgo}daysAgo"`)
+      .replaceAll('"3daysAgo"', `"${iterationDaysAgo}daysAgo"`);
+
+    fs.unlinkSync("reports/api.json");
+    fs.writeFileSync("reports/api.original.json", JSON.stringify(apiReports));
+    fs.writeFileSync("reports/api.json", modifiedApiJsonString);
+    fs.writeFileSync("reports/api.new.json", modifiedApiJsonString);
+
+    logger.info(
+      `Running API reports for ${formatISO(iterationTargetDate, { representation: "date" })}...`,
+    );
+    await api_run();
+    totalDaysAgo = totalDaysAgo - 1;
+    logger.info(
+      `API reports for ${formatISO(iterationTargetDate, { representation: "date" })} complete`,
+    );
+  }
+})();

manifest.yml

@@ -10,7 +10,7 @@ applications:
   health-check-type: process
   buildpacks:
   - nodejs_buildpack
-  command: node deploy/cron.js
+  command: (node deploy/backload.js --date 2024-06-19 && echo SUCCESS || echo FAIL) && sleep infinity
   env:
     # This is a JSON string which has been base64 encoded
     ANALYTICS_CREDENTIALS: ${ANALYTICS_CREDENTIALS}

package-lock.json

@@ -74,7 +74,9 @@
     "googleapis": "^131.0.0",
     "minimist": "^1.2.8",
     "p-retry": "^6.2.0",
-    "winston": "^3.11.0"
+    "winston": "^3.11.0",
+    "commander": "^12.1.0",
+    "date-fns": "^3.6.0"
   },
   "devDependencies": {
     "@cucumber/cucumber": "^10.3.1",