CLOUDP-290416: Testing for ipa s3 dataDump

Author: lovisaberggren

package-lock.json

@@ -35,6 +35,7 @@
         "@eslint/js": "^9.18.0",
         "@jest/globals": "^29.7.0",
         "@stoplight/types": "^14.1.1",
+        "aws-sdk-client-mock": "^4.1.0",
         "babel-plugin-transform-import-meta": "^2.3.2",
         "eslint": "^9.18.0",
         "eslint-plugin-require-extensions": "^0.1.3",

package.json

@@ -0,0 +1,49 @@
+import { beforeEach, describe, expect, it, jest } from '@jest/globals';
+import { uploadMetricCollectionDataToS3 } from '../../metrics/metricS3Upload.js';
+import path from 'path';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { mockClient } from 'aws-sdk-client-mock';
+
+const testPrefix = 's3://bucket-name/path/to/file';
+const testTime = new Date(2020, 3, 1);
+const testMetricResultFilePath = path.join(__dirname, 'data', 'expected-metric-results.json');
+
+const clientMock = mockClient(S3Client);
+const filePathMock = { bucketName: 'bucket-name', key: 'path/to/file' };
+
+jest.mock('../../metrics/utils/dataDumpUtils.js', () => ({
+  ...jest.requireActual('../../metrics/utils/dataDumpUtils.js'),
+  getS3FilePath: jest.fn().mockReturnValue(filePathMock),
+}));
+
+describe('tools/spectral/ipa/metrics/metricS3Upload.js uploadMetricCollectionDataToS3', () => {
+  beforeEach(() => {
+    jest.useFakeTimers();
+    jest.setSystemTime(testTime);
+    jest.clearAllMocks();
+    clientMock.reset();
+    jest.resetModules();
+    process.env = {
+      AWS_ACCESS_KEY_ID: 'id',
+      AWS_SECRET_ACCESS_KEY: 'secret',
+      S3_BUCKET_PREFIX: testPrefix,
+    };
+  });
+
+  it('Outputs the expected metrics collection results', async () => {
+    await uploadMetricCollectionDataToS3(testMetricResultFilePath);
+
+    const clientCalls = clientMock.calls();
+    expect(clientCalls.length).toEqual(1);
+
+    const putObjectCommand = clientMock.commandCalls(PutObjectCommand);
+    expect(putObjectCommand.length).toEqual(1);
+
+    const input = putObjectCommand.at(0).args.at(0).input;
+    expect(input['Bucket']).toEqual(filePathMock.bucketName);
+    expect(input['Key']).toEqual(
+      path.join(filePathMock.key, testTime.toISOString().split('T')[0], 'metric-collection-results.parquet')
+    );
+    expect(input['Body']).not.toBe(undefined);
+  });
+});

tools/spectral/ipa/__tests__/metrics/metricS3Upload.test.js

@@ -0,0 +1,20 @@
+import { beforeEach, describe, expect, it, jest } from '@jest/globals';
+import { getS3FilePath } from '../../../metrics/utils/dataDumpUtils.js';
+
+describe('tools/spectral/ipa/metrics/utils/dataDumpUtils.js getS3FilePath', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    jest.resetModules();
+    process.env = {
+      AWS_ACCESS_KEY_ID: 'id',
+      AWS_SECRET_ACCESS_KEY: 'secret',
+      S3_BUCKET_PREFIX: 's3://bucket-name/path/to/file',
+    };
+  });
+
+  it('Parses the S3 file path correctly', () => {
+    const result = getS3FilePath();
+    expect(result['bucketName']).toEqual('bucket-name');
+    expect(result['key']).toEqual('path/to/file');
+  });
+});

tools/spectral/ipa/__tests__/metrics/utils/dataDumpUtils.test.js

@@ -6,7 +6,7 @@ import {
   loadOpenAPIFile,
   loadRuleset,
   merge,
-} from './utils.js';
+} from './utils/metricCollectionUtils.js';
 
 export async function runMetricCollectionJob(
   {

tools/spectral/ipa/metrics/metricCollection.js

@@ -0,0 +1,42 @@
+import { PutObjectCommand, S3ServiceException } from '@aws-sdk/client-s3';
+import config from './config.js';
+import path from 'path';
+import fs from 'node:fs';
+import { tableFromJSON, tableToIPC } from 'apache-arrow';
+import { getS3Client, getS3FilePath } from './utils/dataDumpUtils.js';
+
+/**
+ * Upload IPA product metrics to Data Warehouse S3
+ * @param filePath file path to the metrics collection results, uses config.js by default
+ */
+export async function uploadMetricCollectionDataToS3(filePath = config.defaultMetricCollectionResultsFilePath) {
+  const client = getS3Client();
+  const formattedDate = new Date().toISOString().split('T')[0];
+  const metricsCollectionData = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  const table = tableFromJSON(metricsCollectionData);
+
+  const s3fileProps = getS3FilePath();
+  const command = new PutObjectCommand({
+    Bucket: s3fileProps.bucketName,
+    Key: path.join(s3fileProps.key, formattedDate, 'metric-collection-results.parquet'),
+    Body: tableToIPC(table, 'stream'),
+  });
+
+  try {
+    const response = await client.send(command);
+    console.log(response);
+  } catch (caught) {
+    if (caught instanceof S3ServiceException && caught.name === 'EntityTooLarge') {
+      console.error(
+        `Error from S3 while uploading object. The object was too large. \
+        To upload objects larger than 5GB, use the S3 console (160GB max) or the multipart upload API (5TB max).`
+      );
+      throw caught;
+    } else if (caught instanceof S3ServiceException) {
+      console.error(`Error from S3 while uploading object.  ${caught.name}: ${caught.message}`);
+      throw caught;
+    } else {
+      throw caught;
+    }
+  }
+}