CLOUDP-297242: Fix parquet file creation for IPA metrics

.gitignore

@@ -22,6 +22,6 @@
 
 *.out
 **/*ipa-collector-results-combined.log
-**/*metric-collection-results.json
+**/*metric-collection-results.parquet
 **/*spectral-output.txt
 **/*spectral-report.xml

package.json

@@ -28,7 +28,8 @@
         "apache-arrow": "^19.0.0",
         "dotenv": "^16.4.7",
         "eslint-plugin-jest": "^28.10.0",
-        "openapi-to-postmanv2": "4.24.0"
+        "openapi-to-postmanv2": "4.24.0",
+        "parquet-wasm": "^0.6.1"
     },
     "devDependencies": {
         "@babel/preset-env": "^7.26.0",

tools/spectral/ipa/metrics/config.js

@@ -11,6 +11,9 @@ const config = {
   defaultOutputsDir: path.join(dirname, 'outputs'),
 };
 
-config.defaultMetricCollectionResultsFilePath = path.join(config.defaultOutputsDir, 'metric-collection-results.json');
+config.defaultMetricCollectionResultsFilePath = path.join(
+  config.defaultOutputsDir,
+  'metric-collection-results.parquet'
+);
 
 export default config;

tools/spectral/ipa/metrics/metricS3Upload.js

@@ -2,7 +2,6 @@ import { PutObjectCommand, S3ServiceException } from '@aws-sdk/client-s3';
 import config from './config.js';
 import path from 'path';
 import fs from 'node:fs';
-import { tableFromJSON, tableToIPC } from 'apache-arrow';
 import { getS3Client, getS3FilePath } from './utils/dataDumpUtils.js';
 
 /**
@@ -11,14 +10,9 @@ import { getS3Client, getS3FilePath } from './utils/dataDumpUtils.js';
  */
 export async function uploadMetricCollectionDataToS3(filePath = config.defaultMetricCollectionResultsFilePath) {
   console.log('Loading metrics collection data from', filePath);
-  const metricsCollectionData = JSON.parse(fs.readFileSync(filePath, 'utf8'));
-  if (metricsCollectionData === undefined || metricsCollectionData.length === 0) {
-    throw new Error('Loaded metrics collection data is empty');
-  }
-
-  const table = tableFromJSON(metricsCollectionData);
-  if (table === undefined) {
-    throw new Error('Unable to transform metrics collection data to table');
+  const metricsData = await fs.readFileSync(filePath);
+  if (metricsData === undefined) {
+    throw new Error('Loaded metrics collection data is undefined');
   }
 
   try {
@@ -32,7 +26,7 @@ export async function uploadMetricCollectionDataToS3(filePath = config.defaultMe
     const command = new PutObjectCommand({
       Bucket: s3fileProps.bucketName,
       Key: path.join(s3fileProps.key, formattedDate, 'metric-collection-results.parquet'),
-      Body: tableToIPC(table, 'stream'),
+      Body: metricsData,
     });
 
     console.log('Dumping data to S3...');

tools/spectral/ipa/metrics/scripts/runMetricCollection.js

@@ -1,6 +1,8 @@
 import fs from 'node:fs';
 import { spawnSync } from 'child_process';
 import spectral from '@stoplight/spectral-core';
+import { Compression, Table, writeParquet, WriterPropertiesBuilder } from 'parquet-wasm';
+import { tableFromJSON, tableToIPC } from 'apache-arrow';
 import config from '../config.js';
 import { runMetricCollectionJob } from '../metricCollection.js';
 
@@ -51,6 +53,12 @@ runMetricCollectionJob(
 )
   .then((results) => {
     console.log('Writing results');
-    fs.writeFileSync(config.defaultMetricCollectionResultsFilePath, JSON.stringify(results));
+    const table = tableFromJSON(results);
+    const wasmTable = Table.fromIPCStream(tableToIPC(table, 'stream'));
+    const parquetUint8Array = writeParquet(
+      wasmTable,
+      new WriterPropertiesBuilder().setCompression(Compression.GZIP).build()
+    );
+    fs.writeFileSync(config.defaultMetricCollectionResultsFilePath, parquetUint8Array);
   })
   .catch((error) => console.error(error.message));