feat: database modernization (web worker + zod)

Author: TenFinges

packages/phoenix-event-display/package.json

@@ -46,7 +46,8 @@
     "jszip": "^3.10.1",
     "lodash": "^4.17.21",
     "stats-js": "^1.0.1",
-    "three": "~0.178.0"
+    "three": "~0.178.0",
+    "zod": "^4.3.5"
   },
   "devDependencies": {
     "@babel/helper-string-parser": "^7.27.1",

packages/phoenix-event-display/src/helpers/validation-schemas.ts

@@ -0,0 +1,48 @@
+import { z } from 'zod';
+
+/**
+ * Validates the structure of the 'Types' object in CMS event data.
+ * The keys are collection names, and the values are arrays of attribute definitions.
+ * Each attribute definition is an array where the first element is the attribute name.
+ */
+export const CMSTypesSchema = z.record(
+    z.string(),
+    z.array(
+        z.tuple([z.string()]).rest(z.union([z.string(), z.number()]))
+    )
+);
+
+/**
+ * Validates the structure of the 'Collections' object in CMS event data.
+ * The keys are collection names, and the values are arrays of objects (rows).
+ * Each row is an array of values corresponding to the attributes defined in 'Types'.
+ */
+export const CMSCollectionsSchema = z.record(
+    z.string(),
+    z.array(z.array(z.union([z.number(), z.string()])))
+);
+
+/**
+ * Validates a single item in an Association.
+ * Typically a tuple of [unknown, index].
+ */
+export const CMSAssociationItemSchema = z.array(z.tuple([z.number(), z.number()]));
+
+/**
+ * Validates the structure of the 'Associations' object in CMS event data.
+ */
+export const CMSAssociationsSchema = z.record(
+    z.string(),
+    z.array(CMSAssociationItemSchema)
+);
+
+/**
+ * Main schema for a CMS Event.
+ */
+export const CMSEventSchema = z.object({
+    Types: CMSTypesSchema,
+    Collections: CMSCollectionsSchema,
+    Associations: CMSAssociationsSchema.optional(),
+}).passthrough(); // Allow other properties like 'run number' etc if they exist at top level (though usually they are extracted)
+
+export type CMSEvent = z.infer<typeof CMSEventSchema>;

packages/phoenix-event-display/src/loaders/cms-loader.ts

@@ -11,13 +11,18 @@ export class CMSLoader extends PhoenixLoader {
   private data: any;
   /** Scale factor for resizing geometry to fit Phoenix event display. */
   private geometryScale: number = 1000;
+  /** Web Worker for off-main-thread parsing. */
+  private worker: Worker;
 
   /**
    * Constructor for the CMS loader.
    */
   constructor() {
     super();
     this.data = {};
+    // Initialize Web Worker
+    // Note: This relies on bundler support (Webpack 5+) for new Worker(new URL(...))
+    this.worker = new Worker(new URL('../workers/cms-loader.worker', import.meta.url));
   }
 
   /**
@@ -50,44 +55,45 @@ export class CMSLoader extends PhoenixLoader {
     this.loadingManager.addLoadableItem('ig_archive');
     const igArchive = new JSZip();
     const eventsDataInIg: any[] = [];
+
     const readArchive = (res: File | ArrayBuffer) => {
-      igArchive.loadAsync(res).then(() => {
+      igArchive.loadAsync(res).then(async () => {
         let allFilesPath = Object.keys(igArchive.files);
         // If the event path or name is given then filter all data to get the required events
         if (eventPathName) {
           allFilesPath = allFilesPath.filter((filePath) =>
             filePath.includes(eventPathName),
           );
         }
-        let i = 1;
+
+        // We'll process files sequentially or in parallel?
+        // Parallel could spawn too many workers messages. But `async` loop is fine.
+        let processedCount = 0;
+        const totalFiles = allFilesPath.length;
+
+        if (totalFiles === 0) {
+          this.loadingManager.itemLoaded('ig_archive');
+          return;
+        }
+
         for (const filePathInIg of allFilesPath) {
           // If the files are in the "Events" folder then process them.
           if (filePathInIg.toLowerCase().startsWith('events')) {
-            igArchive
-              ?.file(filePathInIg)!
-              .async('string')
-              .then((singleEvent: string) => {
-                // The data has some inconsistencies which need to be removed to properly parse JSON
-                singleEvent = singleEvent
-                  .replace(/'/g, '"')
-                  .replace(/\(/g, '[')
-                  .replace(/\)/g, ']')
-                  .replace(/nan/g, '0');
-                const eventJSON = JSON.parse(singleEvent);
-                eventJSON.eventPath = filePathInIg;
-                eventsDataInIg.push(eventJSON);
-                if (i === allFilesPath.length) {
-                  onFileRead(eventsDataInIg);
-                  this.loadingManager.itemLoaded('ig_archive');
-                }
-                i++;
-              });
-          } else {
-            if (i === allFilesPath.length) {
-              onFileRead(eventsDataInIg);
-              this.loadingManager.itemLoaded('ig_archive');
+            try {
+              const singleEvent = await igArchive.file(filePathInIg)!.async('string');
+              // Use Web Worker to parse
+              const eventJSON = await this.parseWithWorker(singleEvent, filePathInIg);
+              eventJSON.eventPath = filePathInIg;
+              eventsDataInIg.push(eventJSON);
+            } catch (error) {
+              console.error(`Error parsing event ${filePathInIg}:`, error);
             }
-            i++;
+          }
+
+          processedCount++;
+          if (processedCount === totalFiles) {
+            onFileRead(eventsDataInIg);
+            this.loadingManager.itemLoaded('ig_archive');
           }
         }
       });
@@ -104,6 +110,26 @@ export class CMSLoader extends PhoenixLoader {
     }
   }
 
+  /**
+   * Helper to send data to worker and await response.
+   */
+  private parseWithWorker(data: string, id: string): Promise<any> {
+    return new Promise((resolve, reject) => {
+      const handler = (event: MessageEvent) => {
+        if (event.data.id === id) {
+          this.worker.removeEventListener('message', handler);
+          if (event.data.type === 'parseCMSResult') {
+            resolve(event.data.data);
+          } else {
+            reject(event.data.error);
+          }
+        }
+      };
+      this.worker.addEventListener('message', handler);
+      this.worker.postMessage({ type: 'parseCMS', data, id });
+    });
+  }
+
   /**
    * Load event data from an ".ig" archive.
    * @param filePath Path to the ".ig" archive file.
@@ -118,7 +144,11 @@ export class CMSLoader extends PhoenixLoader {
     this.readIgArchive(
       filePath,
       (allEvents: any[]) => {
-        onEventRead(allEvents[0]);
+        if (allEvents.length > 0) {
+          onEventRead(allEvents[0]);
+        } else {
+          console.warn('No events found in archive');
+        }
       },
       eventPathName,
     );
@@ -573,3 +603,4 @@ export class CMSLoader extends PhoenixLoader {
     return metadata;
   }
 }
+

packages/phoenix-event-display/src/tests/loaders/cms-loader.worker.test.ts

@@ -0,0 +1,140 @@
+/**
+ * @jest-environment node
+ */
+import { CMSLoader } from '../../loaders/cms-loader';
+
+// Mock dependencies BEFORE imports
+jest.mock('../../loaders/phoenix-loader', () => {
+    return {
+        PhoenixLoader: class MockPhoenixLoader {
+            loadingManager = {
+                addLoadableItem: jest.fn(),
+                itemLoaded: jest.fn()
+            };
+            loadObjectTypes() { }
+            getEventMetadata() { return []; }
+            addObjectType() { }
+        }
+    };
+});
+
+jest.mock('three', () => ({
+    Vector3: class { constructor(x, y, z) { } },
+    QuadraticBezierCurve3: class { constructor() { } getPoints() { return [] } },
+    Group: class { }
+}));
+
+jest.mock('../../loaders/objects/cms-objects', () => ({
+    CMSObjects: {
+        getMuonChamber: jest.fn()
+    }
+}));
+
+const mockJSZipInstance = {
+    loadAsync: jest.fn().mockReturnThis(),
+    files: {},
+    file: jest.fn()
+};
+jest.mock('jszip', () => {
+    return jest.fn(() => mockJSZipInstance);
+});
+
+
+// Mock Worker globally
+const mockWorker = {
+    postMessage: jest.fn(),
+    addEventListener: jest.fn(),
+    removeEventListener: jest.fn(),
+    terminate: jest.fn(),
+};
+
+global.Worker = jest.fn(() => mockWorker) as any;
+global.URL = jest.fn() as any;
+
+
+describe('CMSLoader Web Worker Integration', () => {
+    let cmsLoader: CMSLoader;
+
+    beforeEach(() => {
+        jest.clearAllMocks();
+        // Reset JSZip mock state
+        mockJSZipInstance.loadAsync.mockResolvedValue({});
+        mockJSZipInstance.files = {};
+
+        cmsLoader = new CMSLoader();
+    });
+
+    it('should initialize Web Worker in constructor', () => {
+        expect(global.Worker).toHaveBeenCalledTimes(1);
+    });
+
+    it('should use worker for parsing when reading ig archive', async () => {
+        // Setup mock data
+        const eventData = JSON.stringify({ Collections: {}, Types: {} });
+        const mockFile = {
+            async: jest.fn().mockResolvedValue(eventData)
+        };
+        mockJSZipInstance.files = {
+            'Events/Run_1/Event_1': mockFile
+        };
+        mockJSZipInstance.file.mockReturnValue(mockFile);
+
+        // Access private method helper via type casting or invoke public method
+        const onRead = jest.fn();
+
+        // This is async internally but returns void. We need to wait for the async operations.
+        // Since we can't easily await the internal promise chain of readIgArchive without refactoring,
+        // we will test the `parseWithWorker` method directly if possible, or use timers.
+        // OR we can spy on `parseWithWorker` if we cast to any.
+
+        const parseWithWorkerSpy = jest.spyOn(cmsLoader as any, 'parseWithWorker');
+        parseWithWorkerSpy.mockResolvedValue({ eventPath: 'Events/Run_1/Event_1', successful: true });
+
+        cmsLoader.readIgArchive('dummy.ig', onRead);
+
+        // Wait for promises to resolve
+        await new Promise(resolve => setTimeout(resolve, 100));
+
+        expect(mockJSZipInstance.loadAsync).toHaveBeenCalled();
+        expect(parseWithWorkerSpy).toHaveBeenCalledWith(eventData, 'Events/Run_1/Event_1');
+        expect(onRead).toHaveBeenCalledWith(expect.arrayContaining([{ eventPath: 'Events/Run_1/Event_1', successful: true }]));
+    });
+
+    it('should handle worker messages correctly in parseWithWorker', async () => {
+        const promise = (cmsLoader as any).parseWithWorker('{}', 'test-id');
+
+        expect(mockWorker.postMessage).toHaveBeenCalledWith({
+            type: 'parseCMS',
+            data: '{}',
+            id: 'test-id'
+        });
+
+        // Simulate successful response
+        const messageHandler = (mockWorker.addEventListener as jest.Mock).mock.calls[0][1];
+        messageHandler({
+            data: {
+                type: 'parseCMSResult',
+                id: 'test-id',
+                data: { result: 'ok' }
+            }
+        });
+
+        const result = await promise;
+        expect(result).toEqual({ result: 'ok' });
+    });
+
+    it('should reject when worker returns error', async () => {
+        const promise = (cmsLoader as any).parseWithWorker('bad-json', 'error-id');
+
+        const messageHandler = (mockWorker.addEventListener as jest.Mock).mock.calls[0][1];
+        messageHandler({
+            data: {
+                type: 'parseCMSError',
+                id: 'error-id',
+                error: 'Invalid JSON'
+            }
+        });
+
+        await expect(promise).rejects.toEqual('Invalid JSON');
+    });
+});

packages/phoenix-event-display/src/workers/cms-loader.worker.ts

@@ -0,0 +1,49 @@
+// <reference lib="webworker" />
+import { CMSEventSchema } from '../helpers/validation-schemas';
+
+const ctx: Worker = self as any;
+
+ctx.addEventListener('message', (event) => {
+    const { data, type, id } = event.data;
+
+    if (type === 'parseCMS') {
+        try {
+            // 1. Clean the input string (logic from CMSLoader.ts)
+            let singleEvent = data;
+            singleEvent = singleEvent
+                .replace(/'/g, '"')
+                .replace(/\(/g, '[')
+                .replace(/\)/g, ']')
+                .replace(/nan/g, '0');
+
+            // 2. Parse JSON
+            const eventJSON = JSON.parse(singleEvent);
+
+            // 3. Validation
+            const validationResult = CMSEventSchema.safeParse(eventJSON);
+
+            if (validationResult.success) {
+                ctx.postMessage({
+                    type: 'parseCMSResult',
+                    id, // Pass back ID to correlate response
+                    data: validationResult.data
+                });
+            } else {
+                ctx.postMessage({
+                    type: 'parseCMSError',
+                    id,
+                    error: validationResult.error
+                });
+            }
+
+        } catch (error) {
+            ctx.postMessage({
+                type: 'parseCMSError',
+                id,
+                error: error instanceof Error ? error.message : String(error)
+            });
+        }
+    }
+});
+
+export default null as any;