Compress manifest sources in the database (#1405)

Author: justinfagnani

packages/catalog-server/src/lib/firestore/firestore-repository.ts

@@ -4,6 +4,11 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import {
+  brotliCompress as brotliCompressCallbackStyle,
+  brotliDecompress as brotliDecompressCallbackStyle,
+} from 'node:zlib';
+import {promisify} from 'node:util';
 import {
   FieldValue,
   Query,
@@ -31,6 +36,7 @@ import {
   ReadablePackageVersion,
   ReadablePackageInfo,
   UnreadablePackageStatus,
+  UnreadablePackageVersion,
 } from '@webcomponents/catalog-api/lib/schema.js';
 import {
   Package,
@@ -41,14 +47,22 @@ import {
   packageInfoConverter,
   packageNameToId,
 } from './package-info-converter.js';
-import {packageVersionConverter} from './package-version-converter.js';
+import {
+  CompressedPackageVersion,
+  isReadableCompressedPackageVersion,
+  packageVersionConverter,
+  ReadableCompressedPackageVersion,
+} from './package-version-converter.js';
 import {customElementConverter} from './custom-element-converter.js';
 import {validationProblemConverter} from './validation-problem-converter.js';
 
 const projectId = process.env['GCP_PROJECT_ID'] || 'wc-catalog';
 firebase.initializeApp({projectId});
 export const db = new Firestore({projectId});
 
+const brotliCompress = promisify(brotliCompressCallbackStyle);
+const brotliDecompress = promisify(brotliDecompressCallbackStyle);
+
 export class FirestoreRepository implements Repository {
   /**
    * A namespace suffix to apply to the 'packages' collection to support
@@ -234,8 +248,13 @@ export class FirestoreRepository implements Repository {
       const distTags = packageMetadata['dist-tags'];
       const versionDistTags = getDistTagsForVersion(distTags, version);
 
+      const compressedManifest =
+        customElementsManifestSource &&
+        (await brotliCompress(customElementsManifestSource)).toString('base64');
+
       // Store package data and mark version as ready
       t.set(versionRef, {
+        __typename: 'ReadableCompressedPackageVersion',
         name: packageName,
         version,
         status: VersionStatus.READY,
@@ -246,14 +265,23 @@ export class FirestoreRepository implements Repository {
         author,
         time: new Date(packageTime),
         homepage: packageVersionMetadata.homepage ?? null,
-        customElementsManifest: customElementsManifestSource ?? null,
+        customElementsManifestCompressed: compressedManifest,
       });
     });
     const packageVersion = await db.runTransaction(async (t) => {
       // There doesn't seem to be a way to get a WriteResult and therefore
       // a writeTime inside a transaction, so we read from the database to
       // get the server timestamp.
-      return (await t.get(versionRef)).data() as ReadablePackageVersion;
+      const packageVersionCompressed = (await t.get(versionRef)).data()!;
+      if (isReadableCompressedPackageVersion(packageVersionCompressed)) {
+        return decompressPackageVersion(
+          packageVersionCompressed,
+          versionRef.id
+        );
+      }
+      throw new Error(
+        `Internal error: expected package version ${versionRef.id} to be readable`
+      );
     });
     return packageVersion;
   }
@@ -282,14 +310,14 @@ export class FirestoreRepository implements Repository {
       t.update(versionRef, {
         status,
         lastUpdate: FieldValue.serverTimestamp(),
-      } as UpdateData<PackageVersion> as PackageVersion);
+      } as UpdateData<UnreadablePackageVersion> as UnreadablePackageVersion);
     });
     const packageVersion = await db.runTransaction(async (t) => {
       // There doesn't seem to be a way to get a WriteResult and therefore
       // a writeTime inside a transaction, so we read from the database to
       // get the server timestamp.
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-      return (await t.get(versionRef)).data()!;
+      return (await t.get(versionRef)).data() as UnreadablePackageVersion;
     });
     return packageVersion;
   }
@@ -439,7 +467,14 @@ export class FirestoreRepository implements Repository {
       // If version is valid semver we can build a document reference.
       const versionRef = this.getPackageVersionRef(packageName, versionNumber);
       const versionDoc = await versionRef.get();
-      return versionDoc.data();
+      const packageVersion = versionDoc.data();
+      if (
+        packageVersion &&
+        isReadableCompressedPackageVersion(packageVersion)
+      ) {
+        return decompressPackageVersion(packageVersion, versionRef.id);
+      }
+      return packageVersion;
     } else {
       // If version is not a valid semver it may be a dist-tag
 
@@ -450,7 +485,9 @@ export class FirestoreRepository implements Repository {
       }
 
       // Now query for a version that's assigned this dist-tag
-      let query: CollectionReference<PackageVersion> | Query<PackageVersion> =
+      let query:
+        | CollectionReference<CompressedPackageVersion>
+        | Query<CompressedPackageVersion> =
         this.getPackageVersionCollectionRef(packageName);
       if (versionOrTag === 'latest') {
         query = query.where('isLatest', '==', true);
@@ -459,7 +496,16 @@ export class FirestoreRepository implements Repository {
       }
       const result = await query.limit(1).get();
       if (result.size !== 0) {
-        return result.docs[0]!.data();
+        const doc = result.docs[0]!;
+        const packageVersion = doc.data();
+        // Decompress the custom elements manifest.
+        // Note: We'd like to do this in the packageVersionConverter Firestore
+        // converter so that we don't have to remember to compress/decompress
+        // at every read and write operation, but we can't because we also want
+        // this to be an async operation and not block the main thread.
+        if (isReadableCompressedPackageVersion(packageVersion)) {
+          return decompressPackageVersion(packageVersion, doc.id);
+        }
       }
       return undefined;
     }
@@ -549,6 +595,28 @@ export class FirestoreRepository implements Repository {
   }
 }
 
+export const decompressPackageVersion = async (
+  packageVersion: ReadableCompressedPackageVersion,
+  id: string
+): Promise<ReadablePackageVersion> => {
+  const manifestCompressed = packageVersion.customElementsManifestCompressed;
+  try {
+    const customElementsManifest =
+      manifestCompressed &&
+      (
+        await brotliDecompress(Buffer.from(manifestCompressed, 'base64'))
+      ).toString();
+    return {
+      ...packageVersion,
+      __typename: undefined,
+      customElementsManifest,
+    };
+  } catch (e) {
+    console.error(`Filed to decompress manifest for package version ${id}`);
+    throw e;
+  }
+};
+
 // /**
 //  * Generates a type representing a Firestore document from a GraphQL schema
 //  * type.

packages/catalog-server/src/lib/firestore/package-version-converter.ts

@@ -14,43 +14,70 @@ import {
 import {
   isReadablePackageVersion,
   PackageVersion,
+  ReadablePackageVersion,
+  UnreadablePackageVersion,
 } from '@webcomponents/catalog-api/lib/schema.js';
 
-export const packageVersionConverter: FirestoreDataConverter<PackageVersion> = {
-  fromFirestore(snapshot: QueryDocumentSnapshot<DocumentData>): PackageVersion {
-    return {
-      status: snapshot.get('status'),
-      lastUpdate: (snapshot.get('lastUpdate') as Timestamp).toDate(),
-      version: snapshot.id,
-      distTags: snapshot.get('distTags'),
-      description: snapshot.get('description'),
-      type: snapshot.get('type'),
-      author: snapshot.get('author'),
-      time: snapshot.get('time'),
-      homepage: snapshot.get('homepage'),
-      customElements: snapshot.get('customElements'),
-      customElementsManifest: snapshot.get('customElementsManifest'),
-    };
-  },
-  toFirestore(packageVersion: PackageVersion) {
-    if (isReadablePackageVersion(packageVersion)) {
-      return {
-        author: packageVersion.author,
-        customElementsManifest: packageVersion.customElementsManifest,
-        description: packageVersion.description,
-        distTags: packageVersion.distTags,
-        isLatest: packageVersion.distTags.includes('latest'),
-        homepage: packageVersion.homepage,
-        lastUpdate: packageVersion.lastUpdate,
-        status: packageVersion.status,
-        time: Timestamp.fromDate(packageVersion.time),
-        type: packageVersion.type,
-      };
-    } else {
+export type ReadableCompressedPackageVersion = Omit<
+  ReadablePackageVersion,
+  'customElementsManifest' | '__typename'
+> & {
+  customElementsManifestCompressed?: string;
+  // Prevent this from being assignable to a ReadablePackageVersion to force
+  // consumer to convert correctly.
+  __typename: 'ReadableCompressedPackageVersion';
+};
+
+export type CompressedPackageVersion =
+  | ReadableCompressedPackageVersion
+  | UnreadablePackageVersion;
+
+export const packageVersionConverter: FirestoreDataConverter<CompressedPackageVersion> =
+  {
+    fromFirestore(
+      snapshot: QueryDocumentSnapshot<DocumentData>
+    ): CompressedPackageVersion {
       return {
-        status: packageVersion.status,
-        lastUpdate: packageVersion.lastUpdate,
+        status: snapshot.get('status'),
+        lastUpdate: (snapshot.get('lastUpdate') as Timestamp).toDate(),
+        version: snapshot.id,
+        distTags: snapshot.get('distTags'),
+        description: snapshot.get('description'),
+        type: snapshot.get('type'),
+        author: snapshot.get('author'),
+        time: snapshot.get('time'),
+        homepage: snapshot.get('homepage'),
+        customElements: snapshot.get('customElements'),
+        customElementsManifestCompressed: snapshot.get(
+          'customElementsManifest'
+        ),
       };
-    }
-  },
-};
+    },
+    toFirestore(packageVersion: CompressedPackageVersion) {
+      if (isReadableCompressedPackageVersion(packageVersion)) {
+        return {
+          author: packageVersion.author,
+          customElementsManifest:
+            packageVersion.customElementsManifestCompressed,
+          description: packageVersion.description,
+          distTags: packageVersion.distTags,
+          isLatest: packageVersion.distTags.includes('latest'),
+          homepage: packageVersion.homepage,
+          lastUpdate: packageVersion.lastUpdate,
+          status: packageVersion.status,
+          time: Timestamp.fromDate(packageVersion.time),
+          type: packageVersion.type,
+        };
+      } else {
+        return {
+          status: packageVersion.status,
+          lastUpdate: packageVersion.lastUpdate,
+        };
+      }
+    },
+  };
+
+export const isReadableCompressedPackageVersion = (
+  p: CompressedPackageVersion
+): p is ReadableCompressedPackageVersion =>
+  isReadablePackageVersion(p as PackageVersion);

packages/catalog-server/src/test/lib/catalog_test.ts

@@ -358,4 +358,42 @@ test('Imports a non-existent package', async () => {
   assert.equal(importResult.packageInfo?.status, PackageStatus.NOT_FOUND);
 });
 
+test('Imports a large manifest', async () => {
+  const packageName = 'large-manifest';
+  const packagePath = fileURLToPath(
+    new URL('../test-packages/large-manifest/', import.meta.url)
+  );
+
+  const files = new LocalFsPackageFiles({
+    path: packagePath,
+    packageName,
+    publishedVersions: ['1.0.0'],
+    distTags: {
+      latest: '1.0.0',
+    },
+  });
+  const repository = new FirestoreRepository(TEST_SEQUENCE_THREE);
+  const catalog = new Catalog({files, repository});
+  const importResult = await catalog.importPackage(packageName);
+
+  assert.equal(importResult.problems ?? [], []);
+  assert.equal(importResult.packageInfo?.status, PackageStatus.READY);
+
+  assert.ok(importResult.packageVersion);
+  assert.equal(importResult.packageVersion.status, VersionStatus.READY);
+  const manifest = (importResult.packageVersion as ReadablePackageVersion)
+    .customElementsManifest;
+  assert.ok(manifest);
+  const parsedManifest = JSON.parse(manifest);
+  assert.equal(parsedManifest.schemaVersion, '1.0.0');
+
+  // Check that getting the version through getPackageVersion() decompresses
+  const packageVersion2 = await catalog.getPackageVersion(packageName, '1.0.0');
+  const manifest2 = (packageVersion2 as ReadablePackageVersion)
+    .customElementsManifest;
+  assert.ok(manifest2);
+  const parsedManifest2 = JSON.parse(manifest2);
+  assert.equal(parsedManifest2.schemaVersion, '1.0.0');
+});
+
 test.run();