Refactor manifest storage

Author: jeremydw

src/server.ts

@@ -16,10 +16,10 @@ interface ManifestCache {
   [requestSha: string]: string;
 }
 
-const downloadManifest = async (branchOrRef: string) => {
+const getManifest = async (siteId: string, branchOrRef: string) => {
   const keys = [
-    datastore.key(['Fileset2Manifest', branchOrRef]),
-    datastore.key(['Fileset2Manifest', `branch:${branchOrRef}`]),
+    datastore.key(['Fileset2Manifest', `${siteId}:branch:${branchOrRef}`]),
+    datastore.key(['Fileset2Manifest', `${siteId}:ref:${branchOrRef}`]),
   ];
   const resp = await datastore.get(keys);
   if (!resp || !resp[0]) {
@@ -30,28 +30,23 @@ const downloadManifest = async (branchOrRef: string) => {
   if (!result) {
     return;
   }
-  if (!result.ttls) {
-    return result;
-  }
 
-  if (result.ttls) {
-    // TODO: Allow this to be overwritten.
-    const now = new Date();
-    let latestManifest = null;
-    for (const ttlString in result.ttls) {
-      const ttlDate = new Date(ttlString);
-      const isLaterThanManifestDate = now >= ttlDate;
-      const isLaterThanAllManifests =
-        !latestManifest || ttlDate >= latestManifest.ttl;
-      if (isLaterThanManifestDate && isLaterThanAllManifests) {
-        latestManifest = result.ttls[ttlString];
-        latestManifest.ttl = ttlDate;
-      }
-    }
-    if (latestManifest) {
-      return latestManifest;
+  // TODO: Allow this to be overwritten.
+  const now = new Date();
+  let latestManifest = null;
+  for (const ttlString in result.schedule) {
+    const ttlDate = new Date(ttlString);
+    const isLaterThanManifestDate = now >= ttlDate;
+    const isLaterThanAllManifests =
+      !latestManifest || ttlDate >= latestManifest.ttl;
+    if (isLaterThanManifestDate && isLaterThanAllManifests) {
+      latestManifest = result.schedule[ttlString];
+      latestManifest.ttl = ttlDate;
     }
   }
+  if (latestManifest) {
+    return latestManifest;
+  }
   return result;
 };
 
@@ -73,7 +68,7 @@ const parseHostname = (hostname: string) => {
 };
 
 export function createApp(siteId: string, branchOrRef: string) {
-  // const startupManifest = await downloadManifest(branchOrRef);
+  // const startupManifest = await getManifest(branchOrRef);
   console.log(`Starting server for site: ${siteId} @ ${branchOrRef}`);
 
   const app = express();
@@ -88,7 +83,7 @@ export function createApp(siteId: string, branchOrRef: string) {
       blobPath += 'index.html';
     }
 
-    const manifest = await downloadManifest(requestBranchOrRef);
+    const manifest = await getManifest(requestSiteId, requestBranchOrRef);
     if (!manifest) {
       res
         .status(404)
@@ -99,7 +94,6 @@ export function createApp(siteId: string, branchOrRef: string) {
     }
 
     const manifestPaths = manifest.paths;
-
     if (!manifestPaths) {
       res
         .status(404)
@@ -129,7 +123,7 @@ export function createApp(siteId: string, branchOrRef: string) {
       preserveHeaderKeyCase: true,
     });
     server.on('error', (error, req, res) => {
-      console.log(`An error occurred while serving ${req.url}`, error);
+      console.log(`An error occurred while serving ${req.url} (${error})`);
     });
     server.on('proxyRes', (proxyRes, req, res) => {
       delete proxyRes.headers['x-cloud-trace-context'];

src/upload.ts

@@ -130,54 +130,85 @@ export async function uploadManifest(
   }
 }
 
-async function createEntity(key: entity.Key, manifest: Manifest, ttl?: Date) {
-  const manifestPaths = manifest.toJSON();
-  let props: any = {
-    created: new Date(),
-    site: manifest.site,
+interface ScheduleItem {
+  ttl: Date;
+  ref: string;
+  paths: Record<string, string>;
+}
+
+function createScheduleItem(
+  manifest: Manifest,
+  manifestPaths: Record<string, string>,
+  ttl: Date
+) {
+  return {
+    ttl: ttl,
     ref: manifest.ref,
-    branch: manifest.branch,
     paths: manifestPaths,
-  };
-  if (ttl) {
-    const key = datastore.key([
-      'Fileset2Manifest',
-      `branch:${manifest.branch}`,
-    ]);
-    const resp = await datastore.get(key);
-    if (resp && resp[0]) {
-      console.log(
-        `Adding TTL for branch ${manifest.branch} -> ${manifest.shortSha} @ ${ttl}`
-      );
-      const entity = resp[0];
-      if (entity.ttls) {
-        entity.ttls[ttl.toString()] = props;
-      } else {
-        entity.ttls = {ttl: props};
-      }
-      props = entity;
-    }
-  }
-  return {
-    key: key,
-    excludeFromIndexes: ['paths', 'ttls'],
-    data: props,
-  };
+  } as ScheduleItem;
 }
 
 async function finalize(manifest: Manifest, ttl?: Date) {
+  const manifestPaths = manifest.toJSON();
+  const now = new Date();
+
   // Create shortSha mapping.
-  const key = datastore.key(['Fileset2Manifest', manifest.shortSha]);
-  const ent = await createEntity(key, manifest);
+  const key = datastore.key([
+    'Fileset2Manifest',
+    `${manifest.site}:ref:${manifest.shortSha}`,
+  ]);
+  const scheduleItem = createScheduleItem(manifest, manifestPaths, now);
+  const schedule: Record<string, ScheduleItem> = {};
+  schedule[now.toString()] = scheduleItem;
+  const ent = {
+    key: key,
+    excludeFromIndexes: ['schedule'],
+    data: {
+      site: manifest.site,
+      ref: manifest.ref,
+      branch: manifest.branch,
+      schedule: schedule,
+    },
+  };
   await datastore.save(ent);
 
   // Create branch mapping.
   if (manifest.branch) {
     const branchKey = datastore.key([
       'Fileset2Manifest',
-      `branch:${manifest.branch}`,
+      `${manifest.site}:branch:${manifest.branch}`,
     ]);
-    const branchEnt = await createEntity(branchKey, manifest, ttl);
+    const branchScheduleItem = createScheduleItem(
+      manifest,
+      manifestPaths,
+      ttl || now
+    );
+    const branchSchedule: Record<string, ScheduleItem> = {};
+    const branchScheduleKey = (ttl || now).toString();
+    branchSchedule[branchScheduleKey] = branchScheduleItem;
+    const resp = await datastore.get(branchKey);
+    let existingData = resp && resp[0];
+    if (!existingData) {
+      existingData = {
+        site: manifest.site,
+        ref: manifest.ref,
+        branch: manifest.branch,
+        schedule: branchSchedule,
+      };
+    } else {
+      // TODO: Clean up past scheduled items here.
+      existingData.schedule[branchScheduleKey] = branchScheduleItem;
+    }
+    console.log(
+      `TTLs for branch: ${manifest.branch} -> ${Object.keys(
+        existingData.schedule
+      )}`
+    );
+    const branchEnt = {
+      key: branchKey,
+      excludeFromIndexes: ['schedule'],
+      data: existingData,
+    };
     await datastore.save(branchEnt);
   }