Add TTL support

Author: jeremydw

src/commands/serve.ts

@@ -3,6 +3,7 @@ import {createApp} from '../server';
 interface ServeOptions {
   site: string;
   ref?: string;
+  fcd?: string;
 }
 
 export class ServeCommand {

src/commands/upload.ts

@@ -8,6 +8,7 @@ interface UploadOptions {
   site: string;
   ref?: string;
   branch?: string;
+  ttl?: string;
 }
 
 export class UploadCommand {
@@ -27,6 +28,12 @@ export class UploadCommand {
       console.log(`No files found in -> ${path}`);
       return;
     }
-    upload.uploadManifest(this.options.bucket, manifest, this.options.force);
+    const ttl = this.options.ttl ? new Date(this.options.ttl) : undefined;
+    upload.uploadManifest(
+      this.options.bucket,
+      manifest,
+      this.options.force,
+      ttl
+    );
   }
 }

src/index.ts

@@ -9,11 +9,10 @@ const program = createCommand();
 program
   .command('upload [dir]')
   .description('Uploads a directory to cloud storage')
-  .option('-b, --bucket <bucket>', 'bucket', '')
   .option('-s, --site <site>', 'site', '')
   .option('-r, --ref <ref>', 'ref', '')
-  .option('-f, --force', '', false)
-  .option('-br, --branch <branch>', 'branch', '')
+  .option('-f, --force', 'force', false)
+  .option('-t, --ttl <ttl>', 'ttl', undefined)
   .action((path, options) => {
     const cmd = new UploadCommand(options);
     cmd.run(path);

src/server.ts

@@ -3,7 +3,7 @@ import {GoogleAuth} from 'google-auth-library';
 import * as fsPath from 'path';
 import express = require('express');
 import httpProxy = require('http-proxy');
-import { Manifest } from './manifest';
+import {Manifest} from './manifest';
 
 const URL = 'https://storage.googleapis.com';
 const datastore = new Datastore();
@@ -26,7 +26,33 @@ const downloadManifest = async (branchOrRef: string) => {
     return;
   }
   const entities = resp[0];
-  return entities[0] || entities[1];
+  const result = entities[0] || entities[1];
+  if (!result) {
+    return;
+  }
+  if (!result.ttls) {
+    return result;
+  }
+
+  if (result.ttls) {
+    // TODO: Allow this to be overwritten.
+    const now = new Date();
+    let latestManifest = null;
+    for (const ttlString in result.ttls) {
+      const ttlDate = new Date(ttlString);
+      const isLaterThanManifestDate = now >= ttlDate;
+      const isLaterThanAllManifests =
+        !latestManifest || ttlDate >= latestManifest.ttl;
+      if (isLaterThanManifestDate && isLaterThanAllManifests) {
+        latestManifest = result.ttls[ttlString];
+        latestManifest.ttl = ttlDate;
+      }
+    }
+    if (latestManifest) {
+      return latestManifest;
+    }
+  }
+  return result;
 };
 
 const parseHostname = (hostname: string) => {
@@ -63,6 +89,12 @@ export function createApp(siteId: string, branchOrRef: string) {
     }
 
     const manifest = await downloadManifest(requestBranchOrRef);
+    if (!manifest) {
+      // TODO: Consolidate not found errors.
+      res.sendStatus(404);
+      return;
+    }
+
     const manifestPaths = manifest.paths;
 
     if (!manifestPaths) {
@@ -89,6 +121,9 @@ export function createApp(siteId: string, branchOrRef: string) {
       changeOrigin: true,
       preserveHeaderKeyCase: true,
     });
+    server.on('error', (error, req, res) => {
+      console.log(`An error occurred while serving ${req.url}`, error);
+    });
     server.on('proxyRes', (proxyRes, req, res) => {
       delete proxyRes.headers['x-cloud-trace-context'];
       delete proxyRes.headers['x-goog-generation'];
@@ -103,10 +138,13 @@ export function createApp(siteId: string, branchOrRef: string) {
       delete proxyRes.headers['x-guploader-uploadid'];
       // This cannot be "private, max-age=0" as this kills perf.
       // This also can't be a very small value, as it kills perf. 0036 seems to work correctly.
-      proxyRes.headers['cache-control'] = 'public, max-age=0036';  // The padded 0036 keeps the content length the same per upload.ts.
+      proxyRes.headers['cache-control'] = 'public, max-age=0036'; // The padded 0036 keeps the content length the same per upload.ts.
       proxyRes.headers['x-fileset-blob'] = blobKey;
       proxyRes.headers['x-fileset-ref'] = manifest.ref;
       proxyRes.headers['x-fileset-site'] = siteId;
+      if (manifest.ttl) {
+        proxyRes.headers['x-fileset-ttl'] = manifest.ttl;
+      }
       res.writeHead(proxyRes.statusCode || 200, proxyRes.headers);
     });
   });

src/upload.ts

@@ -42,7 +42,8 @@ const findUploadedFiles = async (manifest: Manifest, storageBucket: any) => {
 export async function uploadManifest(
   bucket: string,
   manifest: Manifest,
-  force?: boolean
+  force?: boolean,
+  ttl?: Date
 ) {
   bucket = bucket || DEFAULT_BUCKET; // If bucket is blank.
   console.log(`Using storage: ${bucket}/${getBlobPath(manifest.site, '')}`);
@@ -122,37 +123,61 @@ export async function uploadManifest(
 
     // @ts-ignore
     bar.on('stop', () => {
-      finalize(manifest);
+      finalize(manifest, ttl);
     });
   } else {
-    finalize(manifest);
+    finalize(manifest, ttl);
   }
 }
 
-function createEntity(key: entity.Key, manifest: Manifest) {
+async function createEntity(key: entity.Key, manifest: Manifest, ttl?: Date) {
+  const manifestPaths = manifest.toJSON();
+  let props: any = {
+    created: new Date(),
+    site: manifest.site,
+    ref: manifest.ref,
+    branch: manifest.branch,
+    paths: manifestPaths,
+  };
+  if (ttl) {
+    const key = datastore.key([
+      'Fileset2Manifest',
+      `branch:${manifest.branch}`,
+    ]);
+    const resp = await datastore.get(key);
+    if (resp && resp[0]) {
+      console.log(
+        `Adding TTL for branch ${manifest.branch} -> ${manifest.shortSha} @ ${ttl}`
+      );
+      const entity = resp[0];
+      if (entity.ttls) {
+        entity.ttls[ttl.toString()] = props;
+      } else {
+        entity.ttls = {ttl: props};
+      }
+      props = entity;
+    }
+  }
   return {
     key: key,
-    excludeFromIndexes: ['paths'],
-    data: {
-      created: new Date(),
-      ref: manifest.ref,
-      branch: manifest.branch,
-      paths: manifest.toJSON(),
-    },
+    excludeFromIndexes: ['paths', 'ttls'],
+    data: props,
   };
 }
 
-async function finalize(manifest: Manifest) {
+async function finalize(manifest: Manifest, ttl?: Date) {
+  // Create shortSha mapping.
   const key = datastore.key(['Fileset2Manifest', manifest.shortSha]);
-  const ent = createEntity(key, manifest);
+  const ent = await createEntity(key, manifest);
   await datastore.save(ent);
 
+  // Create branch mapping.
   if (manifest.branch) {
     const branchKey = datastore.key([
       'Fileset2Manifest',
       `branch:${manifest.branch}`,
     ]);
-    const branchEnt = createEntity(branchKey, manifest);
+    const branchEnt = await createEntity(branchKey, manifest, ttl);
     await datastore.save(branchEnt);
   }