Update documentation

Author: jeremydw

README.md

@@ -36,50 +36,113 @@ serve new files to users, or update redirects, etc.
 NOTE: Fileset uses the default App Engine Google Cloud Storage bucket
 (`gs://appid.appspot.com`) to upload files.
 
-1. Install the Fileset CLI.
-
-```bash
-npm install --save-dev @blinkk/fileset
+1. Within your project, create a directory to house the server files, e.g.
+   `./backend/fileset`. Avoid mixing the server configuration with any tools to
+   build your website. The `package.json`, etc. should be kept separate in order
+   to keep the deployment slim.
+
+2. Create a `package.json` like the below. App Engine will use `npm start` to
+   run the server.
+
+```json
+{
+  "scripts": {
+    "start": "fileset serve"
+  },
+  "dependencies": {
+    "@blinkk/fileset": "^0.1.0"
+  }
+}
 ```
 
-2. Create an `app.yaml` for Google App Engine deployment.
+3. Create an `app.yaml` for Google App Engine deployment.
 
 ```yaml
 service: fileset
 runtime: nodejs10
-entrypoint: fileset serve
 ```
 
-3. Deploy the app.
+4. Deploy the app and enable the Cloud Datastore API.
+
+```bash
+gcloud app deploy --project=<AppID> app.yaml
+gcloud services enable datastore.googleapis.com --project=<AppID>
+```
+
+### Deployment setup
+
+NOTE: Before you can deploy, you'll need to authenticate. Refer to the
+[authentication documentation](#uploader-authentication) if you have not used
+Google Cloud Platform services before and need information on authentication.
+
+1. Create a `fileset.yaml` configuration file.
+
+```yaml
+site: siteId  # Specify a site ID. If blank, `default` will be used.
+schedule:
+  default: master  # Specify a branch for the prod deployment.
+```
+
+2. Generate your files.
+
+Use a static site generator or just manually create a directory containing files
+to upload. In the below example, the files in the directory `build` are
+uploaded.
+
+4. Upload your files.
 
 ```bash
-gcloud app deploy app.yaml
+fileset upload ./build
 ```
 
-### Authentication for deployment
+NOTE: The uploader will look for `fileset.yaml` within the `./build` directory
+first. If it's not found, it will look up in the parent folder for a
+`fileset.yaml` file. If the config file doesn't exist in the `./build` or parent
+folder, the uploader will abort.
 
-Before you are able to deploy your files, you'll need to set up authentication
-to deploy.
+5. That's it! Files have been uploaded to Google Cloud Storage and the uploaded
+   directory is now being served by the application server.
 
-1. Identify the service account to use.
+TODO: Document Identity-Aware Proxy setup and CLI authentication.
+
+## Uploader authentication
+
+You'll need to be authenticated to deploy files and upload the serving manifests.
+
+### Local testing / user account authentication
+
+If you are testing locally, your user account can be used to authenticate to
+Cloud Datastore and Cloud Storage. Simply run the below command to create
+credentials used for authentication:
+
+```bash
+gcloud auth application-default login
+```
+
+### Continuous deployment / service account authentication
+
+If you are using a service account for deployment, you'll need to ensure it has
+the right permissions.
+
+#### 1. Identify the service account to use.
 
 Authentication to upload your files is done using a service account. You'll
 generally want to use one of two service accounts:
 
-  a. When the command is invoked from Google Cloud Build, your project's Cloud
-  Build service account (`<ProjectNumber>@cloudbuild.gserviceaccount.com`) is
-  used.
+__Cloud Build service account__: When the command is invoked from Google Cloud
+Build, your project's Cloud Build service account
+(`<ProjectNumber>@cloudbuild.gserviceaccount.com`) is used.
 
 To determine your project's project number:
 
 ```bash
 gcloud projects describe <AppID>
 ```
 
-  b. When the command is invoked locally (i.e. for testing or for manual uploads),
-  you'll likely want to use your App Engine app's default service account
-  (`<AppID>@appspot.gserviceaccount.com`). You can download a service account key
-  by running:
+__Application default service account__: When the command is invoked locally
+(i.e. for testing or for manual uploads), you'll likely want to use your App
+Engine app's default service account (`<AppID>@appspot.gserviceaccount.com`).
+You can download a service account key by running:
 
 ```bash
 gcloud iam service-accounts keys create \
@@ -91,7 +154,7 @@ NOTE: This will download a `key.json` to your current directory. Avoid
 committing this to your Git repository. You'll want to add `key.json` to
 `.gitignore`.
 
-2. Ensure service account has permissions.
+#### 2. Ensure service account has permissions.
 
 The following permissions are needed:
 
@@ -117,34 +180,6 @@ for role in datastore.owner storage.objectAdmin g; do \
       --role=roles/$role \
 ; done
 ```
-
-### Deployment setup
-
-1. Create a `fileset.yaml` configuration file.
-
-```yaml
-site: siteId  # Specify a site ID. If blank, `default` will be used.
-schedule:
-  default: master  # Specify a branch for the prod deployment.
-```
-
-2. Generate your files.
-
-Use a static site generator or just manually create a directory containing files
-to upload. In the below example, the files in the directory `build` are
-uploaded.
-
-4. Upload your files.
-
-```bash
-fileset upload -s siteId build
-```
-
-5. That's it! Files have been uploaded to Google Cloud Storage and the uploaded
-   directory is now being served by the application server.
-
-TODO: Document Identity-Aware Proxy setup and CLI authentication.
-
 ## Environments
 
 Fileset uses Git branches to determine whether files should be in production
@@ -175,3 +210,14 @@ Staging URL: https://default-f3a9abb-dot-fileset-dot-appid.appspot.com
 ...
 Staging URL: https://default-4fb48ce-dot-fileset-dot-appid.appspot.com
 ```
+
+## Testing
+
+You can verify Fileset server is working as you expect by looking for the following headers:
+
+| Header | Description |
+|-|-|
+| `x-fileset-site` | The site being served. Usually this will be `default` but for multi-site installations, this will be useful for determining which site is serving. |
+| `x-fileset-ref` | The Git commit sha that corresponds to the serving manifest that is handling your request. |
+| `x-fileset-blob` | The blob directory key corresponding to the file being served. This is the SHA-1 hash of the file's content. |
+| `x-fileset-ttl` | For scheduled deployments, the value of this header will correspond to the timestamp for the timed deployment being served. |

app.yaml

@@ -1,4 +1,4 @@
 service: fileset2
 runtime: nodejs10
 instance_class: F4
-entrypoint: node ./dist/src/index.js serve
+entrypoint: fileset serve

src/commands/upload.ts

@@ -43,6 +43,8 @@ export class UploadCommand {
     const gitData = await getGitData(path);
     const config = findConfig(path);
     const ttl = this.options.ttl ? new Date(this.options.ttl) : undefined;
+    const site = this.options.site || config.site;
+
     let bucket = this.options.bucket;
     if (!bucket && config.google_cloud_project) {
       bucket = `${config.google_cloud_project}.appspot.com`;
@@ -54,7 +56,13 @@ export class UploadCommand {
         'Unable to determine which Google Cloud Storage bucket to use. You must specify a `google_cloud_project` in `fileset.yaml` or specify a `GOOGLE_CLOUD_PROJECT` environment variable.'
       );
     }
-    const site = this.options.site || config.site;
+
+    const googleCloudProject =
+      config.google_cloud_project || process.env.GOOGLE_CLOUD_PROJECT;
+    if (!googleCloudProject) {
+      throw new Error('Unable to determine the Google Cloud project.');
+    }
+
     const manifest = new Manifest(
       site,
       this.options.ref || gitData.ref,
@@ -65,6 +73,12 @@ export class UploadCommand {
       console.log(`No files found in -> ${path}`);
       return;
     }
-    upload.uploadManifest(bucket, manifest, this.options.force, ttl);
+    upload.uploadManifest(
+      googleCloudProject,
+      bucket,
+      manifest,
+      this.options.force,
+      ttl
+    );
   }
 }

src/upload.ts

@@ -8,8 +8,8 @@ import {asyncify, mapLimit} from 'async';
 import {Datastore} from '@google-cloud/datastore';
 import {Storage} from '@google-cloud/storage';
 import {entity} from '@google-cloud/datastore/build/src/entity';
+import {google} from '@google-cloud/datastore/build/protos/protos';
 
-const datastore = new Datastore();
 const NUM_CONCURRENT_UPLOADS = 64;
 
 function getBlobPath(siteId: string, hash: string) {
@@ -51,13 +51,16 @@ function createProgressBar() {
 }
 
 export async function uploadManifest(
+  googleCloudProject: string,
   bucket: string,
   manifest: Manifest,
   force?: boolean,
   ttl?: Date
 ) {
   console.log(`Using storage: ${bucket}/${getBlobPath(manifest.site, '')}`);
-  const storageBucket = new Storage().bucket(bucket);
+  const storageBucket = new Storage({
+    projectId: googleCloudProject,
+  }).bucket(bucket);
   const bar = createProgressBar();
 
   // Check for files that have already been uploaded. Files already uploaded do
@@ -72,7 +75,7 @@ export async function uploadManifest(
   );
 
   if (numTotalFiles <= 0) {
-    finalize(manifest, ttl);
+    finalize(googleCloudProject, manifest, ttl);
   } else {
     let bytesTransferred = 0;
     let numProcessedFiles = 0;
@@ -82,7 +85,7 @@ export async function uploadManifest(
     });
     // @ts-ignore
     bar.on('stop', () => {
-      finalize(manifest, ttl);
+      finalize(googleCloudProject, manifest, ttl);
     });
 
     mapLimit(
@@ -130,7 +133,11 @@ export async function uploadManifest(
   }
 }
 
-async function saveManifestEntity(key: entity.Key, data: any) {
+async function saveManifestEntity(
+  datastore: Datastore,
+  key: entity.Key,
+  data: any
+) {
   const ent = {
     key: key,
     excludeFromIndexes: ['paths'],
@@ -139,7 +146,15 @@ async function saveManifestEntity(key: entity.Key, data: any) {
   await datastore.save(ent);
 }
 
-async function finalize(manifest: Manifest, ttl?: Date) {
+async function finalize(
+  googleCloudProject: string,
+  manifest: Manifest,
+  ttl?: Date
+) {
+  const datastore = new Datastore({
+    projectId: googleCloudProject,
+    // keyFilename: '/path/to/keyfile.json',
+  });
   const manifestPaths = manifest.toJSON();
   const now = new Date();
 
@@ -148,7 +163,7 @@ async function finalize(manifest: Manifest, ttl?: Date) {
     'Fileset2Manifest',
     `${manifest.site}:ref:${manifest.shortSha}`,
   ]);
-  await saveManifestEntity(key, {
+  await saveManifestEntity(datastore, key, {
     site: manifest.site,
     ref: manifest.ref,
     branch: manifest.branch,
@@ -162,7 +177,7 @@ async function finalize(manifest: Manifest, ttl?: Date) {
       'Fileset2Manifest',
       `${manifest.site}:branch:${manifest.branch}`,
     ]);
-    await saveManifestEntity(branchKey, {
+    await saveManifestEntity(datastore, branchKey, {
       site: manifest.site,
       ref: manifest.ref,
       branch: manifest.branch,
@@ -181,6 +196,6 @@ async function finalize(manifest: Manifest, ttl?: Date) {
   );
   // TODO: Allow customizing the staging URL using `fileset.yaml` configuration.
   console.log(
-    `Staged: https://${manifest.site}-${manifest.shortSha}-dot-fileset2-dot-${process.env.GOOGLE_CLOUD_PROJECT}.appspot.com`
+    `Staged: https://${manifest.site}-${manifest.shortSha}-dot-fileset2-dot-${googleCloudProject}.appspot.com`
   );
 }