perf: cache files with redis to reduce registry load (#3)

* refactor: cleanup dependencies / SSL with @interactivetraining/acme-client

* feat: add directory listing (#2)

* fix(getPackageFileList): move downloadPackage() call outside promise

* feat: add redis to reduce registry load

Author: adamlacombe

.dockerignore

@@ -3,6 +3,6 @@
 .gitignore
 Dockerfile
 README.md
-.env
 node_modules
 npm-debug.log
+build-for-run.sh

.gitignore

@@ -24,3 +24,5 @@ dist/
 *.pem
 acme/
 cache/
+
+build-for-run.sh

Dockerfile

@@ -1,16 +1,13 @@
 FROM node:10
 
-LABEL version="0.0.1"
+LABEL version="0.0.2"
 
 WORKDIR /usr/src/app
 
 COPY . .
 
 RUN npm install
 
-EXPOSE 443
-EXPOSE 80
-
-VOLUME /usr/src/app/cache
+VOLUME /usr/src/app
 
 CMD [ "npm", "run", "start" ]

README.md

@@ -1,94 +1,17 @@
 # self-hosted-unpkg
 
-## Run locally
-Create .env file. (see sample.env) and run:
+## Requirements
+- Registry token or credentials (if you want to allow access to private packages)
+- Redis (if you want short-term file caching to reduce registry load)
 
-```bash
-docker run -it --rm -p 443:443 -p 80:80 -v /local/path/to/cache:/usr/src/app/cache --env-file ./.env --name self-hosted-unpkg interactivetraining/self-hosted-unpkg
-```
+### For HTTPS
+ - Cloudflare credentials (used to verify domain with Lets Encrypt)
+ - Google Cloud Storage + service account (for storing SSL cert/key)
 
-## Kubernetes Deployment
 
-### Create a secret from your env file
-```bash
-kubectl create secret generic __MY_SECRET_NAME__ --from-env-file ./path/to/.env
-```
+## Example
+Create .env file. (see http.sample.env) and run:
 
-### Add each variable to your deployment
-```yaml
-apiVersion: extensions/v1beta1
-kind: Deployment
-spec:
-  template:
-    spec:
-      containers:
-      - env:
-        - name: DOMAIN
-          valueFrom:
-            secretKeyRef:
-              key: DOMAIN
-              name: __MY_SECRET_NAME__
-        - name: NPM_REGISTRY
-          valueFrom:
-            secretKeyRef:
-              key: NPM_REGISTRY
-              name: __MY_SECRET_NAME__
-        - name: NPM_TOKEN
-          valueFrom:
-            secretKeyRef:
-              key: NPM_TOKEN
-              name: __MY_SECRET_NAME__
-        - name: CLOUDFLARE_EMAIL
-          valueFrom:
-            secretKeyRef:
-              key: CLOUDFLARE_EMAIL
-              name: __MY_SECRET_NAME__
-        - name: CLOUDFLARE_API_KEY
-          valueFrom:
-            secretKeyRef:
-              key: CLOUDFLARE_API_KEY
-              name: __MY_SECRET_NAME__
-        - name: LETS_ENCRYPT_EMAIL
-          valueFrom:
-            secretKeyRef:
-              key: LETS_ENCRYPT_EMAIL
-              name: __MY_SECRET_NAME__
-        - name: LETS_ENCRYPT_AGREE_TO_TOS
-          valueFrom:
-            secretKeyRef:
-              key: LETS_ENCRYPT_AGREE_TO_TOS
-              name: __MY_SECRET_NAME__
-        - name: GOOGLE_CLOUD_BUCKET_NAME
-          valueFrom:
-            secretKeyRef:
-              key: GOOGLE_CLOUD_BUCKET_NAME
-              name: __MY_SECRET_NAME__
-        - name: GOOGLE_CLOUD_PROJECT_ID
-          valueFrom:
-            secretKeyRef:
-              key: GOOGLE_CLOUD_PROJECT_ID
-              name: __MY_SECRET_NAME__
-        - name: GOOGLE_CLOUD_CERT_DB_FILE
-          valueFrom:
-            secretKeyRef:
-              key: GOOGLE_CLOUD_CERT_DB_FILE
-              name: __MY_SECRET_NAME__
-        - name: GOOGLE_CLOUD_CLIENT_EMAIL
-          valueFrom:
-            secretKeyRef:
-              key: GOOGLE_CLOUD_CLIENT_EMAIL
-              name: __MY_SECRET_NAME__
-        - name: GOOGLE_CLOUD_PRIVATE_KEY
-          valueFrom:
-            secretKeyRef:
-              key: GOOGLE_CLOUD_PRIVATE_KEY
-              name: __MY_SECRET_NAME__
-        - name: ENABLE_SSL
-          valueFrom:
-            secretKeyRef:
-              key: ENABLE_SSL
-              name: __MY_SECRET_NAME__
-        image: interactivetraining/self-hosted-unpkg:latest
-        imagePullPolicy: Always
-        name: self-hosted-unpkg
-```
+```bash
+docker run -it --rm -p 443:443 -p 80:80 --env-file ./.env --name self-hosted-unpkg interactivetraining/self-hosted-unpkg
+```

build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+while [[ "$#" -gt 0 ]]; do case $1 in
+  -v|--version) version="$2"; shift;;
+  *) echo "Unknown parameter passed: $1"; exit 1;;
+esac; shift; done
+
+npm run build
+docker build -t "interactivetraining/self-hosted-unpkg:$version" .
+docker push "interactivetraining/self-hosted-unpkg:$version"

http.sample.env

@@ -0,0 +1,11 @@
+ENABLE_SSL=0
+NPM_REGISTRY=https://registry.npmjs.com
+NPM_USER=
+NPM_PASSWORD=
+NPM_TOKEN=
+PORT=80
+REDIS_HOST=
+REDIS_PORT=6379
+REDIS_PASSWORD=
+CACHE_PREFIX=unpkg
+LOG_REQUEST=true

sample.env renamed to https.sample.env

@@ -1,6 +1,6 @@
 DOMAIN=my-self-hosted-unpkg-domain.com
-ENABLE_SSL=0
-NPM_REGISTRY=https://private.npm.registry
+ENABLE_SSL=1
+NPM_REGISTRY=https://registry.npmjs.com
 NPM_USER=
 NPM_PASSWORD=
 NPM_TOKEN=
@@ -10,6 +10,11 @@ LETS_ENCRYPT_EMAIL=
 LETS_ENCRYPT_AGREE_TO_TOS=true
 GOOGLE_CLOUD_BUCKET_NAME=my-certs
 GOOGLE_CLOUD_PROJECT_ID=my-google-cloud-project-id
-GOOGLE_CLOUD_CERT_DB_FILE=cert-db.json
 GOOGLE_CLOUD_CLIENT_EMAIL=[email protected]
 GOOGLE_CLOUD_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nEXAMPLE_PRIVATE_KEY\n-----END PRIVATE KEY-----\n"
+PORT=80
+REDIS_HOST=
+REDIS_PORT=6379
+REDIS_PASSWORD=
+CACHE_PREFIX=unpkg
+LOG_REQUEST=true

package.json

@@ -12,34 +12,31 @@
     "build": "tsc"
   },
   "dependencies": {
-    "@interactivetraining/le-challenge-cloudflare": "1.3.1",
-    "@interactivetraining/le-store-gcloud-storage": "0.0.2",
+    "@interactivetraining/acme-client": "0.0.6",
     "compression": "^1.7.3",
     "cors": "^2.8.5",
     "dotenv": "^5.0.1",
-    "download-npm-package": "^3.1.12",
-    "download-package-tarball": "^1.0.7",
     "express": "^4.16.4",
-    "get-package-json-from-registry": "^2.2.1",
-    "got": "^9.6.0",
-    "greenlock-express": "^2.6.8",
+    "gunzip-maybe": "^1.4.1",
     "helmet": "^3.16.0",
-    "le-challenge-cloudflare": "^1.0.1",
     "mime": "^2.4.0",
+    "mime-types": "^2.1.24",
     "redirect-https": "^1.3.0",
-    "zlib": "^1.0.5"
+    "tar-stream": "^2.1.0",
+    "zlib": "^1.0.5",
+    "ioredis": "^4.14.0"
   },
   "devDependencies": {
     "@types/compression": "0.0.36",
     "@types/cors": "^2.8.4",
     "@types/dotenv": "^6.1.0",
     "@types/express": "^4.16.1",
     "@types/helmet": "0.0.43",
-    "@types/node": "^11.11.1",
-    "@types/shelljs": "^0.8.3",
+    "@types/node": "^12.7.2",
+    "@types/tar-stream": "^1.6.1",
     "nodemon": "^1.18.10",
-    "shelljs": "^0.8.3",
     "ts-node": "^8.0.2",
-    "typescript": "^3.3.3333"
+    "typescript": "^3.5.3",
+    "@types/ioredis": "^4.0.15"
   }
 }

src/cache.ts

@@ -0,0 +1,28 @@
+import * as Redis from 'ioredis';
+import {IPackageParams} from './interfaces';
+
+const redis = (process.env.REDIS_PORT && process.env.REDIS_HOST) ? new Redis({
+  port: parseInt(process.env.REDIS_PORT),
+  host: process.env.REDIS_HOST,
+  password: process.env.REDIS_PASSWORD
+}) : undefined;
+
+export class Cache {
+  static get(key: string): Promise<string | null> {
+    return new Promise((resolve, reject) => {
+      if (!redis) resolve(null);
+      redis.get(key, (err, result) => {
+        (err) ? reject(err) : resolve(result);
+      });
+    })
+  }
+  
+  static set(key: string, value: any, expireSeconds: number = 0): Promise<string> {
+    if (!redis) return;
+    return redis.set(key, value, 'EX', expireSeconds);
+  }
+  
+  static buildKey(pkg: IPackageParams): string {
+    return `${process.env.CACHE_PREFIX}/cache/${(pkg.scope) ? `${pkg.scope}/` : ''}${pkg.package}/${pkg.version}/${pkg['0']}`;
+  }
+}