Inject content into Docker via volumes, not bind mounts

This is useful, because a) bind mounts are slow and problematic (they
come with various warnings) on Docker Desktop on Windows & Mac) and b)
this moves us slowly towards a world where we can successfully intercept
a _remote_ Docker host (though we still can't yet, mostly because we
don't know the right proxy address to use for HTTP Toolkit).

Note that unlike other Docker services, we ensure the Docker volume is
running synchronously on-demand, not just async when we see Docker
proxy activity.

Without that, if the volume didn't exist for some reason (shouldn't
happen, but possible given races etc or deletion elsewhere) then
containers could've been created referencing it when it didn't exist,
whereupon Docker would create it automatically and break everything.

Author: pimterry

src/api-server.ts

@@ -9,21 +9,19 @@ import { GraphQLScalarType } from 'graphql';
 import { graphqlHTTP } from 'express-graphql';
 import gql from 'graphql-tag';
 
-import { generateSPKIFingerprint, MockttpAdminServer } from 'mockttp';
+import { generateSPKIFingerprint } from 'mockttp';
 import { getSystemProxy } from 'os-proxy-config';
 
 import { HtkConfig } from './config';
 import { reportError, addBreadcrumb } from './error-tracking';
 import { buildInterceptors, Interceptor, ActivationError } from './interceptors';
-import { ALLOWED_ORIGINS } from './constants';
+import { ALLOWED_ORIGINS, SERVER_VERSION } from './constants';
 import { delay } from './util/promise';
 import { getDnsServer } from './dns-server';
 import { shutdown } from './shutdown';
 
 const ENABLE_PLAYGROUND = false;
 
-const packageJson = require('../package.json');
-
 /**
  * This file contains the core server API, used by the UI to query
  * machine state that isn't easily visible from the web (cert files,
@@ -119,7 +117,7 @@ const buildResolvers = (
 ) => {
     return {
         Query: {
-            version: () => packageJson.version,
+            version: () => SERVER_VERSION,
             interceptors: () => _.values(interceptors),
             interceptor: (_: any, { id } : { id: string }) => interceptors[id],
             config: () => ({

src/constants.ts

@@ -35,4 +35,6 @@ export const MOCKTTP_ALLOWED_ORIGINS = [
 // certificate check server and Chrome's "hide warning" server. These ports are extra likely
 // not to conflict with normal user usage, and are specifically designated by the IANA for
 // use for dynamic ports.
-export const EPHEMERAL_PORT_RANGE = { startPort: 49152, endPort: 65535 } as const;
+export const EPHEMERAL_PORT_RANGE = { startPort: 49152, endPort: 65535 } as const;
+
+export const SERVER_VERSION = require('../package.json').version as string;

src/interceptors/docker/docker-build-injection.ts

@@ -5,15 +5,12 @@ import * as EventStream from 'event-stream';
 import * as getRawBody from 'raw-body';
 import maybeGunzip = require('gunzip-maybe');
 import * as tarStream from 'tar-stream';
-import * as tarFs from 'tar-fs';
 import { parse as parseDockerfile, CommandEntry } from 'docker-file-parser';
 
-import {
-    getTerminalEnvVars,
-    OVERRIDES_DIR
-} from '../terminal/terminal-env-overrides';
+import { getTerminalEnvVars } from '../terminal/terminal-env-overrides';
 import { getDeferred } from '../../util/promise';
 import { getDockerHostAddress } from './docker-commands';
+import { packOverrideFiles } from './docker-data-injection';
 
 const HTTP_TOOLKIT_INJECTED_PATH = '/http-toolkit-injections';
 const HTTP_TOOLKIT_INJECTED_OVERRIDES_PATH = path.posix.join(HTTP_TOOLKIT_INJECTED_PATH, 'overrides');
@@ -74,7 +71,7 @@ export function injectIntoBuildStream(
     });
     extractionStream.on('finish', async () => {
         repackStream.entry({ name: HTTP_TOOLKIT_CONTEXT_CA_PATH }, config.certContent);
-        await packOverrideFiles(repackStream);
+        await packOverrideFiles(repackStream, HTTP_TOOLKIT_CONTEXT_OVERRIDES_PATH);
         repackStream.finalize();
     });
 
@@ -91,28 +88,6 @@ export function injectIntoBuildStream(
     };
 }
 
-function packOverrideFiles(existingPackStream: tarStream.Pack) {
-    return new Promise<tarStream.Pack>((resolve) => {
-        tarFs.pack(OVERRIDES_DIR, {
-            pack: existingPackStream,
-            map: (fileHeader) => {
-                fileHeader.name = path.posix.join(HTTP_TOOLKIT_CONTEXT_OVERRIDES_PATH, fileHeader.name);
-
-                // Owned by root by default
-                fileHeader.uid = 0;
-                fileHeader.gid = 0;
-
-                // But ensure everything is globally readable & runnable
-                fileHeader.mode = parseInt('555', 8);
-
-                return fileHeader;
-            },
-            finalize: false,
-            finish: resolve
-        });
-    });
-}
-
 // Simplify to just the params we care about
 type DockerCommand = Pick<CommandEntry, 'name' | 'args'> & { raw?: string };
 

src/interceptors/docker/docker-commands.ts

@@ -1,13 +1,10 @@
 import * as _ from 'lodash';
 import * as Docker from 'dockerode';
 import * as path from 'path';
-import * as semver from 'semver';
 
-import {
-    getTerminalEnvVars,
-    OVERRIDES_DIR
-} from '../terminal/terminal-env-overrides';
+import { getTerminalEnvVars } from '../terminal/terminal-env-overrides';
 import { transformComposeCreationLabels } from './docker-compose';
+import { getDockerDataVolumeName } from './docker-data-injection';
 
 // Used to label intercepted docker containers with the port of the proxy
 // that's currently intercepting them.
@@ -76,15 +73,14 @@ const envObjectToArray = (envObject: { [key: string]: string }): string[] =>
  * the container (to make sure we get *all* env vars, for example) and then
  * combine that with the inter
  */
-export function transformContainerCreationConfig(
+export async function transformContainerCreationConfig(
     containerConfig: Docker.ContainerCreateOptions,
     baseImageConfig: Docker.ImageInspectInfo | undefined,
-    { proxyPort, proxyHost, certPath }: {
+    { proxyPort, certContent }: {
         proxyPort: number,
-        proxyHost: string,
-        certPath: string
+        certContent: string
     }
-): Docker.ContainerCreateOptions {
+): Promise<Docker.ContainerCreateOptions> {
     // Get the container-relevant config from the image config first.
     // The image has both .Config and .ContainerConfig. The former
     // is preferred, seems that .ContainerConfig is backward compat.
@@ -127,16 +123,13 @@ export function transformContainerCreationConfig(
         // files into place on top of the existing content:
         Binds: [
             ...(currentConfig.HostConfig?.Binds ?? []).filter((existingMount) =>
-                // Drop any existing mounts for these folders - this allows re-intercepting containers, e.g.
-                // to switch from one proxy port to another.
-                !existingMount.startsWith(`${certPath}:`) &&
-                !existingMount.startsWith(`${OVERRIDES_DIR}:`)
+                // Drop any existing mounts for these folders - this allows re-intercepting containers,
+                // e.g. to switch from one proxy port to another.
+                !existingMount.endsWith(`:${HTTP_TOOLKIT_INJECTED_PATH}:ro`)
             ),
-            // Bind-mount the CA certificate file individually too:
-            `${certPath}:${HTTP_TOOLKIT_INJECTED_CA_PATH}:ro`,
-            // Bind-mount the overrides directory into the container:
-            `${OVERRIDES_DIR}:${HTTP_TOOLKIT_INJECTED_OVERRIDES_PATH}:ro`
-            // ^ Both 'ro' - untrusted containers must not be able to mess with these!
+            // Bind-mount the injected data volume:
+            `${await getDockerDataVolumeName(certContent)}:${HTTP_TOOLKIT_INJECTED_PATH}:ro`,
+            // ^ Note the 'ro' - untrusted containers must not be able to mess with this!
         ]
     };
 
@@ -195,10 +188,9 @@ async function connectNetworks(
 export async function restartAndInjectContainer(
     docker: Docker,
     containerId: string,
-    { proxyPort, certContent, certPath }: {
+    { proxyPort, certContent }: {
         proxyPort: number,
         certContent: string
-        certPath: string
     }
 ) {
     // We intercept containers by stopping them, cloning them, injecting our settings,
@@ -222,22 +214,16 @@ export async function restartAndInjectContainer(
         }
     });
 
-    const proxyHost = getDockerHostAddress(process.platform, containerDetails);
-
     // First we clone the continer, injecting our custom settings:
     const newContainer = await docker.createContainer(
-        transformContainerCreationConfig(
+        await transformContainerCreationConfig(
             // Get options required to directly recreate this container
             deriveContainerCreationConfigFromInspection(
                 containerDetails
             ),
             // We don't need image config - inspect result has *everything*
             undefined,
-            { // The settings to inject:
-                certPath,
-                proxyPort,
-                proxyHost
-            }
+            { proxyPort, certContent }
         )
     );