Store pass through upstream request data in the model

Author: pimterry

src/model/events/categorization.ts

@@ -7,7 +7,7 @@ import { getHeaderValue } from '../../util/headers';
 import { getBaseContentType } from './content-types';
 
 import { HTKEventBase } from './event-base';
-import { HttpExchange, SuccessfulExchange } from '../http/exchange';
+import { ViewableHttpExchange, SuccessfulExchange } from '../http/exchange';
 
 export const getMessageBaseAcceptTypes = (message: ExchangeMessage) =>
     (getHeaderValue(message.headers, 'accept')?.split(',') || [])
@@ -16,7 +16,7 @@ export const getMessageBaseAcceptTypes = (message: ExchangeMessage) =>
 export const getMessageBaseContentType = (message: ExchangeMessage) =>
     getBaseContentType(getHeaderValue(message.headers, 'content-type'));
 
-const isMutatativeExchange = (exchange: HttpExchange) => _.includes([
+const isMutatativeExchange = (exchange: ViewableHttpExchange) => _.includes([
     'POST',
     'PATCH',
     'PUT',
@@ -182,7 +182,9 @@ const highlights = {
     pink: '#dd3a96'
 };
 
-export function getSummaryColor(exchangeOrCategory: HttpExchange | EventCategory): string {
+export function getSummaryColor(
+    exchangeOrCategory: ViewableHttpExchange | EventCategory
+): string {
     const category = typeof exchangeOrCategory === 'string' ?
         exchangeOrCategory : exchangeOrCategory.category;
 

src/model/events/event-base.ts

@@ -3,7 +3,7 @@ import { observable, computed } from 'mobx';
 import {
     FailedTlsConnection,
     TlsTunnel,
-    HttpExchange,
+    ViewableHttpExchange,
     RTCConnection,
     RTCDataChannel,
     RTCMediaTrack,
@@ -17,7 +17,7 @@ export abstract class HTKEventBase {
     abstract get id(): string;
 
     // These can be overriden by subclasses to allow easy type narrowing:
-    isHttp(): this is HttpExchange { return false; }
+    isHttp(): this is ViewableHttpExchange { return false; }
     isWebSocket(): this is WebSocketStream { return false; }
 
     isTlsFailure(): this is FailedTlsConnection { return false; }
@@ -33,10 +33,14 @@ export abstract class HTKEventBase {
     }
 
     @observable
-    public searchIndex: string = '';
+    private _searchIndex: string = '';
+    public get searchIndex(): string { return this._searchIndex; }
+    public set searchIndex(value: string) { this._searchIndex = value; }
 
     @observable
-    public pinned: boolean = false;
+    private _pinned: boolean = false;
+    public get pinned(): boolean { return this._pinned; }
+    public set pinned(value: boolean) { this._pinned = value; }
 
     // Logic elsewhere can put values into these caches to cache calculations
     // about this event weakly, so they GC with the event.

src/model/events/events-store.ts

@@ -28,6 +28,7 @@ import {
     InputRTCMediaStats,
     InputRTCMediaTrackClosed,
     InputRTCExternalPeerAttached,
+    InputRuleEvent,
     TimingEvents
 } from '../../types';
 
@@ -68,6 +69,7 @@ type EventTypesMap = {
     'tls-passthrough-opened': InputTlsPassthrough,
     'tls-passthrough-closed': InputTlsPassthrough,
     'client-error': InputClientError,
+    'rule-event': InputRuleEvent
 } & {
     // MockRTC:
     [K in InputRTCEvent]: InputRTCEventData[K];
@@ -86,7 +88,8 @@ const mockttpEventTypes = [
     'tls-client-error',
     'tls-passthrough-opened',
     'tls-passthrough-closed',
-    'client-error'
+    'client-error',
+    'rule-event'
 ] as const;
 
 const mockRTCEventTypes = [
@@ -260,6 +263,9 @@ export class EventsStore {
                 case 'client-error':
                     return this.addClientError(queuedEvent.event);
 
+                case 'rule-event':
+                    return this.addRuleEvent(queuedEvent.event);
+
                 case 'peer-connected':
                     return this.addRTCPeerConnection(queuedEvent.event);
                 case 'external-peer-attached':
@@ -494,6 +500,31 @@ export class EventsStore {
         this.events.push(exchange);
     }
 
+    @action
+    private addRuleEvent(event: InputRuleEvent) {
+        const exchange = _.find(this.exchanges, { id: event.requestId });
+
+        if (!exchange) {
+            // Handle this later, once the request has arrived
+            this.orphanedEvents[event.requestId] = { type: 'rule-event', event };
+            return;
+        };
+
+        switch (event.eventType) {
+            case 'passthrough-request-head':
+                exchange.updateFromUpstreamRequestHead(event.eventData);
+                break;
+            case 'passthrough-request-body':
+                exchange.updateFromUpstreamRequestBody(event.eventData);
+                break;
+            // Only request events handled for now
+            case 'passthrough-response-head':
+            case 'passthrough-response-body':
+            case 'passthrough-abort':
+                break;
+        }
+    }
+
     @action
     private addRTCPeerConnection(event: InputRTCPeerConnected) {
         this.events.push(new RTCConnection(event));

src/model/http/exchange.ts

@@ -15,6 +15,8 @@ import {
     MockttpBreakpointedResponse,
     InputCompletedRequest,
     MockttpBreakpointResponseResult,
+    InputRuleEventDataMap,
+    RawHeaders
 } from "../../types";
 import {
     fakeBuffer,
@@ -25,8 +27,10 @@ import { UnreachableCheck } from '../../util/error';
 import { lazyObservablePromise, ObservablePromise, observablePromise } from "../../util/observable";
 import {
     asHeaderArray,
-    getHeaderValue
+    getHeaderValue,
+    getHeaderValues
 } from '../../util/headers';
+import { ParsedUrl } from '../../util/url';
 
 import { logError } from '../../errors';
 
@@ -50,8 +54,14 @@ import {
     getResponseBreakpoint,
     getDummyResponseBreakpoint
 } from './exchange-breakpoint';
+import { UpstreamHttpExchange } from './upstream-exchange';
 
-function tryParseUrl(request: InputRequest): (URL & { parseable: true }) | undefined  {
+export type HttpVersion = 2 | 1;
+export function parseHttpVersion(version: string | undefined): HttpVersion {
+    return version === '2.0' ? 2 : 1;
+}
+
+function tryParseUrl(request: InputRequest): ParsedUrl | undefined  {
     try {
         return Object.assign(
             new URL(request.url, `${request.protocol}://${request.hostname || 'unknown.invalid'}`),
@@ -64,7 +74,7 @@ function tryParseUrl(request: InputRequest): (URL & { parseable: true }) | undef
     }
 }
 
-function getFallbackUrl(request: InputRequest): URL & { parseable: false } {
+function getFallbackUrl(request: InputRequest): ParsedUrl {
     try {
         return Object.assign(
             new URL("/[unparseable]", `${request.protocol}://${request.hostname || 'unknown.invalid'}`),
@@ -106,8 +116,8 @@ function addResponseMetadata(response: InputResponse): HtkResponse {
 export class HttpBody implements MessageBody {
 
     constructor(
-        message: InputMessage,
-        headers: Headers
+        message: InputMessage | { body: Uint8Array },
+        headers: Headers | RawHeaders
     ) {
         if (!('body' in message) || !message.body) {
             this._encoded = stringToBuffer("");
@@ -118,7 +128,7 @@ export class HttpBody implements MessageBody {
             this._decoded = message.body.decoded;
         }
 
-        this._contentEncoding = asHeaderArray(headers['content-encoding']);
+        this._contentEncoding = asHeaderArray(getHeaderValues(headers, 'content-encoding'));
     }
 
     private _contentEncoding: string[];
@@ -179,17 +189,61 @@ export class HttpBody implements MessageBody {
     }
 }
 
-export type CompletedRequest = Omit<HttpExchange, 'request'> & {
+export type CompletedRequest = Omit<ViewableHttpExchange, 'request'> & {
     matchedRule: { id: string, handlerRype: HandlerClassKey } | false
 };
-export type CompletedExchange = Omit<HttpExchange, 'response'> & {
+export type CompletedExchange = Omit<ViewableHttpExchange, 'response'> & {
     response: HtkResponse | 'aborted'
 };
-export type SuccessfulExchange = Omit<HttpExchange, 'response'> & {
+export type SuccessfulExchange = Omit<ViewableHttpExchange, 'response'> & {
     response: HtkResponse
 };
 
-export class HttpExchange extends HTKEventBase {
+/**
+ * HttpExchanges actually come in two types: downstream (client input to Mockttp)
+ * and upstream (Mockttp extra data for what we really forwarded - e.g. after transform).
+ * The events actually stored in the event store's list are always downstream
+ * exchanges, but in many cases elsewhere either can be provided.
+ *
+ * Both define the exact same interface, and various higher-layer components
+ * may switch which is passed to the final view components depending on user
+ * configuration. Any code that just reads exchange data (i.e. which doesn't
+ * update it from events, create/import exchanges, handle breakpoints, etc)
+ * should generally use the ViewableHttpExchange readonly interface where possible.
+ */
+export interface ViewableHttpExchange extends HTKEventBase {
+
+    get downstream(): HttpExchange;
+    /**
+     * Upstream is set if forwarded, but otherwise undefined
+     */
+    get upstream(): UpstreamHttpExchange | undefined;
+
+    get request(): HtkRequest;
+    get response(): HtkResponse | 'aborted' | undefined;
+    get abortMessage(): string | undefined;
+    get api(): ApiExchange | undefined;
+
+    get httpVersion(): 1 | 2;
+    get matchedRule(): { id: string, handlerStepTypes: HandlerClassKey[] } | false | undefined;
+    get tags(): string[];
+    get timingEvents(): TimingEvents;
+
+    isHttp(): this is ViewableHttpExchange;
+    isCompletedRequest(): this is CompletedRequest;
+    isCompletedExchange(): this is CompletedExchange;
+    isSuccessfulExchange(): this is SuccessfulExchange;
+    hasRequestBody(): this is CompletedRequest;
+    hasResponseBody(): this is SuccessfulExchange;
+
+    get requestBreakpoint(): RequestBreakpoint | undefined;
+    get responseBreakpoint(): ResponseBreakpoint | undefined;
+
+    hideErrors: boolean;
+
+}
+
+export class HttpExchange extends HTKEventBase implements ViewableHttpExchange {
 
     constructor(apiStore: ApiStore, request: InputRequest) {
         super();
@@ -217,9 +271,13 @@ export class HttpExchange extends HTKEventBase {
         this._apiMetadataPromise = apiStore.getApi(this.request);
     }
 
-    public readonly request: HtkRequest;
     public readonly id: string;
 
+    public readonly request: HtkRequest;
+
+    public readonly downstream = this;
+    public upstream: UpstreamHttpExchange | undefined;
+
     @observable
     // Undefined initially, defined for completed requests, false for 'not available'
     public matchedRule: { id: string, handlerStepTypes: HandlerClassKey[] } | false | undefined;
@@ -232,7 +290,7 @@ export class HttpExchange extends HTKEventBase {
 
     @computed
     get httpVersion() {
-        return this.request.httpVersion === '2.0' ? 2 : 1;
+        return parseHttpVersion(this.request.httpVersion);
     }
 
     isHttp(): this is HttpExchange {
@@ -261,7 +319,7 @@ export class HttpExchange extends HTKEventBase {
     }
 
     @observable
-    public readonly timingEvents: Partial<TimingEvents>; // May be {} if using an old server (<0.1.7)
+    public readonly timingEvents: TimingEvents;
 
     @observable.ref
     public response: HtkResponse | 'aborted' | undefined;
@@ -295,6 +353,20 @@ export class HttpExchange extends HTKEventBase {
         this.tags = _.union(this.tags, request.tags);
     }
 
+    updateFromUpstreamRequestHead(head: InputRuleEventDataMap['passthrough-request-head']) {
+        if (!this.upstream) {
+            this.upstream = new UpstreamHttpExchange(this);
+        }
+        this.upstream.updateWithRequestHead(head);
+    }
+
+    updateFromUpstreamRequestBody(body: InputRuleEventDataMap['passthrough-request-body']) {
+        if (!this.upstream) {
+            this.upstream = new UpstreamHttpExchange(this);
+        }
+        this.upstream.updateWithRequestBody(body);
+    }
+
     markAborted(request: InputFailedRequest) {
         this.response = 'aborted';
         this.searchIndex += '\naborted';
@@ -357,6 +429,10 @@ export class HttpExchange extends HTKEventBase {
             this.response.cache.clear();
             this.response.body.cleanup();
         }
+
+        if (this.upstream) {
+            this.upstream.cleanup();
+        }
     }
 
     // API metadata:
@@ -366,7 +442,7 @@ export class HttpExchange extends HTKEventBase {
 
     // Parsed API info for this specific request, loaded & parsed lazily, only if it's used
     @observable.ref
-    private _apiPromise = lazyObservablePromise(async () => {
+    private _apiPromise = lazyObservablePromise(async (): Promise<ApiExchange | undefined> => {
         const apiMetadata = await this._apiMetadataPromise;
 
         if (apiMetadata) {