Merge pull request #135 from emaheuxPEREN/grpc_compressed_payloads

Feature: support gRPC compressed payloads

Author: pimterry

src/components/editor/content-viewer.tsx

@@ -5,10 +5,12 @@ import { observer } from 'mobx-react';
 import { SchemaObject } from 'openapi3-ts';
 import * as portals from 'react-reverse-portal';
 
+import { Headers } from '../../types';
 import { styled } from '../../styles';
 import { ObservablePromise, isObservablePromise } from '../../util/observable';
 import { asError, unreachableCheck } from '../../util/error';
 import { stringToBuffer } from '../../util/buffer';
+import { lastHeader } from '../../util/headers';
 
 import { ViewableContentType } from '../../model/events/content-types';
 import { Formatters, isEditorFormatter } from '../../model/events/body-formatting';
@@ -22,7 +24,7 @@ interface ContentViewerProps {
     children: Buffer | string;
     schema?: SchemaObject;
     expanded: boolean;
-    rawContentType?: string;
+    headers?: Headers;
     contentType: ViewableContentType;
     editorNode: portals.HtmlPortalNode<typeof SelfSizedEditor | typeof ContainerSizedEditor>;
     cache: Map<Symbol, unknown>;
@@ -199,7 +201,7 @@ export class ContentViewer extends React.Component<ContentViewerProps> {
             return <FormatterContainer expanded={this.props.expanded}>
                 <formatterConfig.Component
                     content={this.contentBuffer}
-                    rawContentType={this.props.rawContentType}
+                    rawContentType={lastHeader(this.props.headers?.['content-type'])}
                 />
             </FormatterContainer>;
         }

src/components/editor/monaco.ts

@@ -62,7 +62,7 @@ async function loadMonacoEditor(retries = 5): Promise<void> {
                             id: 'protobuf-decoding-header',
                             command: {
                                 id: '', // No actual command defined here
-                                title: "Automatically decoded from raw Protobuf data",
+                                title: "Automatically decoded from raw Protobuf/gRPC data",
                             },
                         },
                     ],

src/components/send/sent-response-body.tsx

@@ -81,7 +81,8 @@ export class SentResponseBodyCard extends React.Component<ExpandableCardProps &
             ? getCompatibleTypes(
                 message.contentType,
                 lastHeader(message.headers['content-type']),
-                message.body
+                message.body,
+                message.headers,
             )
             : ['text'] as const;
 
@@ -121,7 +122,7 @@ export class SentResponseBodyCard extends React.Component<ExpandableCardProps &
                     <ContentViewer
                         contentId={message.id}
                         editorNode={this.props.editorNode}
-                        rawContentType={lastHeader(message.headers['content-type'])}
+                        headers={message.headers}
                         contentType={decodedContentType}
                         expanded={!!expanded}
                         cache={message.cache}

src/components/view/http/http-body-card.tsx

@@ -82,7 +82,8 @@ export class HttpBodyCard extends React.Component<ExpandableCardProps & {
         const compatibleContentTypes = getCompatibleTypes(
             message.contentType,
             lastHeader(message.headers['content-type']),
-            message.body
+            message.body,
+            message.headers,
         );
         const decodedContentType = compatibleContentTypes.includes(this.selectedContentType!)
             ? this.selectedContentType!
@@ -121,7 +122,7 @@ export class HttpBodyCard extends React.Component<ExpandableCardProps & {
                     <ContentViewer
                         contentId={`${message.id}-${direction}`}
                         editorNode={this.props.editorNode}
-                        rawContentType={lastHeader(message.headers['content-type'])}
+                        headers={message.headers}
                         contentType={decodedContentType}
                         schema={apiBodySchema}
                         expanded={!!expanded}

src/model/events/body-formatting.ts

@@ -1,3 +1,4 @@
+import { Headers } from '../../types';
 import { styled } from '../../styles';
 
 import { ViewableContentType } from '../events/content-types';
@@ -13,7 +14,7 @@ export interface EditorFormatter {
     language: string;
     cacheKey: Symbol;
     isEditApplicable: boolean; // Can you apply this manually during editing to format an input?
-    render(content: Buffer): string | ObservablePromise<string>;
+    render(content: Buffer, headers?: Headers): string | ObservablePromise<string>;
 }
 
 type FormatComponentProps = {
@@ -35,17 +36,17 @@ export function isEditorFormatter(input: any): input is EditorFormatter {
 }
 
 const buildAsyncRenderer = (formatKey: WorkerFormatterKey) =>
-    (input: Buffer) => observablePromise(
-        formatBufferAsync(input, formatKey)
+    (input: Buffer, headers?: Headers) => observablePromise(
+        formatBufferAsync(input, formatKey, headers)
     );
 
 export const Formatters: { [key in ViewableContentType]: Formatter } = {
     raw: {
         language: 'text',
         cacheKey: Symbol('raw'),
         isEditApplicable: false,
-        render: (input: Buffer) => {
-            if (input.byteLength < 2000) {
+        render: (input: Buffer, headers?: Headers) => {
+            if (input.byteLength < 2_000) {
                 try {
                     // For short-ish inputs, we return synchronously - conveniently this avoids
                     // showing the loading spinner that churns the layout in short content cases.
@@ -55,7 +56,7 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = {
                 }
             } else {
                 return observablePromise(
-                    formatBufferAsync(input, 'raw')
+                    formatBufferAsync(input, 'raw', headers)
                 );
             }
         }
@@ -102,24 +103,26 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = {
         language: 'json',
         cacheKey: Symbol('json'),
         isEditApplicable: true,
-        render: (input: Buffer) => {
-            if (input.byteLength < 10000) {
+        render: (input: Buffer, headers?: Headers) => {
+            if (input.byteLength < 10_000) {
                 const inputAsString = bufferToString(input);
 
                 try {
                     // For short-ish inputs, we return synchronously - conveniently this avoids
                     // showing the loading spinner that churns the layout in short content cases.
                     return JSON.stringify(
                         JSON.parse(inputAsString),
-                    null, 2);
+                        null,
+                        2
+                    );
                     // ^ Same logic as in UI-worker-formatter
                 } catch (e) {
                     // Fallback to showing the raw un-formatted JSON:
                     return inputAsString;
                 }
             } else {
                 return observablePromise(
-                    formatBufferAsync(input, 'json')
+                    formatBufferAsync(input, 'json', headers)
                 );
             }
         }

src/model/events/content-types.ts

@@ -1,8 +1,11 @@
 import * as _ from 'lodash';
-import { MessageBody } from '../../types';
+
+import { Headers, MessageBody } from '../../types';
 import {
     isProbablyProtobuf,
-    isValidProtobuf
+    isValidProtobuf,
+    isProbablyGrpcProto,
+    isValidGrpcProto,
 } from '../../util/protobuf';
 
 // Simplify a mime type as much as we can, without throwing any errors
@@ -21,7 +24,7 @@ export const getBaseContentType = (mimeType: string | undefined) => {
         return type + '/' + combinedSubTypes;
     }
 
-    // Otherwise, wr collect a list of types from most specific to most generic: [svg, xml] for image/svg+xml
+    // Otherwise, we collect a list of types from most specific to most generic: [svg, xml] for image/svg+xml
     // and then look through in order to see if there are any matches here:
     const subTypes = combinedSubTypes.split('+');
     const possibleTypes = subTypes.map(st => type + '/' + st);
@@ -112,6 +115,9 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = {
     'application/x-protobuffer': 'protobuf', // Commonly seen in Google apps
 
     'application/grpc+proto': 'grpc-proto', // Used in GRPC requests (protobuf but with special headers)
+    'application/grpc+protobuf': 'grpc-proto',
+    'application/grpc-proto': 'grpc-proto',
+    'application/grpc-protobuf': 'grpc-proto',
 
     'application/octet-stream': 'raw'
 } as const;
@@ -147,19 +153,32 @@ export function getDefaultMimeType(contentType: ViewableContentType): string {
     return _.findKey(mimeTypeToContentTypeMap, (c) => c === contentType)!;
 }
 
-function isValidBase64Byte(byte: number) {
+function isAlphaNumOrEquals(byte: number) {
     return (byte >= 65 && byte <= 90) || // A-Z
         (byte >= 97 && byte <= 122) ||   // a-z
         (byte >= 48 && byte <= 57) ||    // 0-9
-        byte === 43 ||                   // +
-        byte === 47 ||                   // /
         byte === 61;                     // =
 }
 
+function isValidStandardBase64Byte(byte: number) {
+    // + / (standard)
+    return byte === 43 ||
+        byte === 47 ||
+        isAlphaNumOrEquals(byte);
+}
+
+function isValidURLSafeBase64Byte(byte: number) {
+    // - _ (URL-safe version)
+    return byte === 45 ||
+        byte === 95 ||
+        isAlphaNumOrEquals(byte);
+}
+
 export function getCompatibleTypes(
     contentType: ViewableContentType,
     rawContentType: string | undefined,
-    body: MessageBody | Buffer | undefined
+    body: MessageBody | Buffer | undefined,
+    headers?: Headers,
 ): ViewableContentType[] {
     let types = new Set([contentType]);
 
@@ -180,33 +199,41 @@ export function getCompatibleTypes(
         types.add('xml');
     }
 
-    if (!types.has('grpc-proto') && rawContentType === 'application/grpc') {
-        types.add('grpc-proto')
-    }
-
     if (
         body &&
-        isProbablyProtobuf(body) &&
         !types.has('protobuf') &&
         !types.has('grpc-proto') &&
+        isProbablyProtobuf(body) &&
         // If it's probably unmarked protobuf, and it's a manageable size, try
         // parsing it just to check:
         (body.length < 100_000 && isValidProtobuf(body))
     ) {
         types.add('protobuf');
     }
 
+    if (
+        body &&
+        !types.has('grpc-proto') &&
+        isProbablyGrpcProto(body, headers ?? {}) &&
+        // If it's probably unmarked gRPC, and it's a manageable size, try
+        // parsing it just to check:
+        (body.length < 100_000 && isValidGrpcProto(body, headers ?? {}))
+    ) {
+        types.add('grpc-proto');
+    }
+
     // SVGs can always be shown as XML
     if (rawContentType && rawContentType.startsWith('image/svg')) {
         types.add('xml');
     }
 
     if (
         body &&
-        body.length > 0 &&
-        body.length % 4 === 0 && // Multiple of 4 bytes
-        body.length < 1000 * 100 && // < 100 KB of content
-        body.every(isValidBase64Byte)
+        !types.has('base64') &&
+        body.length >= 8 &&
+        // body.length % 4 === 0 && // Multiple of 4 bytes (final padding may be omitted)
+        body.length < 100_000 && // < 100 KB of content
+        (body.every(isValidStandardBase64Byte) || body.every(isValidURLSafeBase64Byte))
     ) {
         types.add('base64');
     }

src/model/http/har.ts

@@ -51,14 +51,14 @@ interface HarLog extends HarFormat.Log {
 export type RequestContentData = {
     text: string;
     size: number;
-    encoding?: 'base64';
+    encoding: 'base64';
     comment?: string;
 };
 
 export interface ExtendedHarRequest extends HarFormat.Request {
     _requestBodyStatus?:
         | 'discarded:too-large'
-        | 'discarded:not-representable'
+        | 'discarded:not-representable' // to indicate that extended field `_content` is populated with base64 `postData`
         | 'discarded:not-decodable';
     _content?: RequestContentData;
     _trailers?: HarFormat.Header[];
@@ -302,7 +302,7 @@ async function generateHarResponse(
 
     const decoded = await response.body.decodedPromise;
 
-    let responseContent: { text: string, encoding?: string } | { comment: string};
+    let responseContent: { text: string, encoding?: string } | { comment: string };
     try {
         if (!decoded || decoded.byteLength > options.bodySizeLimit) {
             // If no body or the body is too large, don't include it
@@ -751,7 +751,7 @@ function parseHttpVersion(
 }
 
 function parseHarRequestContents(data: RequestContentData): Buffer {
-    if (data.encoding && Buffer.isEncoding(data.encoding)) {
+    if (Buffer.isEncoding(data.encoding)) {
         return Buffer.from(data.text, data.encoding);
     }
 

src/services/ui-worker-api.ts

@@ -21,7 +21,7 @@ import type {
     ParseCertResponse
 } from './ui-worker';
 
-import { Omit } from '../types';
+import { Headers, Omit } from '../types';
 import type { ApiMetadata, ApiSpec } from '../model/api/api-interfaces';
 import { WorkerFormatterKey } from './ui-worker-formatters';
 
@@ -149,10 +149,11 @@ export async function parseCert(buffer: ArrayBuffer) {
     })).result;
 }
 
-export async function formatBufferAsync(buffer: ArrayBuffer, format: WorkerFormatterKey) {
+export async function formatBufferAsync(buffer: ArrayBuffer, format: WorkerFormatterKey, headers?: Headers) {
     return (await callApi<FormatRequest, FormatResponse>({
         type: 'format',
         buffer,
-        format
+        format,
+        headers,
     })).formatted;
 }