Merge pull request #126 from bluemods/main

Update Protobuf Content Types

Author: pimterry

src/model/events/body-formatting.ts

@@ -142,6 +142,12 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = {
         isEditApplicable: false,
         render: buildAsyncRenderer('protobuf')
     },
+    'grpc-proto': {
+        language: 'protobuf',
+        cacheKey: Symbol('grpc-proto'),
+        isEditApplicable: false,
+        render: buildAsyncRenderer('grpc-proto')
+    },
     'url-encoded': {
         layout: 'scrollable',
         Component: styled(ReadOnlyParams).attrs((p: FormatComponentProps) => ({

src/model/events/content-types.ts

@@ -8,18 +8,32 @@ import {
 // Simplify a mime type as much as we can, without throwing any errors
 export const getBaseContentType = (mimeType: string | undefined) => {
     const typeWithoutParams = (mimeType || '').split(';')[0];
-    const [type, combinedSubTypes] = typeWithoutParams.split(/\/(.+)/);
 
+    let [type, combinedSubTypes] = typeWithoutParams.split(/\/(.+)/);
     if (!combinedSubTypes) return type;
 
-    // A list of types from most specific to most generic: [svg, xml] for image/svg+xml
-    const subTypes = combinedSubTypes.split('+');
+    if (DEFAULT_SUBTYPE[combinedSubTypes]) {
+        combinedSubTypes = `${combinedSubTypes}+${DEFAULT_SUBTYPE[combinedSubTypes]}`;
+    }
+
+    // If this is a known type with an exact match, return that directly:
+    if (mimeTypeToContentTypeMap[type + '/' + combinedSubTypes]) {
+        return type + '/' + combinedSubTypes;
+    }
 
+    // Otherwise, wr collect a list of types from most specific to most generic: [svg, xml] for image/svg+xml
+    // and then look through in order to see if there are any matches here:
+    const subTypes = combinedSubTypes.split('+');
     const possibleTypes = subTypes.map(st => type + '/' + st);
-    return _.find(possibleTypes, t => !!mimeTypeToContentTypeMap[t]) ||
+
+    return _.find(possibleTypes, t => !!mimeTypeToContentTypeMap[t]) || // Subtype match
         _.last(possibleTypes)!; // If we recognize none - return the most generic
 }
 
+const DEFAULT_SUBTYPE: { [type: string]: string } = {
+    'grpc': 'proto' // Protobuf is the default gRPC content type (but not the only one!)
+};
+
 export type ViewableContentType =
     | 'raw'
     | 'text'
@@ -33,7 +47,8 @@ export type ViewableContentType =
     | 'markdown'
     | 'yaml'
     | 'image'
-    | 'protobuf';
+    | 'protobuf'
+    | 'grpc-proto';
 
 export const EditableContentTypes = [
     'text',
@@ -94,6 +109,9 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = {
     'application/vnd.google.protobuf': 'protobuf',
     'application/x-google-protobuf': 'protobuf',
     'application/proto': 'protobuf', // N.b. this covers all application/XXX+proto values
+    'application/x-protobuffer': 'protobuf', // Commonly seen in Google apps
+
+    'application/grpc+proto': 'grpc-proto', // Used in GRPC requests (protobuf but with special headers)
 
     'application/octet-stream': 'raw'
 } as const;
@@ -120,6 +138,7 @@ export function getContentEditorName(contentType: ViewableContentType): string {
         : contentType === 'json' ? 'JSON'
         : contentType === 'css' ? 'CSS'
         : contentType === 'url-encoded' ? 'URL-Encoded'
+        : contentType === 'grpc-proto' ? 'gRPC'
         : _.capitalize(contentType);
 }
 
@@ -161,10 +180,15 @@ export function getCompatibleTypes(
         types.add('xml');
     }
 
+    if (!types.has('grpc-proto') && rawContentType === 'application/grpc') {
+        types.add('grpc-proto')
+    }
+
     if (
         body &&
         isProbablyProtobuf(body) &&
         !types.has('protobuf') &&
+        !types.has('grpc-proto') &&
         // If it's probably unmarked protobuf, and it's a manageable size, try
         // parsing it just to check:
         (body.length < 100_000 && isValidProtobuf(body))
@@ -192,4 +216,4 @@ export function getCompatibleTypes(
     types.add('raw');
 
     return Array.from(types);
-}
+}

src/services/ui-worker-formatters.ts

@@ -6,7 +6,7 @@ import {
 import * as beautifyXml from 'xml-beautifier';
 
 import { bufferToHex, bufferToString, getReadableSize } from '../util/buffer';
-import { parseRawProtobuf } from '../util/protobuf';
+import { parseRawProtobuf, extractProtobufFromGrpc } from '../util/protobuf';
 
 const truncationMarker = (size: string) => `\n[-- Truncated to ${size} --]`;
 const FIVE_MB = 1024 * 1024 * 5;
@@ -78,6 +78,27 @@ const WorkerFormatters = {
             prefix: ''
         });
 
+        return JSON.stringify(data, (_key, value) => {
+            // Buffers have toJSON defined, so arrive here in JSONified form:
+            if (value.type === 'Buffer' && Array.isArray(value.data)) {
+                const buffer = Buffer.from(value.data);
+
+                return {
+                    "Type": `Buffer (${getReadableSize(buffer)})`,
+                    "As string": bufferToString(buffer, 'detect-encoding'),
+                    "As hex": bufferToHex(buffer)
+                }
+            } else {
+                return value;
+            }
+        }, 2);
+    },
+    'grpc-proto': (content: Buffer) => {
+        const protobufMessages = extractProtobufFromGrpc(content);
+
+        let data = protobufMessages.map((msg) => parseRawProtobuf(msg, { prefix: '' }));
+        if (data.length === 1) data = data[0];
+
         return JSON.stringify(data, (_key, value) => {
             // Buffers have toJSON defined, so arrive here in JSONified form:
             if (value.type === 'Buffer' && Array.isArray(value.data)) {

src/util/protobuf.ts

@@ -28,6 +28,28 @@ export function isProbablyProtobuf(input: Uint8Array) {
 
 export const parseRawProtobuf = parseRawProto;
 
+// GRPC message structure:
+// The repeated sequence of Length-Prefixed-Message items is delivered in DATA frames
+// Length-Prefixed-Message → Compressed-Flag Message-Length Message
+// Compressed-Flag → 0 / 1 ; encoded as 1 byte unsigned integer
+// Message-Length → {length of Message} ; encoded as 4 byte unsigned integer (big endian)
+// Message → *{binary octet}
+export const extractProtobufFromGrpc = (input: Buffer) => {
+    const protobufMessasges: Buffer[] = [];
+
+    while (input.length > 0) {
+        if (input.readInt8() != 0) {
+            throw new Error("Compressed gRPC messages not yet supported")
+        }
+
+        const length = input.readInt32BE(1);
+        protobufMessasges.push(input.slice(5, 5 + length));
+        input = input.subarray(5 + length);
+    }
+
+    return protobufMessasges;
+}
+
 export const isValidProtobuf = (input: Uint8Array) => {
     try {
         parseRawProtobuf(input);

test/unit/model/http/content-types.spec.ts

@@ -49,6 +49,21 @@ describe('Content type parsing', () => {
             expect(ct).to.equal('raw');
         });
 
+        it('should render application/grpc as protobuf grpc', () => {
+            const ct = getContentType('application/grpc');
+            expect(ct).to.equal('grpc-proto');
+        });
+
+        it('should render application/grpc+proto as protobuf grpc', () => {
+            const ct = getContentType('application/grpc+proto');
+            expect(ct).to.equal('grpc-proto');
+        });
+
+        it('should render application/grpc+json as JSON', () => {
+            const ct = getContentType('application/grpc+json');
+            expect(ct).to.equal('json');
+        });
+
         it('should return undefined for unknown content', () => {
             const ct = getContentType('application/unknownsomething');
             expect(ct).to.equal(undefined);