feat: json-records content type support

Author: WeihanLi

src/model/events/body-formatting.ts

@@ -1,9 +1,9 @@
 import { Headers } from '../../types';
 import { styled } from '../../styles';
 
-import { ViewableContentType } from '../events/content-types';
+import { jsonRecordsSeparators, ViewableContentType } from '../events/content-types';
 import { ObservablePromise, observablePromise } from '../../util/observable';
-import { bufferToString, bufferToHex } from '../../util/buffer';
+import { bufferToString, bufferToHex, splitBuffer } from '../../util/buffer';
 
 import type { WorkerFormatterKey } from '../../services/ui-worker-formatters';
 import { formatBufferAsync } from '../../services/ui-worker-api';
@@ -127,6 +127,43 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = {
             }
         }
     },
+    'json-records': {
+        language: 'json',
+        cacheKey: Symbol('json-records'),
+        isEditApplicable: false,
+        render: (input: Buffer, headers?: Headers) => {
+            if (input.byteLength < 10_000) {
+                const inputAsString = bufferToString(input);
+                
+                try {
+                    let records = new Array();
+                    jsonRecordsSeparators.forEach((separator) => {
+                         splitBuffer(input, separator).forEach((recordBuffer: Buffer) => {
+                            if (recordBuffer.length > 0) {
+                                const record = recordBuffer.toString('utf-8');
+                                records.push(JSON.parse(record.trim()));
+                            }
+                        });
+                    });
+                    // For short-ish inputs, we return synchronously - conveniently this avoids
+                    // showing the loading spinner that churns the layout in short content cases.
+                    return JSON.stringify(
+                        records,
+                        null,
+                        2
+                    );
+                    // ^ Same logic as in UI-worker-formatter
+                } catch (e) {
+                    // Fallback to showing the raw un-formatted:
+                    return inputAsString;
+                }
+            } else {
+                return observablePromise(
+                    formatBufferAsync(input, 'json', headers)
+                );
+            }
+        }
+    },
     javascript: {
         language: 'javascript',
         cacheKey: Symbol('javascript'),

src/model/events/content-types.ts

@@ -51,7 +51,9 @@ export type ViewableContentType =
     | 'yaml'
     | 'image'
     | 'protobuf'
-    | 'grpc-proto';
+    | 'grpc-proto'
+    | 'json-records'
+    ;
 
 export const EditableContentTypes = [
     'text',
@@ -122,6 +124,10 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = {
     'application/octet-stream': 'raw'
 } as const;
 
+export const jsonRecordsSeparators = [
+    0x1E
+];
+
 export function getContentType(mimeType: string | undefined): ViewableContentType | undefined {
     const baseContentType = getBaseContentType(mimeType);
     return mimeTypeToContentTypeMap[baseContentType!];
@@ -141,6 +147,7 @@ export function getEditableContentType(mimeType: string | undefined): EditableCo
 
 export function getContentEditorName(contentType: ViewableContentType): string {
     return contentType === 'raw' ? 'Hex'
+        : contentType === 'json-records' ? 'JSON Records'
         : contentType === 'json' ? 'JSON'
         : contentType === 'css' ? 'CSS'
         : contentType === 'url-encoded' ? 'URL-Encoded'
@@ -189,6 +196,16 @@ export function getCompatibleTypes(
     // Examine the first char of the body, assuming it's ascii
     const firstChar = body && body.subarray(0, 1).toString('ascii');
 
+    // Allow optionally formatting non-JSON-records as JSON-records, if it looks like it might be
+    if (body && body.length > 2 && firstChar === '{' 
+            && jsonRecordsSeparators.indexOf(body[body.length - 1]) > -1
+    ) {
+        const secondToLastChar = body.subarray(body.length - 2, body.length - 1).toString('ascii');
+        if (secondToLastChar === '}') {
+            types.add('json-records');
+        }
+    }
+
     // Allow optionally formatting non-JSON as JSON, if it looks like it might be
     if (firstChar === '{' || firstChar === '[') {
         types.add('json');

src/model/events/stream-message.ts

@@ -3,6 +3,7 @@ import { computed, observable } from 'mobx';
 import { InputStreamMessage } from "../../types";
 import { asBuffer } from '../../util/buffer';
 import { ObservableCache } from '../observable-cache';
+import { jsonRecordsSeparators } from './content-types';
 
 export class StreamMessage {
 
@@ -57,7 +58,12 @@ export class StreamMessage {
             startOfMessage.includes('{') ||
             startOfMessage.includes('[') ||
             this.subprotocol?.includes('json')
-        ) return 'json';
+        ) {
+            if (jsonRecordsSeparators.indexOf(this.content[this.content.length - 1]) > -1)
+                return 'json-records';
+            else
+                return 'json';
+        }
 
         else return 'text';
     }

src/services/ui-worker-formatters.ts

@@ -6,8 +6,9 @@ import {
 import * as beautifyXml from 'xml-beautifier';
 
 import { Headers } from '../types';
-import { bufferToHex, bufferToString, getReadableSize } from '../util/buffer';
+import { bufferToHex, bufferToString, getReadableSize, splitBuffer } from '../util/buffer';
 import { parseRawProtobuf, extractProtobufFromGrpc } from '../util/protobuf';
+import { jsonRecordsSeparators } from '../model/events/content-types';
 
 const truncationMarker = (size: string) => `\n[-- Truncated to ${size} --]`;
 const FIVE_MB = 1024 * 1024 * 5;
@@ -80,6 +81,24 @@ const WorkerFormatters = {
             return asString;
         }
     },
+    'json-records': (content: Buffer) => {
+        const asString = content.toString('utf8');
+        
+        try {
+            let records = new Array();
+            jsonRecordsSeparators.forEach((separator) => {
+                    splitBuffer(content, separator).forEach((recordBuffer: Buffer) => {
+                    if (recordBuffer.length > 0) {
+                        const record = recordBuffer.toString('utf-8');
+                        records.push(JSON.parse(record.trim()));
+                    }
+                });
+            });
+            return JSON.stringify(records, null, 2);
+        } catch (e) {
+            return asString;
+        }
+    },
     javascript: (content: Buffer) => {
         return beautifyJs(content.toString('utf8'), {
             indent_size: 2

src/util/buffer.ts

@@ -149,4 +149,33 @@ export function getReadableSize(input: number | Buffer | string, siUnits = true)
     let unitName = bytes === 1 ? 'byte' : units[unitIndex];
 
     return (bytes / Math.pow(thresh, unitIndex)).toFixed(1).replace(/\.0$/, '') + ' ' + unitName;
+}
+
+/**
+ * Splits a Buffer into an array of Buffers using a specified separator.
+ * @param buffer The Buffer to split.
+ * @param separator The byte or Buffer sequence to split by.
+ * @returns An array of Buffers.
+ */
+export function splitBuffer(buffer: Buffer, separator: number | Buffer): Buffer[] {
+    const result: Buffer[] = [];
+    let currentOffset = 0;
+    let separatorIndex: number;
+
+    // Handle single byte separator vs. multi-byte separator
+    const separatorLength = typeof separator === 'number' ? 1 : separator.length;
+
+    while ((separatorIndex = buffer.indexOf(separator, currentOffset)) !== -1) {
+        // Add the chunk before the separator
+        result.push(buffer.slice(currentOffset, separatorIndex));
+        // Move the offset past the separator
+        currentOffset = separatorIndex + separatorLength;
+    }
+
+    // Add the last chunk (or the whole buffer if no separator was found)
+    if (currentOffset <= buffer.length) {
+        result.push(buffer.slice(currentOffset));
+    }
+
+    return result;
 }