diff --git a/package-lock.json b/package-lock.json index dd78a9b4..f86f71f7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,6 +73,7 @@ "har-validator": "^5.1.3", "http-encoding": "^2.0.1", "js-beautify": "^1.8.8", + "jsonc-parser": "^3.3.1", "jsonwebtoken": "^8.4.0", "localforage": "^1.7.3", "lodash": "^4.17.21", @@ -12394,10 +12395,10 @@ } }, "node_modules/jsonc-parser": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", - "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", - "dev": true + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "license": "MIT" }, "node_modules/jsonp": { "version": "0.2.1", @@ -20854,6 +20855,13 @@ "vscode-languageserver-types": "^3.6.0-next.1" } }, + "node_modules/vscode-emmet-helper/node_modules/jsonc-parser": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", + "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", + "dev": true, + "license": "MIT" + }, "node_modules/vscode-languageserver-types": { "version": "3.15.1", "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz", @@ -31539,10 +31547,9 @@ "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==" }, "jsonc-parser": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", - "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", - "dev": true + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==" }, "jsonp": { "version": "0.2.1", @@ -37912,6 +37919,14 @@ "@emmetio/extract-abbreviation": "0.1.6", "jsonc-parser": "^1.0.0", "vscode-languageserver-types": "^3.6.0-next.1" + }, + "dependencies": { + "jsonc-parser": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-1.0.3.tgz", + "integrity": "sha512-hk/69oAeaIzchq/v3lS50PXuzn5O2ynldopMC+SWBql7J2WtdptfB9dy8Y7+Og5rPkTCpn83zTiO8FMcqlXJ/g==", + "dev": true + } } }, "vscode-languageserver-types": { diff --git a/package.json b/package.json index ec0f18aa..409c791e 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "har-validator": "^5.1.3", "http-encoding": "^2.0.1", "js-beautify": "^1.8.8", + "jsonc-parser": "^3.3.1", "jsonwebtoken": "^8.4.0", "localforage": "^1.7.3", "lodash": "^4.17.21", diff --git a/src/components/editor/monaco.ts b/src/components/editor/monaco.ts index 245345dd..8ef7914d 100644 --- a/src/components/editor/monaco.ts +++ b/src/components/editor/monaco.ts @@ -1,12 +1,13 @@ import type * as MonacoTypes from 'monaco-editor'; import type { default as _MonacoEditor, MonacoEditorProps } from 'react-monaco-editor'; +import { observable, runInAction } from 'mobx'; import { defineMonacoThemes } from '../../styles'; import { delay } from '../../util/promise'; import { asError } from '../../util/error'; -import { observable, runInAction } from 'mobx'; -import { setupXMLValidation } from './xml-validation'; + +import { ContentValidator, validateJsonRecords, validateXml } from '../../model/events/content-validation'; export type { MonacoTypes, @@ -75,7 +76,12 @@ async function loadMonacoEditor(retries = 5): Promise { }, }); - setupXMLValidation(monaco); + monaco.languages.register({ + id: 'json-records' + }); + + addValidator(monaco, 'xml', validateXml); + addValidator(monaco, 'json-records', validateJsonRecords); MonacoEditor = rmeModule.default; } catch (err) { @@ -89,6 +95,43 @@ async function loadMonacoEditor(retries = 5): Promise { } } +function addValidator(monaco: typeof MonacoTypes, modeId: string, validator: ContentValidator) { + function validate(model: MonacoTypes.editor.ITextModel) { + const text = model.getValue(); + const markers = validator(text, model); + monaco.editor.setModelMarkers(model, modeId, markers); + } + + const contentChangeListeners = new Map(); + + function manageContentChangeListener(model: MonacoTypes.editor.ITextModel) { + const isActiveMode = model.getModeId() === modeId; + const listener = contentChangeListeners.get(model); + + if (isActiveMode && !listener) { + contentChangeListeners.set(model, model.onDidChangeContent(() => + validate(model) + )); + validate(model); + } else if (!isActiveMode && listener) { + listener.dispose(); + contentChangeListeners.delete(model); + monaco.editor.setModelMarkers(model, modeId, []); + } + } + + monaco.editor.onWillDisposeModel(model => { + contentChangeListeners.delete(model); + }); + monaco.editor.onDidChangeModelLanguage(({ model }) => { + manageContentChangeListener(model); + }); + monaco.editor.onDidCreateModel(model => { + manageContentChangeListener(model); + }); + +} + export function reloadMonacoEditor() { return monacoLoadingPromise = loadMonacoEditor(0); } diff --git a/src/components/editor/xml-validation.ts b/src/components/editor/xml-validation.ts deleted file mode 100644 index d71f632d..00000000 --- a/src/components/editor/xml-validation.ts +++ /dev/null @@ -1,58 +0,0 @@ -import type * as MonacoTypes from 'monaco-editor' -import { XMLValidator } from 'fast-xml-parser' - -export function setupXMLValidation(monaco: typeof MonacoTypes) { - const markerId = 'xml-validation' - - function validate(model: MonacoTypes.editor.ITextModel) { - const markers: MonacoTypes.editor.IMarkerData[] = [] - const text = model.getValue() - - if (text.trim()) { - const validationResult = XMLValidator.validate(text, { - allowBooleanAttributes: true, - }) - - if (validationResult !== true) { - markers.push({ - severity: monaco.MarkerSeverity.Error, - startLineNumber: validationResult.err.line, - startColumn: validationResult.err.col, - endLineNumber: validationResult.err.line, - endColumn: model.getLineContent(validationResult.err.line).length + 1, - message: validationResult.err.msg, - }) - } - } - - monaco.editor.setModelMarkers(model, markerId, markers) - } - - const contentChangeListeners = new Map() - function manageContentChangeListener(model: MonacoTypes.editor.ITextModel) { - const isXml = model.getModeId() === 'xml' - const listener = contentChangeListeners.get(model) - - if (isXml && !listener) { - contentChangeListeners.set( - model, - model.onDidChangeContent(() => validate(model)) - ) - validate(model) - } else if (!isXml && listener) { - listener.dispose() - contentChangeListeners.delete(model) - monaco.editor.setModelMarkers(model, markerId, []) - } - } - - monaco.editor.onWillDisposeModel(model => { - contentChangeListeners.delete(model) - }) - monaco.editor.onDidChangeModelLanguage(({ model }) => { - manageContentChangeListener(model) - }) - monaco.editor.onDidCreateModel(model => { - manageContentChangeListener(model) - }) -} diff --git a/src/model/events/body-formatting.ts b/src/model/events/body-formatting.ts index 82c557d1..cff17b45 100644 --- a/src/model/events/body-formatting.ts +++ b/src/model/events/body-formatting.ts @@ -9,6 +9,7 @@ import type { WorkerFormatterKey } from '../../services/ui-worker-formatters'; import { formatBufferAsync } from '../../services/ui-worker-api'; import { ReadOnlyParams } from '../../components/common/editable-params'; import { ImageViewer } from '../../components/editor/image-viewer'; +import { formatJson } from '../../util/json'; export interface EditorFormatter { language: string; @@ -106,20 +107,7 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { render: (input: Buffer, headers?: Headers) => { if (input.byteLength < 10_000) { const inputAsString = bufferToString(input); - - try { - // For short-ish inputs, we return synchronously - conveniently this avoids - // showing the loading spinner that churns the layout in short content cases. - return JSON.stringify( - JSON.parse(inputAsString), - null, - 2 - ); - // ^ Same logic as in UI-worker-formatter - } catch (e) { - // Fallback to showing the raw un-formatted JSON: - return inputAsString; - } + return formatJson(inputAsString, { formatRecords: false }); } else { return observablePromise( formatBufferAsync(input, 'json', headers) @@ -127,6 +115,21 @@ export const Formatters: { [key in ViewableContentType]: Formatter } = { } } }, + 'json-records': { + language: 'json-records', + cacheKey: Symbol('json-records'), + isEditApplicable: false, + render: (input: Buffer, headers?: Headers) => { + if (input.byteLength < 10_000) { + const inputAsString = bufferToString(input); + return formatJson(inputAsString, { formatRecords: true }); + } else { + return observablePromise( + formatBufferAsync(input, 'json-records', headers) + ); + } + } + }, javascript: { language: 'javascript', cacheKey: Symbol('javascript'), diff --git a/src/model/events/content-types.ts b/src/model/events/content-types.ts index 09520bc3..376001b6 100644 --- a/src/model/events/content-types.ts +++ b/src/model/events/content-types.ts @@ -7,6 +7,7 @@ import { isProbablyGrpcProto, isValidGrpcProto, } from '../../util/protobuf'; +import { isProbablyJson, isProbablyJsonRecords } from '../../util/json'; // Simplify a mime type as much as we can, without throwing any errors export const getBaseContentType = (mimeType: string | undefined) => { @@ -51,7 +52,9 @@ export type ViewableContentType = | 'yaml' | 'image' | 'protobuf' - | 'grpc-proto'; + | 'grpc-proto' + | 'json-records' + ; export const EditableContentTypes = [ 'text', @@ -119,6 +122,14 @@ const mimeTypeToContentTypeMap: { [mimeType: string]: ViewableContentType } = { 'application/grpc-proto': 'grpc-proto', 'application/grpc-protobuf': 'grpc-proto', + // Nobody can quite agree on the names for the various sequence-of-JSON formats: + 'application/jsonlines': 'json-records', + 'application/json-lines': 'json-records', + 'application/x-jsonlines': 'json-records', + 'application/jsonl': 'json-records', + 'application/x-ndjson': 'json-records', + 'application/json-seq': 'json-records', + 'application/octet-stream': 'raw' } as const; @@ -141,6 +152,7 @@ export function getEditableContentType(mimeType: string | undefined): EditableCo export function getContentEditorName(contentType: ViewableContentType): string { return contentType === 'raw' ? 'Hex' + : contentType === 'json-records' ? 'JSON Records' : contentType === 'json' ? 'JSON' : contentType === 'css' ? 'CSS' : contentType === 'url-encoded' ? 'URL-Encoded' @@ -186,16 +198,18 @@ export function getCompatibleTypes( body = body.decodedData; } - // Examine the first char of the body, assuming it's ascii - const firstChar = body && body.subarray(0, 1).toString('ascii'); + // Allow optionally formatting non-JSON-records as JSON-records, if it looks like it might be + if (!types.has('json-records') && isProbablyJsonRecords(body)) { + types.add('json-records'); + } - // Allow optionally formatting non-JSON as JSON, if it looks like it might be - if (firstChar === '{' || firstChar === '[') { + if (!types.has('json-records') && isProbablyJson(body)) { + // Allow optionally formatting non-JSON as JSON, if it's anything remotely close types.add('json'); } // Allow optionally formatting non-XML as XML, if it looks like it might be - if (firstChar === '<') { + if (body?.subarray(0, 1).toString('ascii') === '<') { types.add('xml'); } diff --git a/src/model/events/content-validation.ts b/src/model/events/content-validation.ts new file mode 100644 index 00000000..9bf99ea3 --- /dev/null +++ b/src/model/events/content-validation.ts @@ -0,0 +1,107 @@ +import type * as MonacoTypes from 'monaco-editor'; + +import { XMLValidator } from 'fast-xml-parser'; +import * as jsonCParser from 'jsonc-parser'; + +import { camelToSentenceCase } from '../../util/text'; +import { RECORD_SEPARATOR_CHARS } from '../../util/json'; + +export interface ContentValidator { + (text: string, model: MonacoTypes.editor.ITextModel): ValidationMarker[]; +} + +// A minimal more generic version of Monaco's MonacoTypes.editor.IMarkerData type: +export interface ValidationMarker { + severity: MarkerSeverity; + message: string; + startLineNumber: number; + startColumn: number; + endLineNumber: number; + endColumn: number; +} + +enum MarkerSeverity { + Hint = 1, + Info = 2, + Warning = 4, + Error = 8 +} + +export function validateXml(text: string): ValidationMarker[] { + const markers: ValidationMarker[] = []; + + if (!text.trim()) return markers; + + const validationResult = XMLValidator.validate(text, { + allowBooleanAttributes: true, + }) + + if (validationResult !== true) { + markers.push({ + severity: MarkerSeverity.Error, + startLineNumber: validationResult.err.line, + startColumn: validationResult.err.col, + endLineNumber: validationResult.err.line, + endColumn: Infinity, + message: validationResult.err.msg, + }) + } + + return markers; +} + +export function validateJsonRecords(text: string, model: MonacoTypes.editor.ITextModel): ValidationMarker[] { + const markers: ValidationMarker[] = []; + if (!text.trim()) return markers; + + let offset = 0; + let remainingText = text.trimEnd(); + while (remainingText) { + if (RECORD_SEPARATOR_CHARS.includes(remainingText[0])) { + remainingText = remainingText.slice(1); + offset++; + continue; + } + + const errors: jsonCParser.ParseError[] = []; + const result = jsonCParser.parseTree(remainingText, errors, { + allowTrailingComma: false, + disallowComments: true + }); + + const parsedContentLength = result + ? result.offset + result.length + : Math.max(...errors.map((err) => err.offset + err.length)); + + // We show the first error for each record, except any errors after the end of + // a completely parsed value (i.e. due to hitting the subsequent record instead + // of an expected EOF). We'll handle that in the next iteration one way or another. + if (errors.length) { + const firstError = errors[0]; + if (firstError && firstError.offset < parsedContentLength) { + const position = model.getPositionAt(offset + firstError.offset); + markers.push({ + severity: MarkerSeverity.Error, + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: position.column, + endColumn: position.column + firstError.length, + message: camelToSentenceCase(jsonCParser.printParseErrorCode(firstError.error)) + }); + } + } + + if (parsedContentLength === 0) { + // Should never happen, but this should at least give us some debug info if it ever does: + console.log(`Parsing ${remainingText} (${remainingText.length}) parsedContentLength was 0 with ${ + result + } result and ${JSON.stringify(errors)} errors`); + throw new Error(`JSON record parsed content length was 0`); + } + + remainingText = remainingText.slice(parsedContentLength); + offset += parsedContentLength; + } + + return markers; +} diff --git a/src/model/events/stream-message.ts b/src/model/events/stream-message.ts index 3b1a79ab..0f01b569 100644 --- a/src/model/events/stream-message.ts +++ b/src/model/events/stream-message.ts @@ -3,6 +3,7 @@ import { computed, observable } from 'mobx'; import { InputStreamMessage } from "../../types"; import { asBuffer } from '../../util/buffer'; import { ObservableCache } from '../observable-cache'; +import { isProbablyJson, isProbablyJsonRecords } from '../../util/json'; export class StreamMessage { @@ -52,12 +53,11 @@ export class StreamMessage { } // prefix+JSON is very common, so we try to parse anything JSON-ish optimistically: - const startOfMessage = this.content.subarray(0, 10).toString('utf-8').trim(); - if ( - startOfMessage.includes('{') || - startOfMessage.includes('[') || - this.subprotocol?.includes('json') - ) return 'json'; + if (isProbablyJsonRecords(this.content)) { + return 'json-records'; + } else if (isProbablyJson(this.content) || this.subprotocol?.includes('json')) { + return 'json'; + } else return 'text'; } diff --git a/src/services/ui-worker-formatters.ts b/src/services/ui-worker-formatters.ts index 96a2e632..53110a61 100644 --- a/src/services/ui-worker-formatters.ts +++ b/src/services/ui-worker-formatters.ts @@ -8,6 +8,7 @@ import * as beautifyXml from 'xml-beautifier'; import { Headers } from '../types'; import { bufferToHex, bufferToString, getReadableSize } from '../util/buffer'; import { parseRawProtobuf, extractProtobufFromGrpc } from '../util/protobuf'; +import { formatJson } from '../util/json'; const truncationMarker = (size: string) => `\n[-- Truncated to ${size} --]`; const FIVE_MB = 1024 * 1024 * 5; @@ -74,11 +75,11 @@ const WorkerFormatters = { }, json: (content: Buffer) => { const asString = content.toString('utf8'); - try { - return JSON.stringify(JSON.parse(asString), null, 2); - } catch (e) { - return asString; - } + return formatJson(asString, { formatRecords: false }); + }, + 'json-records': (content: Buffer) => { + const asString = content.toString('utf8'); + return formatJson(asString, { formatRecords: true }); }, javascript: (content: Buffer) => { return beautifyJs(content.toString('utf8'), { diff --git a/src/util/json.ts b/src/util/json.ts new file mode 100644 index 00000000..29333e8f --- /dev/null +++ b/src/util/json.ts @@ -0,0 +1,271 @@ +import { + createScanner as createJsonScanner, + SyntaxKind as JsonSyntaxKind +} from 'jsonc-parser'; + +const JSON_START_REGEX = /^\s*[\[\{tfn"\d-]/; // Optional whitespace, then start array/object/true/false/null/string/number + +const JSON_TEXT_SEQ_START_REGEX = /^\u001E\s*[\[\{]/; // Record separate, optional whitespace, then array/object + +const SIGNALR_HUB_START_REGEX = /^\s*\{/; // Optional whitespace, then start object +const SIGNALR_HUB_END_REGEX = /\}\s*\u001E$/; // End object, optional whitespace, then record separator + +const JSON_LINES_END_REGEX = /[\]\}](\r?\n)+$/; // Array/object end, then optional newline(s) +const JSON_LINES_INNER_REGEX = /[\]\}](\r?\n)+[\{\[]/; // Object/array end, then newline(s), then start object/array +const JSON_LINES_SCAN_LIMIT = 16 * 1024; + +export const isProbablyJson = (text: Buffer | undefined) => { + if (!text || text.length < 2) return false; + + const startChunk = text.subarray(0, 6).toString('utf8'); + return JSON_START_REGEX.test(startChunk); +} + +export const isProbablyJsonRecords = (text: Buffer | undefined) => { + if (!text || text.length < 3) return false; + + // This has some false negatives: e.g. standalone JSON primitive values, or unusual + // extra whitespace in certain places, but I think it should provide pretty good coverage + // the rest of the time. + + const startChunk = text.subarray(0, 6).toString('utf8'); + const endChunk = text.subarray(-6).toString('utf8'); + + if (JSON_TEXT_SEQ_START_REGEX.test(startChunk)) { + // JSON text sequence: https://www.rfc-editor.org/rfc/rfc7464.html + return true; + } + + if (SIGNALR_HUB_START_REGEX.test(startChunk) && SIGNALR_HUB_END_REGEX.test(endChunk)) { + // SignalR hub protocol: + // https://github.com/dotnet/aspnetcore/blob/main/src/SignalR/docs/specs/HubProtocol.md#json-encoding + return true; + } + + if ( + JSON_START_REGEX.test(startChunk) && + // Technically for JSON Lines the end newline is optional, but strongly recommended & common AFAICT + JSON_LINES_END_REGEX.test(endChunk) && + // If the end looks like JSON lines/NDJSON, so scan a bigger chunk to check: + JSON_LINES_INNER_REGEX.test(text.subarray(0, JSON_LINES_SCAN_LIMIT).toString('utf8')) + ) { + // JSON Lines or NDJSON: https://jsonlines.org/ + return true; + } + + return false; +} + +export const RECORD_SEPARATOR_CHARS = [ + '\u001E', // ASCII Record Separator (used by SignalR and others) + '\n', + '\r' +]; + +// A *very* forgiving & flexible JSON formatter. This will correctly format even things that +// will fail validation later, such as trailing commas, unquoted keys, comments, etc. +export function formatJson(text: string, options: { formatRecords: boolean } = { formatRecords: false }): string { + const scanner = createJsonScanner(text); + + let result = ""; + let indent = 0; + let token: JsonSyntaxKind; + + const indentString = ' '; + let needsIndent = false; + let previousToken: JsonSyntaxKind | null = null; + + let betweenRecords = false; + + while ((token = scanner.scan()) !== JsonSyntaxKind.EOF) { + const tokenOffset = scanner.getTokenOffset(); + const tokenLength = scanner.getTokenLength(); + const tokenText = text.slice(tokenOffset, tokenOffset + tokenLength); + + if (options.formatRecords && indent === 0) { + betweenRecords = true; + } + + // Skip over explicit 'record separator' characters, which can cause parsing problems + // when parsing JSON records: + if (betweenRecords && tokenText[0] === '\u001E') { + scanner.setPosition(tokenOffset + 1); + continue; + } + + // Ignore irrelevant whitespace (internally or between records) - we'll handle that ourselves + if (token === JsonSyntaxKind.Trivia || token === JsonSyntaxKind.LineBreakTrivia) { + continue; + } + + if (betweenRecords) { + // We've finished one record and we have another coming that won't get a newline + // automatically: add an extra newline to properly separate records if required. + betweenRecords = false; + if (result && result[result.length - 1] !== '\n' && !isValueToken(token)) { + result += '\n'; + } + } + + if (needsIndent) { + result += indentString.repeat(indent); + needsIndent = false; + } + + switch (token) { + case JsonSyntaxKind.OpenBraceToken: + case JsonSyntaxKind.OpenBracketToken: + result += tokenText; + indent++; + + const afterOpener = scanAhead(scanner); + const isClosing = afterOpener === JsonSyntaxKind.CloseBraceToken || + afterOpener === JsonSyntaxKind.CloseBracketToken; + if ( + !isClosing && + afterOpener !== JsonSyntaxKind.EOF && + afterOpener !== JsonSyntaxKind.LineCommentTrivia + ) { + result += '\n'; + needsIndent = true; + } + break; + + case JsonSyntaxKind.CloseBraceToken: + case JsonSyntaxKind.CloseBracketToken: + const wasEmpty = previousToken === JsonSyntaxKind.OpenBraceToken || + previousToken === JsonSyntaxKind.OpenBracketToken; + + let indentUnderflow = indent === 0; + indent = Math.max(0, indent - 1); + + if (!wasEmpty) { + if (!result.endsWith('\n')) { + result += '\n'; + } + result += indentString.repeat(indent); + } + + result += tokenText; + if (indentUnderflow) result += '\n'; + + break; + + case JsonSyntaxKind.CommaToken: + result += tokenText; + + const afterComma = scanAhead(scanner); + if ( + afterComma !== JsonSyntaxKind.LineCommentTrivia && + afterComma !== JsonSyntaxKind.BlockCommentTrivia && + afterComma !== JsonSyntaxKind.CloseBraceToken && + afterComma !== JsonSyntaxKind.CloseBracketToken && + afterComma !== JsonSyntaxKind.EOF && + afterComma !== JsonSyntaxKind.CommaToken + ) { + result += '\n'; + needsIndent = true; + } + break; + + case JsonSyntaxKind.ColonToken: + result += tokenText; + result += ' '; + break; + + case JsonSyntaxKind.LineCommentTrivia: + const needsNewlineBefore = ( + previousToken === JsonSyntaxKind.OpenBraceToken || + previousToken === JsonSyntaxKind.OpenBracketToken + ) && !result.endsWith('\n'); + + if (needsNewlineBefore) { + result += '\n'; + needsIndent = true; + } + + if (needsIndent) { + result += indentString.repeat(indent); + needsIndent = false; + result += tokenText; + } else { + const trimmedResult = result.trimEnd(); + if (result.length > trimmedResult.length) { + result = trimmedResult + tokenText; + } else { + result += ' ' + tokenText; + } + } + + const afterComment = scanAhead(scanner); + if ( + afterComment !== JsonSyntaxKind.CloseBraceToken && + afterComment !== JsonSyntaxKind.CloseBracketToken && + afterComment !== JsonSyntaxKind.EOF + ) { + result += '\n'; + needsIndent = true; + } + break; + + case JsonSyntaxKind.BlockCommentTrivia: + const prevChar = result[result.length - 1]; + if (prevChar === '\n' || (prevChar === ' ' && result[result.length - 2] === '\n')) { + result += tokenText; + } else { + result += ' ' + tokenText; + } + + const afterBlock = scanAhead(scanner); + if ( + afterBlock !== JsonSyntaxKind.CommaToken && + afterBlock !== JsonSyntaxKind.CloseBraceToken && + afterBlock !== JsonSyntaxKind.CloseBracketToken && + afterBlock !== JsonSyntaxKind.EOF + ) { + result += '\n'; + needsIndent = true; + } + break; + + default: + const followsValue = isValueToken(previousToken); + if (followsValue && isValueToken(token) && !result.endsWith('\n')) { + // Missing comma detected between sequential values, + // so add a newline for readability + result += '\n'; + result += indentString.repeat(indent); + } + + result += tokenText; + break; + } + + previousToken = token; + } + + return result; +} + +function isValueToken(token: JsonSyntaxKind | null): boolean { + return token === JsonSyntaxKind.StringLiteral || + token === JsonSyntaxKind.NumericLiteral || + token === JsonSyntaxKind.TrueKeyword || + token === JsonSyntaxKind.FalseKeyword || + token === JsonSyntaxKind.NullKeyword || + token === JsonSyntaxKind.CloseBraceToken || + token === JsonSyntaxKind.CloseBracketToken; +} + +function scanAhead(scanner: any): JsonSyntaxKind { + const savedPosition = scanner.getPosition(); + + let nextToken = scanner.scan(); + while (nextToken === JsonSyntaxKind.Trivia || + nextToken === JsonSyntaxKind.LineBreakTrivia) { + nextToken = scanner.scan(); + } + + scanner.setPosition(savedPosition); + return nextToken; +} diff --git a/src/util/text.ts b/src/util/text.ts index f59f7e98..e319b603 100644 --- a/src/util/text.ts +++ b/src/util/text.ts @@ -20,4 +20,11 @@ export function aOrAn(value: string) { export function uppercaseFirst(value: string) { return value[0].toUpperCase() + value.slice(1); +} + +export function camelToSentenceCase(value: string) { + return uppercaseFirst( + value.replace(/([a-z])([A-Z])/g, '$1 $2') + .toLowerCase() + ); } \ No newline at end of file diff --git a/test/unit/model/events/content-validation.spec.tsx b/test/unit/model/events/content-validation.spec.tsx new file mode 100644 index 00000000..32b47532 --- /dev/null +++ b/test/unit/model/events/content-validation.spec.tsx @@ -0,0 +1,123 @@ +import { expect } from "chai"; +import * as monaco from 'monaco-editor'; + +import { + validateJsonRecords, + validateXml +} from "../../../../src/model/events/content-validation"; + +const contentModel = (content: string) => monaco.editor.createModel(content, "json-records"); + +describe("XML validation", () => { + it("should validate correct XML", () => { + const text = `Content`; + const markers = validateXml(text); + expect(markers).to.deep.equal([]); + }); + + it("should reject incorrect XML", () => { + const text = ` + Content + `; + const markers = validateXml(text); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Expected closing tag 'root' (opened in line 1, col 1) instead of closing tag 'wrong'.", + startLineNumber: 3, + endLineNumber: 3, + startColumn: 9, + endColumn: Infinity + } + ]); + }); +}) + +describe("JSON Records Validation", () => { + it("should validate correct normal JSON", () => { + const text = '{"name":"John"}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([]); + }); + + it("should validate correct JSON records", () => { + const text = '{"name":"John"}\n\u001E\n{"name":"Jane"}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([]); + }); + + it("should reject incorrect JSON", () => { + const text = '{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 1, + startColumn: 9, + endLineNumber: 1, + endColumn: 10 + } + ]); + }); + + it("should reject incorrect record-separator JSON records", () => { + const text = '\u001E{"name":"John"}\u001E{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 1, + startColumn: 26, + endLineNumber: 1, + endColumn: 27 + } + ]); + }); + + it("should reject incorrect newline-separator JSON records", () => { + const text = '{"name":"John"}\n{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 2, + startColumn: 9, + endLineNumber: 2, + endColumn: 10 + } + ]); + }); + + it("should reject incorrect record-separator JSON records with newlines separators too", () => { + const text = '\n{"name":"John"}\n\u001E\n{"name":}\n'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 4, + startColumn: 9, + endLineNumber: 4, + endColumn: 10 + } + ]); + }); + + it("should reject incorrect JSON records with good line numbers despite spurious newlines", () => { + const text = '\n{"name":\n\n"John"\n}\n\u001E\n\u001E\n{"name":}'; + const markers = validateJsonRecords(text, contentModel(text)); + expect(markers).to.deep.equal([ + { + severity: 8, + message: "Value expected", + startLineNumber: 8, + startColumn: 9, + endLineNumber: 8, + endColumn: 10 + } + ]); + }); +}); \ No newline at end of file diff --git a/test/unit/util/json.spec.ts b/test/unit/util/json.spec.ts new file mode 100644 index 00000000..45bb3604 --- /dev/null +++ b/test/unit/util/json.spec.ts @@ -0,0 +1,339 @@ +import { expect } from "chai"; + +import { formatJson } from "../../../src/util/json"; + +describe("JSON formatting", () => { + + describe("given valid JSON", () => { + it("should format a simple object", () => { + const input = '{"b":2,"a":1}'; + const expected = `{ + "b": 2, + "a": 1 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should correctly format an object with various data types", () => { + const input = '{"str":"hello world","num":-123.45,"bool":false,"n":null,"emp_str":""}'; + const expected = `{ + "str": "hello world", + "num": -123.45, + "bool": false, + "n": null, + "emp_str": "" +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should correctly format an array with mixed data types", () => { + const input = '["a string",-123.45,false,null,{"key":"value"},["nested"]]'; + const expected = `[ + "a string", + -123.45, + false, + null, + { + "key": "value" + }, + [ + "nested" + ] +]`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should correctly format strings with escaped characters", () => { + const input = '{"path":"C:\\\\Users\\\\Test","quote":"\\"hello\\""}'; + const expected = `{ + "path": "C:\\\\Users\\\\Test", + "quote": "\\"hello\\"" +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should handle deeply nested structures correctly", () => { + const input = '{"a":{"b":{"c":{"d":[1,{"e":2},3]}}}}'; + const expected = `{ + "a": { + "b": { + "c": { + "d": [ + 1, + { + "e": 2 + }, + 3 + ] + } + } + } +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should normalize inconsistent whitespace and newlines", () => { + const input = ` + { + "a" : 1, + "b" : [ 2, + 3 ] + } + `; + const expected = `{ + "a": 1, + "b": [ + 2, + 3 + ] +}`; + expect(formatJson(input)).to.equal(expected); + }); + }); + + describe("given invalid JSON", () => { + + it("should preserve missing quotes", () => { + const invalidInput = '{a: 1}'; + const expected = `{ + a: 1 +}`; + expect(() => formatJson(invalidInput)).to.not.throw(); + expect(formatJson(invalidInput)).to.equal(expected); + }); + + it("should preserve single quotes", () => { + const invalidInput = '{\'a\': 1}'; + const expected = `{ + 'a': 1 +}`; + expect(() => formatJson(invalidInput)).to.not.throw(); + expect(formatJson(invalidInput)).to.equal(expected); + }); + + it("should preserve a trailing comma", () => { + const input = '{"a":1, "b":2,}'; + const expected = `{ + "a": 1, + "b": 2, +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a single-line comment at the end of a line", () => { + const input = '{"a": 1, // My line comment\n"b": 2}'; + const expected = `{ + "a": 1, // My line comment + "b": 2 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a block comment at the end of a line", () => { + const input = '{"a": 1 /* comment */, "b": 2}'; + const expected = `{ + "a": 1 /* comment */, + "b": 2 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a block comment before a closing brace", () => { + const input = '{ "a": 1 /* final comment */ }'; + const expected = `{ + "a": 1 /* final comment */ +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve both single-line and block comments", () => { + const input = `{ + // This is a comment to be kept + "a": 1, /* This should be kept */ + "b": 2 // This should also be kept + }`; + const expected = `{ + // This is a comment to be kept + "a": 1, /* This should be kept */ + "b": 2 // This should also be kept +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve a trailing comma when followed by comments", () => { + const input = '{"a":1, "b":2, // trailing comma\n}'; + const expected = `{ + "a": 1, + "b": 2, // trailing comma +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format lines correctly despite a missing comma between object properties", () => { + const input = '{"a": 1 "b": 2}'; + const expected = `{ + "a": 1 + "b": 2 +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format lines correctly despite a missing comma between array elements", () => { + const input = '[1 "a" false]'; + const expected = `[ + 1 + "a" + false +]`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should preserve multiple consecutive or trailing commas", () => { + const input = '{"a":1,,"b":2,,}'; + const expected = `{ + "a": 1,, + "b": 2,, +}`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format an unclosed object without adding the closing brace", () => { + const input = '{"a": 1, "b": 2'; + const expected = `{ + "a": 1, + "b": 2`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should format an unclosed array without adding the closing bracket", () => { + const input = '[1, {"a": 2'; + const expected = `[ + 1, + { + "a": 2`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should handle overly closed JSON (negative indents)", () => { + const input = '{}}} []]]'; + const expected = `{} +} +} +[] +] +] +`; + expect(formatJson(input)).to.equal(expected); + }); + + it("should not format complicated JSON records at all if formatRecords is false", () => { + const input = '\u001E{"a":1}\n\u001Enull\n\u001Etrue\n\u001E"hi"\n\u001E[1,2,3]\n\u001E{"b":2}\n'; + const expected = `\u001E{ + "a": 1 +}\u001Enull\u001Etrue\u001E"hi"\u001E[ + 1, + 2, + 3 +]\u001E{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: false })).to.equal(expected); + }); + }); + + describe("with record formatting enabled", () => { + + it("should correctly format newline-separated JSON records", () => { + const input = '{"a":1}\n{"b":2}'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format newline-separated JSON records with weird newlines", () => { + const input = '\n{\n"a"\n:1}\r\n\n\n{\n"b"\n:\n2\r}\n\r\n'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format record-separator-separated JSON records (SignalR style)", () => { + const input = '{"a":1}\u001E{"b":2}'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format heavily record-separator-separated JSON records", () => { + const input = '\u001E\u001E{"a":1}\u001E\u001E{"b":2}\u001E'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format heavily record-separator-separated JSON records with funky newlines", () => { + const input = '\r\n\u001E\n{"a":1}\n\u001E{"b":2}\n\u001E'; + const expected = `{ + "a": 1 +} +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should correctly format record-separator-separated JSON records with literal values (json-seq style)", () => { + const input = '\u001E{"a":1}\n\u001Enull\n\u001Etrue\n\u001E"hi"\n\u001E[1,2,3]\n\u001E{"b":2}\n'; + const expected = `{ + "a": 1 +} +null +true +"hi" +[ + 1, + 2, + 3 +] +{ + "b": 2 +}`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + it("should still correctly format an array with mixed data types", () => { + const input = '["a string",-123.45,false,null,{"key":"value"},["nested"]]'; + const expected = `[ + "a string", + -123.45, + false, + null, + { + "key": "value" + }, + [ + "nested" + ] +]`; + expect(formatJson(input, { formatRecords: true })).to.equal(expected); + }); + + }); + +}); \ No newline at end of file