diff --git a/builder/Cargo.lock b/builder/Cargo.lock index c7885f0b..9cfb766e 100644 --- a/builder/Cargo.lock +++ b/builder/Cargo.lock @@ -80,9 +80,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" dependencies = [ "clap_builder", "clap_derive", @@ -90,9 +90,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" dependencies = [ "anstream", "anstyle", diff --git a/builder/src/main.rs b/builder/src/main.rs index 24fcfa87..e68ca3dc 100644 --- a/builder/src/main.rs +++ b/builder/src/main.rs @@ -359,7 +359,7 @@ fn run_update() -> io::Result<()> { cargo update --manifest-path=../builder/Cargo.toml; cargo update; )?; - // Simply display outdated dependencies, but don't considert them an error. + // Simply display outdated dependencies, but don't consider them an error. run_script("npm", &["outdated"], "../client", false)?; run_script("npm", &["outdated"], "../extensions/VSCode", false)?; run_cmd!( diff --git a/client/package-lock.json b/client/package-lock.json index 2d2c777d..71a4a5eb 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "codechat-editor-client", - "version": "0.1.24", + "version": "0.1.26", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codechat-editor-client", - "version": "0.1.24", + "version": "0.1.26", "license": "GPL-3.0-or-later", "dependencies": { "@codemirror/lang-cpp": "^6", @@ -28,14 +28,16 @@ "graphviz-webcomponent": "^2", "mermaid": "^11", "npm-check-updates": "^18", - "pdfjs-dist": "^5", - "tinymce": "^8" + "pdfjs-dist": "~5 < 5.4.54 || ~5.4.55", + "tinymce": "^8", + "toastify-js": "^1" }, "devDependencies": { "@types/chai": "^5", "@types/js-beautify": "^1", "@types/mocha": "^10", "@types/node": "^24", + "@types/toastify-js": "^1.12.4", "@typescript-eslint/eslint-plugin": "^8", "@typescript-eslint/parser": "^8", "chai": "^5", @@ -1608,9 +1610,9 @@ "license": "MIT" }, "node_modules/@types/d3-dispatch": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.6.tgz", - "integrity": "sha512-4fvZhzMeeuBJYZXRXrRIQnvUYfyXwYmLsdiN7XXmVNQKKw1cM8a5WdID0g1hVFZDqT9ZqZEY5pD44p24VS7iZQ==", + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", + "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", "license": "MIT" }, "node_modules/@types/d3-drag": { @@ -1828,6 +1830,13 @@ "undici-types": "~7.8.0" } }, + "node_modules/@types/toastify-js": { + "version": "1.12.4", + "resolved": "https://registry.npmjs.org/@types/toastify-js/-/toastify-js-1.12.4.tgz", + "integrity": "sha512-zfZHU4tKffPCnZRe7pjv/eFKzTVHozKewFCKaCjZ4gFinKgJRz/t0bkZiMCXJxPhv/ZoeDGNOeRD09R0kQZ/nw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -2547,9 +2556,9 @@ } }, "node_modules/cytoscape": { - "version": "3.32.1", - "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.32.1.tgz", - "integrity": "sha512-dbeqFTLYEwlFg7UGtcZhCCG/2WayX72zK3Sq323CEX29CY81tYfVhw1MIdduCtpstB0cTOhJswWlM/OEB3Xp+Q==", + "version": "3.33.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.0.tgz", + "integrity": "sha512-2d2EwwhaxLWC8ahkH1PpQwCyu6EY3xDRdcEJXrLTb4fOUtVc+YWQalHU67rFS1a6ngj1fgv9dQLtJxP/KAFZEw==", "license": "MIT", "engines": { "node": ">=0.10" @@ -5985,9 +5994,9 @@ "license": "MIT" }, "node_modules/tinymce": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/tinymce/-/tinymce-8.0.0.tgz", - "integrity": "sha512-E1OwCXXCzmZLx6sQVeMHdb61Hsp+7AxWtYstXp7Yw59Et4AdHQ0N36n7InVaYDmq2aBlCM8qkTQYKEqKgecP3A==", + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/tinymce/-/tinymce-8.0.1.tgz", + "integrity": "sha512-KQ/+KaWmkIzSkNCYmqhXD2mftt+EEhz1bd1QCVopa2DNkoJ/rYFXhMnYGg1gVcRQa43xkmmv0Jj0ph+05VY0hQ==", "license": "GPL-2.0-or-later" }, "node_modules/to-regex-range": { @@ -6003,6 +6012,12 @@ "node": ">=8.0" } }, + "node_modules/toastify-js": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/toastify-js/-/toastify-js-1.12.0.tgz", + "integrity": "sha512-HeMHCO9yLPvP9k0apGSdPUWrUbLnxUKNFzgUoZp1PHCLploIX/4DSQ7V8H25ef+h4iO9n0he7ImfcndnN6nDrQ==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", diff --git a/client/package.json b/client/package.json index 0b46ed8f..c4fe53f7 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "codechat-editor-client", - "version": "0.1.24", + "version": "0.1.26", "description": "The CodeChat Editor Client, part of a web-based literate programming editor (the CodeChat Editor).", "homepage": "https://github.com/bjones1/CodeChat_Editor", "type": "module", @@ -17,6 +17,7 @@ "@types/js-beautify": "^1", "@types/mocha": "^10", "@types/node": "^24", + "@types/toastify-js": "^1.12.4", "@typescript-eslint/eslint-plugin": "^8", "@typescript-eslint/parser": "^8", "chai": "^5", @@ -41,14 +42,15 @@ "@codemirror/lang-rust": "^6", "@codemirror/lang-xml": "^6", "@codemirror/view": "^6", - "codemirror": "^6", - "graphviz-webcomponent": "^2", "@mathjax/mathjax-newcm-font": "4.0.0-rc.4", "@mathjax/src": "4.0.0-rc.4", + "codemirror": "^6", + "graphviz-webcomponent": "^2", "mermaid": "^11", "npm-check-updates": "^18", - "pdfjs-dist": "^5", - "tinymce": "^8" + "pdfjs-dist": "~5 < 5.4.54 || ~5.4.55", + "tinymce": "^8", + "toastify-js": "^1" }, "repository": { "type": "git", diff --git a/client/src/CodeChatEditor.mts b/client/src/CodeChatEditor.mts index 487b4437..1fe14c9a 100644 --- a/client/src/CodeChatEditor.mts +++ b/client/src/CodeChatEditor.mts @@ -56,10 +56,12 @@ import { CodeMirror_save, mathJaxTypeset, mathJaxUnTypeset, + scroll_to_line, + set_CodeMirror_positions, } from "./CodeMirror-integration.mjs"; import "./EditorComponents.mjs"; import "./graphviz-webcomponent-setup.mts"; -// This must be imported*after* the previous setup import, so it's placed here, +// This must be imported *after* the previous setup import, so it's placed here, // instead of in the third-party category above. import "graphviz-webcomponent"; import { tinymce, init, Editor } from "./tinymce-config.mjs"; @@ -70,6 +72,7 @@ import { UpdateMessageContents, CodeMirror, } from "./shared_types.mjs"; +import { show_toast } from "./show_toast.mjs"; // ### CSS import "./css/CodeChatEditor.css"; @@ -113,8 +116,13 @@ declare global { interface Window { CodeChatEditor: { // Called by the Client Framework. - open_lp: (code_chat_for_web: CodeChatForWeb) => Promise; + open_lp: ( + codechat_for_web: CodeChatForWeb, + cursor_position?: number, + ) => Promise; on_save: (_only_if_dirty: boolean) => Promise; + scroll_to_line: (line: number) => void; + show_toast: (text: string) => void; allow_navigation: boolean; }; CodeChatEditor_test: any; @@ -180,6 +188,8 @@ export const page_init = () => { window.CodeChatEditor = { open_lp, on_save, + scroll_to_line, + show_toast, allow_navigation: false, }; }); @@ -210,10 +220,17 @@ const is_doc_only = () => { }; // Wait for the DOM to load before opening the file. -const open_lp = async (code_chat_for_web: CodeChatForWeb) => - on_dom_content_loaded(() => _open_lp(code_chat_for_web)); - -// Store the HTML sent for CodeChat Editor documents. We can't simply use TinyMCE's [getContent](https://www.tiny.cloud/docs/tinymce/latest/apis/tinymce.editor/#getContent), since this modifies the content based on cleanup rules before returning it -- which causes applying diffs to this unexpectedly modified content to produce incorrect results. This text is the unmodified content sent from the IDE. +const open_lp = async ( + codechat_for_web: CodeChatForWeb, + cursor_position?: number, +) => on_dom_content_loaded(() => _open_lp(codechat_for_web, cursor_position)); + +// Store the HTML sent for CodeChat Editor documents. We can't simply use +// TinyMCE's +// [getContent](https://www.tiny.cloud/docs/tinymce/latest/apis/tinymce.editor/#getContent), +// since this modifies the content based on cleanup rules before returning it -- +// which causes applying diffs to this unexpectedly modified content to produce +// incorrect results. This text is the unmodified content sent from the IDE. let doc_content = ""; // This function is called on page load to "load" a file. Before this point, the @@ -223,7 +240,8 @@ let doc_content = ""; const _open_lp = async ( // A data structure provided by the server, containing the source and // associated metadata. See[`AllSource`](#AllSource). - code_chat_for_web: CodeChatForWeb, + codechat_for_web: CodeChatForWeb, + cursor_position?: number, ) => { // Use[URLSearchParams](https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams) // to parse out the search parameters of this window's URL. @@ -238,16 +256,18 @@ const _open_lp = async ( const editorMode = EditorMode[urlParams.get("mode") ?? "edit"]; // Get thecurrent_metadata from - // the provided `code_chat_for_web` struct and store it as a global variable. - current_metadata = code_chat_for_web["metadata"]; - const source = code_chat_for_web["source"]; + // the provided `code_chat_for_web` struct and store it as a global + // variable. + current_metadata = codechat_for_web["metadata"]; + const source = codechat_for_web["source"]; const codechat_body = document.getElementById( "CodeChat-body", ) as HTMLDivElement; // Disable autosave when updating the document. autosaveEnabled = false; clearAutosaveTimer(); - // Before calling any MathJax, make sure it's fully loaded and the initial render is finished. + // Before calling any MathJax, make sure it's fully loaded and the initial + // render is finished. await window.MathJax.startup.promise; // Per // the[docs](https://docs.mathjax.org/en/latest/web/typeset.html#updating-previously-typeset-content), @@ -299,7 +319,12 @@ const _open_lp = async ( } mathJaxTypeset(codechat_body); } else { - await CodeMirror_load(codechat_body, source, current_metadata.mode, []); + await CodeMirror_load( + codechat_body, + codechat_for_web, + [], + cursor_position, + ); } autosaveEnabled = true; @@ -312,45 +337,48 @@ const _open_lp = async ( } }; -const save_lp = () => { - /// @ts-expect-error - let code_mirror_diffable: CodeMirrorDiffable = {}; - if (is_doc_only()) { - // Untypeset all math before saving the document. - const codechat_body = document.getElementById( - "CodeChat-body", - ) as HTMLDivElement; - mathJaxUnTypeset(codechat_body); - // To save a document only, simply get the HTML from the only Tiny MCE - // div. - tinymce.activeEditor!.save(); - const html = tinymce.activeEditor!.getContent(); - ( - code_mirror_diffable as { - Plain: CodeMirror; - } - ).Plain = { - doc: turndownService.turndown(html), - doc_blocks: [], - }; - // Retypeset all math after saving the document. - mathJaxTypeset(codechat_body); - } else { - code_mirror_diffable = CodeMirror_save(); - assert("Plain" in code_mirror_diffable); - codechat_html_to_markdown(code_mirror_diffable.Plain.doc_blocks); - } - +const save_lp = (is_dirty: boolean) => { let update: UpdateMessageContents = { // The Framework will fill in this value. file_path: "", - contents: { + }; + set_CodeMirror_positions(update); + + // Add the contents only if the document is dirty. + if (is_dirty) { + /// @ts-expect-error + let code_mirror_diffable: CodeMirrorDiffable = {}; + if (is_doc_only()) { + // Untypeset all math before saving the document. + const codechat_body = document.getElementById( + "CodeChat-body", + ) as HTMLDivElement; + mathJaxUnTypeset(codechat_body); + // To save a document only, simply get the HTML from the only Tiny MCE + // div. + tinymce.activeEditor!.save(); + const html = tinymce.activeEditor!.getContent(); + ( + code_mirror_diffable as { + Plain: CodeMirror; + } + ).Plain = { + doc: turndownService.turndown(html), + doc_blocks: [], + }; + // Retypeset all math after saving the document. + mathJaxTypeset(codechat_body); + } else { + code_mirror_diffable = CodeMirror_save(); + assert("Plain" in code_mirror_diffable); + codechat_html_to_markdown(code_mirror_diffable.Plain.doc_blocks); + } + update.contents = { metadata: current_metadata, source: code_mirror_diffable, - }, - scroll_position: null, - cursor_position: null, - }; + }; + } + return update; }; @@ -373,7 +401,9 @@ const on_save = async (only_if_dirty: boolean = false) => { const webSocketComm = parent.window.CodeChatEditorFramework.webSocketComm; console.log("Sent Update - saving document."); await new Promise(async (resolve) => { - webSocketComm.send_message({ Update: save_lp() }, () => resolve(0)); + webSocketComm.send_message({ Update: save_lp(is_dirty) }, () => + resolve(0), + ); }); is_dirty = false; }; diff --git a/client/src/CodeChatEditorFramework.mts b/client/src/CodeChatEditorFramework.mts index 8f1a8343..1af8e044 100644 --- a/client/src/CodeChatEditorFramework.mts +++ b/client/src/CodeChatEditorFramework.mts @@ -30,6 +30,7 @@ // // #### Third-party import ReconnectingWebSocket from "./third-party/ReconnectingWebSocket.cjs"; +import { show_toast as show_toast_core } from "./show_toast.mjs"; // #### Local import { assert } from "./assert.mjs"; @@ -58,7 +59,7 @@ let webSocketComm: WebSocketComm; class WebSocketComm { // Use a unique ID for each websocket message sent. See the Implementation // section on Message IDs for more information. - ws_id = -9007199254740990; + ws_id = -9007199254740988; // The websocket used by this class. Really a `ReconnectingWebSocket`, but // that's not a type. ws: WebSocket; @@ -91,7 +92,9 @@ class WebSocketComm { // Provide logging to help track down errors. this.ws.onerror = (event: any) => { - console.error(`CodeChat Client: websocket error ${event}.`); + report_error( + `CodeChat Client: websocket error ${JSON.stringify(event)}.`, + ); }; this.ws.onclose = (event: any) => { @@ -109,10 +112,10 @@ class WebSocketComm { console.log( `Received data id = ${id}, message = ${JSON.stringify(message).substring(0, MAX_MESSAGE_LENGTH)}`, ); - console.assert(id !== undefined); - console.assert(message !== undefined); + assert(id !== undefined); + assert(message !== undefined); const keys = Object.keys(message); - console.assert(keys.length === 1); + assert(keys.length === 1); const key = keys[0]; const value = Object.values(message)[0]; @@ -128,30 +131,36 @@ class WebSocketComm { current_update.file_path !== this.current_filename ) { const msg = `Ignoring update for ${current_update.file_path} because it's not the current file ${this.current_filename}.`; - console.log(msg); + report_error(msg); this.send_result(id, msg); break; } - let result = null; const contents = current_update.contents; - if (contents !== null && contents !== undefined) { + const cursor_position = current_update.cursor_position; + if (contents !== undefined) { // If the page is still loading, wait until the load // completed before updating the editable contents. if (this.onloading) { root_iframe!.onload = () => { - set_content(contents); + set_content( + contents, + current_update.cursor_position, + ); this.onloading = false; }; } else { - set_content(contents); + set_content( + contents, + current_update.cursor_position, + ); } - } else { - // TODO: handle scroll/cursor updates. - result = `Unhandled Update message: ${current_update}`; - console.log(result); + } else if (cursor_position !== undefined) { + root_iframe!.contentWindow!.CodeChatEditor.scroll_to_line( + cursor_position, + ); } - this.send_result(id, result); + this.send_result(id, null); break; case "CurrentFile": @@ -205,17 +214,16 @@ class WebSocketComm { // Report if this was an error. const result_contents = value as MessageResult; if ("Err" in result_contents) { - console.log( + report_error( `Error in message ${id}: ${result_contents.Err}.`, ); } break; default: - console.log( - `Received unhandled message ${key}(${JSON.stringify(value).substring(0, MAX_MESSAGE_LENGTH)})`, - ); - this.send_result(id, `Unhandled message ${key}(${value})`); + const msg = `Received unhandled message ${key}(${JSON.stringify(value).substring(0, MAX_MESSAGE_LENGTH)})`; + report_error(msg); + this.send_result(id, msg); break; } }; @@ -239,7 +247,7 @@ class WebSocketComm { // Report an error from the server. report_server_timeout = (message_id: number) => { delete this.pending_messages[message_id]; - console.log(`Error: server timeout for message id ${message_id}`); + report_error(`Error: server timeout for message id ${message_id}`); }; // Send a message expecting a result to the server. @@ -251,7 +259,7 @@ class WebSocketComm { this.ws_id += 3; // Add in the current filename to the message, if it's an `Update`. if (typeof message == "object" && "Update" in message) { - console.assert(this.current_filename !== undefined); + assert(this.current_filename !== undefined); message.Update.file_path = this.current_filename!; } console.log( @@ -310,7 +318,7 @@ const get_client = () => root_iframe?.contentWindow?.CodeChatEditor; // Assign content to either the Client (if it's loaded) or the webpage (if not) // in the `root_iframe`. -const set_content = (contents: CodeChatForWeb) => { +const set_content = (contents: CodeChatForWeb, cursor_position?: number) => { let client = get_client(); if (client === undefined) { // See if this is the [simple viewer](#Client-simple-viewer). Otherwise, @@ -326,7 +334,10 @@ const set_content = (contents: CodeChatForWeb) => { cw.document.write(contents.source.Plain.doc); cw.document.close(); } else { - root_iframe!.contentWindow!.CodeChatEditor.open_lp(contents); + root_iframe!.contentWindow!.CodeChatEditor.open_lp( + contents, + cursor_position, + ); } }; @@ -385,3 +396,16 @@ declare global { CodeChatEditor_test: any; } } + +const show_toast = (text: string) => { + if (get_client() === undefined) { + show_toast_core(text); + } else { + root_iframe!.contentWindow!.CodeChatEditor.show_toast(text); + } +}; + +const report_error = (text: string) => { + console.error(text); + show_toast(text); +}; diff --git a/client/src/CodeMirror-integration.mts b/client/src/CodeMirror-integration.mts index 9b813eb9..cbf8a974 100644 --- a/client/src/CodeMirror-integration.mts +++ b/client/src/CodeMirror-integration.mts @@ -87,8 +87,10 @@ import { CodeMirrorDiffable, CodeMirrorDocBlockTuple, StringDiff, + UpdateMessageContents, } from "./shared_types.mjs"; import { assert } from "./assert.mjs"; +import { show_toast } from "./show_toast.mjs"; // Globals // ------- @@ -97,6 +99,9 @@ let tinymce_singleton: Editor | undefined; // When true, don't update on the next call to `on_dirty`. See that function for // more info. let ignore_next_dirty = false; +// True to ignore the next text selection change, since updates to the +// cursor or scroll position from the Client trigged this change. +let ignore_selection_change = false; // Options used when creating a `Decoration`. const decorationOptions = { @@ -154,6 +159,15 @@ export const docBlockField = StateField.define({ // a doc block, as requested by this effect. for (let effect of tr.effects) if (effect.is(addDocBlock)) { + // Check that we're not overwriting text. + const newlines = current_view.state.doc + .slice(effect.value.from, effect.value.to) + .toString(); + if (newlines !== "\n".repeat(newlines.length)) { + report_error(`Attempt to overwrite text: "${newlines}".`); + window.close(); + assert(false); + } // Perform an // [update](https://codemirror.net/docs/ref/#state.RangeSet.update) // by adding the requested doc block. @@ -203,10 +217,11 @@ export const docBlockField = StateField.define({ // doc block. if (prev !== undefined) { console.error({ doc_blocks, effect }); - assert( - false, + report_error( "More than one doc block at one location found.", ); + window.close(); + assert(false); } prev = value; to = to_found; @@ -220,7 +235,21 @@ export const docBlockField = StateField.define({ ); if (prev === undefined) { console.error({ doc_blocks, effect }); - assert(false, "No doc block found."); + report_error("No doc block found."); + window.close(); + assert(false); + } + // Determine the final from/to values. + to = effect.value.to ?? to; + const from = effect.value.from_new ?? effect.value.from; + // Check that we're not overwriting text. + const newlines = current_view.state.doc + .slice(from, to) + .toString(); + if (newlines !== "\n".repeat(newlines.length)) { + report_error(`Attempt to overwrite text: "${newlines}".`); + window.close(); + assert(false); } doc_blocks = doc_blocks.update({ // Remove the old doc block. We assume there's only one @@ -234,7 +263,8 @@ export const docBlockField = StateField.define({ Decoration.replace({ widget: new DocBlockWidget( effect.value.indent ?? prev.spec.widget.indent, - effect.value.delimiter, + effect.value.delimiter ?? + prev.spec.widget.delimiter, typeof effect.value.contents === "string" ? effect.value.contents : apply_diff_str( @@ -244,10 +274,7 @@ export const docBlockField = StateField.define({ effect.value.dom ?? prev.spec.widget.dom, ), ...decorationOptions, - }).range( - effect.value.from_new ?? effect.value.from, - effect.value.to ?? to, - ), + }).range(from, to), ], }); } else if (effect.is(deleteDocBlock)) { @@ -338,7 +365,7 @@ type updateDocBlockType = { from_new?: number; to?: number; indent?: string; - delimiter: string; + delimiter?: string; contents: string | StringDiff[]; dom?: HTMLDivElement; }; @@ -493,7 +520,7 @@ export const mathJaxTypeset = async ( // internal MathJax promises. window.MathJax.whenReady(afterTypesetFunc); } catch (err: any) { - console.log("Typeset failed: " + err.message); + report_error(`Typeset failed: ${err.message}`); } }; @@ -831,6 +858,13 @@ const autosaveExtension = EditorView.updateListener.of( if (isChanged) { set_is_dirty(); startAutosaveTimer(); + } else if (v.selectionSet) { + if (ignore_selection_change) { + ignore_selection_change = false; + return; + } + // Send an update if only the selection changed. + startAutosaveTimer(); } }, ); @@ -841,13 +875,12 @@ export const CodeMirror_load = async ( // The div to place the loaded document in. codechat_body: HTMLDivElement, // The document to load. - source: CodeChatForWeb["source"], - // The name of the lexer to use. - lexer_name: string, + codechat_for_web: CodeChatForWeb, // Additional extensions. extensions: Array, + cursor_position?: number, ) => { - if ("Plain" in source) { + if ("Plain" in codechat_for_web.source) { // Although the // [docs](https://codemirror.net/docs/ref/#state.EditorState^fromJSON) // specify a @@ -855,9 +888,9 @@ export const CodeMirror_load = async ( // which contains `doc` and `selection`, the implementation requires // these to be present in the `json` (first) argument. Therefore: const editor_state_json = { - doc: source.Plain.doc, + doc: codechat_for_web.source.Plain.doc, selection: EditorSelection.single(0).toJSON(), - doc_blocks: source.Plain.doc_blocks, + doc_blocks: codechat_for_web.source.Plain.doc_blocks, }; // Save the current scroll position, to prevent the view from scrolling // back to the top after an update/reload. @@ -873,7 +906,7 @@ export const CodeMirror_load = async ( '
'; let parser; // TODO: dynamically load the parser. - switch (lexer_name) { + switch (codechat_for_web.metadata.mode) { // Languages with a parser case "sh": parser = cpp(); @@ -937,7 +970,9 @@ export const CodeMirror_load = async ( default: parser = javascript(); - console.log(`Unknown lexer name ${lexer_name}`); + report_error( + `Unknown lexer name ${codechat_for_web.metadata.mode}`, + ); break; } const state = EditorState.fromJSON( @@ -1006,7 +1041,7 @@ export const CodeMirror_load = async ( // blocks aren't changed; without this, the diff won't work (since // from/to values of doc blocks are changed by unfrozen text edits). current_view.dispatch({ - changes: source.Diff.doc, + changes: codechat_for_web.source.Diff.doc, annotations: docBlockFreezeAnnotation.of(true), }); // Now, apply the diff in a separate transaction. Applying them in the @@ -1014,7 +1049,7 @@ export const CodeMirror_load = async ( // the doc block effects, even when changes to the doc block state is // frozen. const stateEffects: StateEffect[] = []; - for (const transaction of source.Diff.doc_blocks) { + for (const transaction of codechat_for_web.source.Diff.doc_blocks) { if ("Add" in transaction) { const add = transaction.Add; stateEffects.push( @@ -1037,6 +1072,19 @@ export const CodeMirror_load = async ( // Update the view with these changes to the state. current_view.dispatch({ effects: stateEffects }); } + // If provided, scroll the cursor position into view. + if (cursor_position !== undefined) { + scroll_to_line(cursor_position); + } +}; + +export const scroll_to_line = (line: number) => { + ignore_selection_change = true; + const line_range = current_view.state.doc.line(line); + current_view.dispatch({ + selection: EditorSelection.cursor(line_range.from), + scrollIntoView: true, + }); }; // Apply a `StringDiff` to the before string to produce the after string. @@ -1068,3 +1116,17 @@ export const CodeMirror_save = (): CodeMirrorDiffable => { return { Plain: code_mirror }; }; + +export const set_CodeMirror_positions = ( + update_message_contents: UpdateMessageContents, +) => { + update_message_contents.cursor_position = current_view.state.doc.lineAt( + current_view.state.selection.main.from, + ).number; + update_message_contents.scroll_position = current_view.viewport.from; +}; + +const report_error = (text: string) => { + console.error(text); + show_toast(text); +}; diff --git a/client/src/show_toast.mts b/client/src/show_toast.mts new file mode 100644 index 00000000..0d859382 --- /dev/null +++ b/client/src/show_toast.mts @@ -0,0 +1,34 @@ +// Copyright (C) 2025 Bryan A. Jones. +// +// This file is part of the CodeChat Editor. The CodeChat Editor is free +// software: you can redistribute it and/or modify it under the terms of the GNU +// General Public License as published by the Free Software Foundation, either +// version 3 of the License, or (at your option) any later version. +// +// The CodeChat Editor is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +// details. +// +// You should have received a copy of the GNU General Public License along with +// the CodeChat Editor. If not, see +// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). +// +// `show_toast.mts` -- Show a toast message +// ======================================== +import Toastify from "toastify-js"; +import "toastify-js/src/toastify.css"; + +export const show_toast = (text: string) => + Toastify({ + text, + duration: 3000, + newWindow: true, + close: true, + gravity: "top", + position: "right", + stopOnFocus: true, + style: { + background: "linear-gradient(to right, #b00049ff, #e76d8bff)", + }, + }).showToast(); diff --git a/docs/changelog.md b/docs/changelog.md index b0ab2075..5f727391 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -20,6 +20,15 @@ Changelog ========= * [Github master](https://github.com/bjones1/CodeChat_Editor): + * Additional data corruption fixes when applying edits. + * Add more checks to detect data corruption. + * Update the file watcher to support the diff protocol. + * Send only changed fields when using the diff protocol. + * Provide basic synchronization between the IDE and Client. +* Β v0.1.25, 2025-29-Jul: + * Show notifications in Client when errors occur. + * For safety, close the Client if applying edits fails. +* v0.1.24, 2025-Jul-25:Β  * Fix indexing in diffs for characters that use more than one UTF-16 code unit, such as πŸ˜„,πŸ‘‰πŸΏ,πŸ‘¨β€πŸ‘¦, and πŸ‡ΊπŸ‡³. * Fix data corruption with adjacent doc blocks. diff --git a/docs/implementation.md b/docs/implementation.md index 64b51ad0..bd0a4067 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -159,129 +159,6 @@ Editor for short) exchange messages with each other, mediated by the CodeChat Server. The Server forwards messages from one client to the other, translating as necessary (for example, between source code and the Editor format). -### Editor-overlay filesystem - -When the Client displays a file provided by the IDE, that file may not exist in -the filesystem (a newly-created document), the IDE's content may be newer than -the filesystem content (an unsaved file), or the file may exist only in the -filesystem (for examples, images referenced by a file). The Client loads files -by sending HTTP requests to the Server with a URL which includes the path to the -desired file. Therefore, the Server must first ask the IDE if it has the -requested file; if so, it must deliver the IDE's file contents; if not, it must -load thee requested file from the filesystem. This process -- fetching from the -IDE if possible, then falling back to the filesystem -- defines the -editor-overlay filesystem. - -#### Network interfaces between the Client, Server, and IDE - -* The startup phase loads the Client framework into a browser: - - - sequenceDiagram - participant IDE - participant Server - participant Client - note over IDE, Client: Startup - IDE ->> Server: Opened(IdeType) - Server ->> IDE: Result(String: OK) - Server ->> IDE: ClientHtml(String: HTML or URL) - IDE ->> Server: Result(String: OK) - note over IDE, Client: Open browser (Client framework HTML or URL) - loop - Client -> Server: HTTP request(/static URL) - Server -> Client: HTTP response(/static data) - end - - -* If the current file in the IDE changes (including the initial startup, when - the change is from no file to the current file), or a link is followed in - the Client's iframe: - - - sequenceDiagram - participant IDE - participant Server - participant Client - alt IDE loads file - IDE ->> Client: CurrentFile(String: Path of main.py) - opt If Client document is dirty - Client ->> IDE: Update(String: contents of main.py) - IDE ->> Client: Response(OK) - end - Client ->> IDE: Response(OK) - else Client loads file - Client ->> IDE: CurrentFile(String: URL of main.py) - IDE ->> Client: Response(OK) - end - Client ->> Server: HTTP request(URL of main.py) - Server ->> IDE: LoadFile(String: path to main.py) - IDE ->> Server: Response(LoadFile(String: file contents of main.py)) - alt main.py is editable - Server ->> Client: HTTP response(contents of Client) - Server ->> Client: Update(String: contents of main.py) - Client ->> Server: Response(OK) - loop - Client ->> Server: HTTP request(URL of supporting file in main.py) - Server ->> IDE: LoadFile(String: path of supporting file) - alt Supporting file in IDE - IDE ->> Server: Response(LoadFile(contents of supporting file) - Server ->> Client: HTTP response(contents of supporting file) - else Supporting file not in IDE - IDE ->> Server: Response(LoadFile(None)) - Server ->> Client: HTTP response(contents of supporting file from filesystem) - end - end - else main.py not editable and not a project - Server ->> Client: HTTP response(contents of main.py) - else main.py not editable and is a project - Server ->> Client: HTTP response(contents of Client Simple Viewer) - Client ->> Server: HTTP request (URL?raw of main.py) - Server ->> Client: HTTP response(contents of main.py) - end - - -* If the current file's contents in the IDE are edited: - - - sequenceDiagram - participant IDE - participant Server - participant Client - IDE ->> Server: Update(String: new text contents) - alt Main file is editable - Server ->> Client: Update(String: new Client contents) - else Main file is not editable - Server ->> Client: Update(String: new text contents) - end - Client ->> IDE: Response(String: OK)
- - -* If the current file's contents in the Client are edited, the Client sends - the IDE an `Update` with the revised contents. - -* When the PC goes to sleep then wakes up, the IDE client and the Editor - client both reconnect to the websocket URL containing their assigned ID. - -* If the Editor client or the IDE client are closed, they close their - websocket, which sends a `Close` message to the other websocket, causes it - to also close and ending the session. - -* If the server is stopped (or crashes), both clients shut down after several - reconnect retries. - -#### Message IDs - -The message system connects the IDE, Server, and Client; all three can serve as -the source or destination for a message. Any message sent should produce a -Response message in return. Therefore, we need globally unique IDs for each -message. To achieve this, the Server uses IDs that are multiples of 3 (0, 3, 6, -...), the Client multiples of 3 + 1 (1, 4, 7, ...) and the IDE multiples of 3 + -2 (2, 5, 8, ...). A double-precision floating point number (the standard -[numeric -type](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#number_type) -in JavaScript) has a 53-bit mantissa, meaning IDs won't wrap around for a very -long time. - #### Architecture **Reviewed to here** @@ -310,53 +187,7 @@ To decouple these low-level websocket details from high-level processing (such as translating between source code and its web equivalent), the websocket tasks enqueue all high-level messages to the processing task; they listen to any enqueued messages in the client or ide queue, passing these on via the websocket -connection. The following diagram illustrates this approach: - - - -The queues use multiple-sender, single receiver (mpsc) types; hence, a single -task in the diagram receives data from a queue, while multiple tasks send data -to a queue. When the IDE processing task writes updated content to the file -being edited, it notifies the file watcher to ignore the next file update (hence -the dotted arrow). - -The exception to this pattern is the HTTP endpoint. This endpoint is invoked -with each HTTP request, rather than operating as a single, long-running task. It -sends the request to the processing task using an mpsc queue; this request -includes a one-shot channel which enables the request to return a response to -this specific request instance. The endpoint then returns the provided response. +connection. Simplest non-IDE integration: the file watcher. @@ -384,7 +215,7 @@ directly through a native interface. For example, [NAPI-RS](https://napi.rs/) provide a way to call Rust from Node.js; [JNI](https://docs.rs/jni/latest/jni/) allows calling Rust from Java. -### Efficent websocket communication +### Efficient websocket communication When an edit occurs, it's best to send only changed data, rather than the whole file. The diff crate provides easy access to determining a diff. The idea: @@ -715,7 +546,4 @@ approach](https://auth0.com/blog/build-an-api-in-rust-with-jwt-authentication-us A better approach to make macros accessible where they're defined, instead of at the crate root: see -[SO](https://stackoverflow.com/questions/26731243/how-do-i-use-a-macro-across-module-files/67140319#67140319). - -When using VSCode with Rust, set `"rust-analyzer.cargo.targetDir": true`. See -[this issue](https://github.com/rust-lang/rust-analyzer/issues/17807). \ No newline at end of file +[SO](https://stackoverflow.com/questions/26731243/how-do-i-use-a-macro-across-module-files/67140319#67140319). \ No newline at end of file diff --git a/extensions/VSCode/package-lock.json b/extensions/VSCode/package-lock.json index 8294d79d..7a09f2c6 100644 --- a/extensions/VSCode/package-lock.json +++ b/extensions/VSCode/package-lock.json @@ -1,12 +1,12 @@ { "name": "codechat-editor-client", - "version": "0.1.24", + "version": "0.1.26", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codechat-editor-client", - "version": "0.1.24", + "version": "0.1.26", "license": "GPL-3.0-only", "dependencies": { "escape-html": "^1", @@ -182,9 +182,9 @@ } }, "node_modules/@azure/msal-browser": { - "version": "4.16.0", - "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.16.0.tgz", - "integrity": "sha512-yF8gqyq7tVnYftnrWaNaxWpqhGQXoXpDfwBtL7UCGlIbDMQ1PUJF/T2xCL6NyDNHoO70qp1xU8GjjYTyNIefkw==", + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-4.18.0.tgz", + "integrity": "sha512-esQwdtHHVkFJhcKWnysnCTchiKsy3dmNZGs8AckD9PO3t8Lp5VtY0xcrbCBC0JbttG/5w2/xukUQOsMpoUFKrg==", "dev": true, "license": "MIT", "dependencies": { @@ -3698,9 +3698,9 @@ "license": "ISC" }, "node_modules/follow-redirects": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", - "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", "dev": true, "funding": [ { @@ -7307,9 +7307,9 @@ "license": "MIT" }, "node_modules/undici": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.12.0.tgz", - "integrity": "sha512-GrKEsc3ughskmGA9jevVlIOPMiiAHJ4OFUtaAH+NhfTUSiZ1wMPIQqQvAJUrJspFXJt3EBWgpAeoHEDVT1IBug==", + "version": "7.13.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.13.0.tgz", + "integrity": "sha512-l+zSMssRqrzDcb3fjMkjjLGmuiiK2pMIcV++mJaAc9vhjSGpvM7h43QgP+OAMb1GImHmbPyG2tBXeuyG5iY4gA==", "dev": true, "license": "MIT", "engines": { diff --git a/extensions/VSCode/package.json b/extensions/VSCode/package.json index fe854993..9fbccb5e 100644 --- a/extensions/VSCode/package.json +++ b/extensions/VSCode/package.json @@ -1,6 +1,6 @@ { "name": "codechat-editor-client", - "version": "0.1.24", + "version": "0.1.26", "publisher": "CodeChat", "engines": { "vscode": "^1.61.0" diff --git a/extensions/VSCode/src/extension.ts b/extensions/VSCode/src/extension.ts index 268269ff..a756cbf7 100644 --- a/extensions/VSCode/src/extension.ts +++ b/extensions/VSCode/src/extension.ts @@ -30,11 +30,19 @@ import process from "node:process"; // ### Third-party packages import escape from "escape-html"; -import vscode, { commands, Position, Range } from "vscode"; +import vscode, { + commands, + Position, + Range, + TextDocument, + TextEditor, +} from "vscode"; import { WebSocket } from "ws"; // ### Local packages import { + CodeChatForWeb, + CodeMirror, EditorMessage, EditorMessageContents, MessageResult, @@ -94,8 +102,14 @@ let ignore_text_document_change = false; // True to ignore the next active editor change event, since a `CurrentFile` // message from the Client caused this change. let ignore_active_editor_change = false; +// True to ignore the next text selection change, since updates to the +// cursor or scroll position from the Client trigged this change. +let ignore_selection_change = false; // True to not report the next error. let quiet_next_error = false; +// True if the editor contents have changed (are dirty) from the perspective +// of the CodeChat Editor (not if the contents are saved to disk). +let is_dirty = false; // Activation/deactivation // ----------------------- @@ -130,7 +144,7 @@ export const activate = (context: vscode.ExtensionContext) => { event.reason }, ${format_struct(event.contentChanges)}.`, ); - start_render(); + send_update(true); }), ); @@ -144,6 +158,18 @@ export const activate = (context: vscode.ExtensionContext) => { current_file(); }), ); + + context.subscriptions.push( + vscode.window.onDidChangeTextEditorSelection( + (_event) => { + if (ignore_selection_change) { + ignore_selection_change = false; + return; + } + send_update(false); + }, + ), + ); } // Get the CodeChat Client's location from the VSCode @@ -233,7 +259,7 @@ export const activate = (context: vscode.ExtensionContext) => { // Only render if the webview was activated; // this event also occurs when it's deactivated. if (webview_panel?.active) { - start_render(); + send_update(true); } }, ); @@ -276,9 +302,6 @@ export const activate = (context: vscode.ExtensionContext) => { let was_error: boolean = false; websocket.on("error", (err: ErrorEvent) => { - console.error( - `CodeChat Editor extension: error in Server connection: ${err.message}`, - ); was_error = true; show_error( `Error communicating with the CodeChat Editor Server: ${err.message}. Re-run the CodeChat Editor extension to restart it.`, @@ -338,7 +361,9 @@ export const activate = (context: vscode.ExtensionContext) => { data.toString(), ) as EditorMessage; console.log( - `CodeChat Editor extension: Received data id = ${id}, message = ${format_struct(message)}.`, + `CodeChat Editor extension: Received data id = ${id}, message = ${format_struct( + message, + )}.`, ); assert(id !== undefined); assert(message !== undefined); @@ -361,7 +386,7 @@ export const activate = (context: vscode.ExtensionContext) => { }); break; } - if (current_update.contents !== null) { + if (current_update.contents !== undefined) { const source = current_update.contents.source; // Is this plain text, or a diff? This will @@ -393,13 +418,21 @@ export const activate = (context: vscode.ExtensionContext) => { // beginning of the document to a // `Position` (line, then offset on that // line) needed by VSCode. - const from = doc.positionAt(diff.from); + const from = doc.positionAt( + diff.from, + ); if (diff.to === undefined) { // This is an insert. - wse.insert(doc.uri, from, diff.insert); + wse.insert( + doc.uri, + from, + diff.insert, + ); } else { // This is a replace or delete. - const to = doc.positionAt(diff.to); + const to = doc.positionAt( + diff.to, + ); wse.replace( doc.uri, new Range(from, to), @@ -408,11 +441,39 @@ export const activate = (context: vscode.ExtensionContext) => { } } } - vscode.workspace.applyEdit(wse).then(() => ignore_text_document_change = false); - } else { - // TODO: handle cursor/scroll position - // updates. - assert(false); + vscode.workspace + .applyEdit(wse) + .then( + () => + (ignore_text_document_change = false), + ); + } + // Update the cursor position if provided. + let line = current_update.cursor_position; + if (line !== undefined) { + const editor = get_text_editor(doc); + if (editor) { + ignore_selection_change = true; + // The VSCode line is zero-based; the CodeMirror line is one-based. + line -= 1; + console.log(`Moving to line ${line}.`); + const position = new vscode.Position( + line, + line, + ); + editor.selections = [ + new vscode.Selection( + position, + position, + ), + ]; + editor.revealRange( + new vscode.Range( + position, + position, + ), + ); + } } send_result(id); break; @@ -488,13 +549,9 @@ export const activate = (context: vscode.ExtensionContext) => { // Report if this was an error. const result_contents = value as MessageResult; if ("Err" in result_contents) { - const msg = `Error in message ${id}: ${result_contents.Err}`; - console.error(msg); - // Warning: Calling `show_error` shuts down - // the client. Do this deliberately, since - // timeouts (missed messages) can cause data - // corruption. - show_error(msg); + show_error( + `Error in message ${id}: ${result_contents.Err}`, + ); } break; } @@ -527,7 +584,9 @@ export const activate = (context: vscode.ExtensionContext) => { default: console.error( - `Unhandled message ${key}(${format_struct(value)}`, + `Unhandled message ${key}(${format_struct( + value, + )}`, ); break; } @@ -616,7 +675,8 @@ const send_result = (id: number, result: MessageResult = { Ok: "Void" }) => { // This is called after an event such as an edit, or when the CodeChat panel // becomes visible. Wait a bit in case any other events occur, then request a // render. -const start_render = () => { +const send_update = (this_is_dirty: boolean) => { + is_dirty ||= this_is_dirty; if (can_render()) { // Render after some inactivity: cancel any existing timer, then ... if (idle_timer !== undefined) { @@ -626,21 +686,29 @@ const start_render = () => { idle_timer = setTimeout(() => { if (can_render()) { const ate = vscode.window.activeTextEditor!; - send_message({ - Update: { - file_path: ate.document.fileName, - contents: { - metadata: { mode: "" }, - source: { - Plain: { - doc: ate.document.getText(), - doc_blocks: [], - }, + // The [Position](https://code.visualstudio.com/api/references/vscode-api#Position) + // encodes the line as a zero-based value. In contrast, CodeMirror [Text.line](https://codemirror.net/docs/ref/#state.Text.line) + // is 1-based. + const current_line = ate.selection.active.line + 1; + const Update: UpdateMessageContents = { + file_path: ate.document.fileName, + cursor_position: current_line, + }; + // Send contents only if necessary. + if (is_dirty) { + Update.contents = { + metadata: { mode: "" }, + source: { + Plain: { + doc: ate.document.getText(), + doc_blocks: [], }, }, - cursor_position: null, - scroll_position: null, - }, + }; + is_dirty = false; + } + send_message({ + Update, }); } }, 300); @@ -693,6 +761,7 @@ const show_error = (message: string) => { quiet_next_error = false; return; } + console.error(`CodeChat Editor extension: ${message}`); if (webview_panel !== undefined) { // If the panel was displaying other content, reset it for errors. if ( @@ -750,6 +819,12 @@ const get_document = (file_path: string) => { return undefined; }; +const get_text_editor = (doc: TextDocument): TextEditor | undefined => { + for (const editor of vscode.window.visibleTextEditors) { + if (editor.document === doc) return editor; + } +}; + const get_port = (): number => { const port = vscode.workspace .getConfiguration("CodeChatEditor.Server") diff --git a/server/Cargo.lock b/server/Cargo.lock index 94d13f22..c54400d9 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -129,7 +129,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2", + "socket2 0.5.10", "tokio", "tracing", ] @@ -191,7 +191,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.5.10", "time", "tracing", "url", @@ -551,9 +551,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882" dependencies = [ "clap_builder", "clap_derive", @@ -561,9 +561,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966" dependencies = [ "anstream", "anstyle", @@ -591,7 +591,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "codechat-editor-server" -version = "0.1.24" +version = "0.1.26" dependencies = [ "actix-files", "actix-http", @@ -2035,9 +2035,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.15" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ "bitflags 2.9.1", ] @@ -2079,9 +2079,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rustc-demangle" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] name = "rustc_version" @@ -2270,6 +2270,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2468,9 +2478,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" dependencies = [ "backtrace", "bytes", @@ -2481,9 +2491,9 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2517,7 +2527,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.9.2", - "socket2", + "socket2 0.5.10", "tokio", "tokio-util", "whoami", @@ -3019,7 +3029,7 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.2", + "windows-targets 0.53.3", ] [[package]] @@ -3055,10 +3065,11 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.2" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ + "windows-link", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", diff --git a/server/Cargo.toml b/server/Cargo.toml index e74e4a38..4000ed24 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -31,7 +31,7 @@ license = "GPL-3.0-only" name = "codechat-editor-server" readme = "../README.md" repository = "https://github.com/bjones1/CodeChat_Editor" -version = "0.1.24" +version = "0.1.26" # This library allows other packages to use core CodeChat Editor features. [lib] diff --git a/server/src/ide.rs b/server/src/ide.rs new file mode 100644 index 00000000..9a1bba68 --- /dev/null +++ b/server/src/ide.rs @@ -0,0 +1,17 @@ +// Copyright (C) 2025 Bryan A. Jones. +// +// This file is part of the CodeChat Editor. The CodeChat Editor is free +// software: you can redistribute it and/or modify it under the terms of the GNU +// General Public License as published by the Free Software Foundation, either +// version 3 of the License, or (at your option) any later version. +// +// The CodeChat Editor is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +// details. +// +// You should have received a copy of the GNU General Public License along with +// the CodeChat Editor. If not, see +// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). +pub mod filewatcher; +pub mod vscode; diff --git a/server/src/ide/filewatcher.rs b/server/src/ide/filewatcher.rs new file mode 100644 index 00000000..6e111863 --- /dev/null +++ b/server/src/ide/filewatcher.rs @@ -0,0 +1,1113 @@ +// Copyright (C) 2025 Bryan A. Jones. +// +// This file is part of the CodeChat Editor. The CodeChat Editor is free +// software: you can redistribute it and/or modify it under the terms of the GNU +// General Public License as published by the Free Software Foundation, either +// version 3 of the License, or (at your option) any later version. +// +// The CodeChat Editor is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +// details. +// +// You should have received a copy of the GNU General Public License along with +// the CodeChat Editor. If not, see +// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). +/// `filewatcher.rs` -- Implement the File Watcher "IDE" +/// ==================================================== +// Imports +// ------- +// +// ### Standard library +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; + +// ### Third-party +use actix_web::{ + HttpRequest, HttpResponse, Responder, + error::{self, Error}, + get, + http::header::{self, ContentType}, + web, +}; +use dunce::simplified; +use indoc::formatdoc; +use lazy_static::lazy_static; +use log::{error, info, warn}; +use notify_debouncer_full::{ + DebounceEventResult, new_debouncer, + notify::{EventKind, RecursiveMode}, +}; +use regex::Regex; +use tokio::{ + fs::DirEntry, + fs::{self, File}, + io::AsyncReadExt, + select, + sync::mpsc, +}; +use urlencoding; +#[cfg(target_os = "windows")] +use win_partitions::win_api::get_logical_drive; + +// ### Local +use crate::{ + processing::CodeMirrorDiffable, + queue_send, + webserver::{ + INITIAL_IDE_MESSAGE_ID, MESSAGE_ID_INCREMENT, ResultOkTypes, filesystem_endpoint, + get_test_mode, url_to_path, + }, +}; +use crate::{ + processing::{CodeChatForWeb, CodeMirror, SourceFileMetadata}, + translation::{create_translation_queues, translation_task}, + webserver::{ + AppState, EditorMessage, EditorMessageContents, RESERVED_MESSAGE_ID, UpdateMessageContents, + client_websocket, get_client_framework, html_not_found, html_wrapper, path_display, + send_response, + }, +}; + +// Globals +// ------- +lazy_static! { + /// Matches a bare drive letter. Only needed on Windows. + static ref DRIVE_LETTER_REGEX: Regex = Regex::new("^[a-zA-Z]:$").unwrap(); +} + +pub const FILEWATCHER_PATH_PREFIX: &[&str] = &["fw", "fsc"]; + +/// File browser endpoints +/// ---------------------- +/// +/// The file browser provides a very crude interface, allowing a user to select +/// a file from the local filesystem for editing. Long term, this should be +/// replaced by something better. +/// +/// Redirect from the root of the filesystem to the actual root path on this OS. +pub async fn filewatcher_root_fs_redirect() -> impl Responder { + HttpResponse::TemporaryRedirect() + .insert_header((header::LOCATION, "/fw/fsb/")) + .finish() +} + +/// Dispatch to support functions which serve either a directory listing, a +/// CodeChat Editor file, or a normal file. +/// +/// `fsb` stands for "FileSystem Browser" -- directories provide a simple +/// navigation GUI; files load the Client framework. +/// +/// Omit code coverage -- this is a temporary interface, until IDE integration +/// replaces this. +#[cfg(not(tarpaulin_include))] +#[get("/fw/fsb/{path:.*}")] +async fn filewatcher_browser_endpoint( + req: HttpRequest, + app_state: web::Data, + orig_path: web::Path, +) -> Result { + #[cfg(not(target_os = "windows"))] + let fixed_path = orig_path.to_string(); + #[cfg(target_os = "windows")] + let mut fixed_path = orig_path.to_string(); + #[cfg(target_os = "windows")] + // On Windows, a path of `drive_letter:` needs a `/` appended. + if DRIVE_LETTER_REGEX.is_match(&fixed_path) { + fixed_path += "/"; + } else if fixed_path.is_empty() { + // If there's no drive letter yet, we will always use `dir_listing` to + // select a drive. + return Ok(dir_listing("", Path::new("")).await); + } + // All other cases (for example, `C:\a\path\to\file.txt`) are OK. + + // For Linux/OS X, prepend a slash, so that `a/path/to/file.txt` becomes + // `/a/path/to/file.txt`. + #[cfg(not(target_os = "windows"))] + let fixed_path = "/".to_string() + &fixed_path; + + // Handle any + // [errors](https://doc.rust-lang.org/std/fs/fn.canonicalize.html#errors). + let canon_path = match Path::new(&fixed_path).canonicalize() { + Ok(p) => p, + Err(err) => { + return Ok(html_not_found(&format!( + "

The requested path {fixed_path} is not valid: {err}.

" + ))); + } + }; + if canon_path.is_dir() { + return Ok(dir_listing(orig_path.as_str(), &canon_path).await); + } else if canon_path.is_file() { + // Get an ID for this connection. + let connection_id_raw = get_connection_id_raw(&app_state); + return processing_task(&canon_path, req, app_state, connection_id_raw).await; + } + + // It's not a directory or a file...we give up. For simplicity, don't handle + // symbolic links. + Ok(html_not_found(&format!( + "

The requested path {} is not a directory or a file.

", + path_display(&canon_path) + ))) +} + +/// ### Directory browser +/// +/// Create a web page listing all files and subdirectories of the provided +/// directory. +/// +/// Omit code coverage -- this is a temporary interface, until IDE integration +/// replaces this. +#[cfg(not(tarpaulin_include))] +async fn dir_listing(web_path: &str, dir_path: &Path) -> HttpResponse { + // Special case on Windows: list drive letters. + #[cfg(target_os = "windows")] + if dir_path == Path::new("") { + // List drive letters in Windows + let mut drive_html = String::new(); + let logical_drives = match get_logical_drive() { + Ok(v) => v, + Err(err) => return html_not_found(&format!("Unable to list drive letters: {err}.")), + }; + for drive_letter in logical_drives { + drive_html.push_str(&format!( + "
  • {drive_letter}:
    • \n" + )); + } + + return HttpResponse::Ok() + .content_type(ContentType::html()) + .body(html_wrapper(&formatdoc!( + " +

    Drives

    +
      + {drive_html} +
    + " + ))); + } + + // List each file/directory with appropriate links. + let mut unwrapped_read_dir = match fs::read_dir(dir_path).await { + Ok(p) => p, + Err(err) => { + return html_not_found(&format!( + "

    Unable to list the directory {}: {err}/

    ", + path_display(dir_path) + )); + } + }; + + // Get a listing of all files and directories + let mut files: Vec = Vec::new(); + let mut dirs: Vec = Vec::new(); + loop { + match unwrapped_read_dir.next_entry().await { + Ok(v) => { + if let Some(dir_entry) = v { + let file_type = match dir_entry.file_type().await { + Ok(x) => x, + Err(err) => { + return html_not_found(&format!( + "

    Unable to determine the type of {}: {err}.", + path_display(&dir_entry.path()), + )); + } + }; + if file_type.is_file() { + files.push(dir_entry); + } else { + // Group symlinks with dirs. + dirs.push(dir_entry); + } + } else { + break; + } + } + Err(err) => { + return html_not_found(&format!("

    Unable to read file in directory: {err}.")); + } + }; + } + // Sort them -- case-insensitive on Windows, normally on Linux/OS X. + #[cfg(target_os = "windows")] + let file_name_key = |a: &DirEntry| { + Ok::(a.file_name().into_string()?.to_lowercase()) + }; + #[cfg(not(target_os = "windows"))] + let file_name_key = |a: &DirEntry| a.file_name().into_string(); + files.sort_unstable_by_key(file_name_key); + dirs.sort_unstable_by_key(file_name_key); + + // Put this on the resulting webpage. List directories first. + let mut dir_html = String::new(); + // Add a separator if the web path doesn't end with it. + let separator = if web_path.ends_with('/') || web_path.is_empty() { + "" + } else { + "/" + }; + for dir in dirs { + let dir_name = match dir.file_name().into_string() { + Ok(v) => v, + Err(err) => { + return html_not_found(&format!( + "

    Unable to decode directory name '{err:?}' as UTF-8." + )); + } + }; + let encoded_dir = urlencoding::encode(&dir_name); + dir_html += &format!( + "

  • {dir_name}
    • \n", + ); + } + + // List files second. + let mut file_html = String::new(); + for file in files { + let file_name = match file.file_name().into_string() { + Ok(v) => v, + Err(err) => { + return html_not_found( + &format!("

    Unable to decode file name {err:?} as UTF-8.",), + ); + } + }; + let encoded_file = urlencoding::encode(&file_name); + file_html += &formatdoc!( + r#" +

  • {file_name}
    • + "# + ); + } + let body = formatdoc!( + " +

    Directory {}

    +

    Subdirectories

    +
      + {dir_html} +
    +

    Files

    +
      + {file_html} +
    + ", + path_display(dir_path) + ); + + HttpResponse::Ok() + .content_type(ContentType::html()) + .body(html_wrapper(&body)) +} + +const FW: &str = "fw-"; + +/// `fsc` stands for "FileSystem Client", and provides the Client contents from +/// the filesystem. +#[get("/fw/fsc/{connection_id_raw}/{file_path:.*}")] +async fn filewatcher_client_endpoint( + request_path: web::Path<(String, String)>, + req: HttpRequest, + app_state: web::Data, +) -> HttpResponse { + let (connection_id_raw, file_path) = request_path.into_inner(); + filesystem_endpoint( + format!("{FW}{connection_id_raw}"), + file_path, + &req, + &app_state, + ) + .await +} + +async fn processing_task( + file_path: &Path, + req: HttpRequest, + app_state: web::Data, + connection_id_raw: u32, +) -> Result { + // #### Filewatcher IDE + // + // This is a CodeChat Editor file. Start up the Filewatcher IDE tasks: + // + // 1. A task to watch for changes to the file, notifying the CodeChat + // Editor Client when the file should be reloaded. + // 2. A task to receive and respond to messages from the CodeChat Editor + // Client. + // + // First, allocate variables needed by these two tasks. + // + // The path to the currently open CodeChat Editor file. + let Ok(current_filepath) = file_path.to_path_buf().canonicalize() else { + let msg = format!("Unable to canonicalize path {file_path:?}."); + error!("{msg}"); + return Err(error::ErrorBadRequest(msg)); + }; + let mut current_filepath = Some(PathBuf::from(simplified(¤t_filepath))); + + let connection_id_raw = connection_id_raw.to_string(); + let connection_id = format!("{FW}{connection_id_raw}"); + + let created_translation_queues_result = + create_translation_queues(connection_id.clone(), app_state.clone()); + let (from_ide_rx, to_ide_tx, from_client_rx, to_client_tx) = + match created_translation_queues_result { + Err(err) => { + error!("{err}"); + return Err(error::ErrorBadRequest(err)); + } + Ok(tqr) => ( + tqr.from_ide_rx, + tqr.to_ide_tx, + tqr.from_client_rx, + tqr.to_client_tx, + ), + }; + + // Transfer the queues from the global state to this task. + let (from_ide_tx, mut to_ide_rx) = + match app_state.ide_queues.lock().unwrap().remove(&connection_id) { + Some(queues) => (queues.from_websocket_tx.clone(), queues.to_websocket_rx), + None => { + let err = "No websocket queues for connection id {connection_id}."; + error!("{err}"); + return Err(error::ErrorBadRequest(err)); + } + }; + + // #### The filewatcher task. + let connection_id_raw_task = connection_id_raw.clone(); + actix_rt::spawn(async move { + let mut shutdown_only = true; + let mut id: f64 = INITIAL_IDE_MESSAGE_ID; + + // Use a channel to send from the watcher (which runs in another thread) + // into this async (task) context. + let (watcher_tx, mut watcher_rx) = mpsc::channel(10); + // Watch this file. Use the debouncer, to avoid multiple notifications + // for the same file. This approach returns a result of either a working + // debouncer or any errors that occurred. The debouncer's scope needs + // live as long as this connection does; dropping it early means losing + // file change notifications. + let Ok(mut debounced_watcher) = new_debouncer( + Duration::from_secs(2), + None, + // Note that this runs in a separate thread created by the watcher, + // not in an async context. Therefore, use a blocking send. + move |result: DebounceEventResult| { + if let Err(err) = watcher_tx.blocking_send(result) { + // Note: we can't break here, since this runs in a separate + // thread. We have no way to shut down the task (which would + // be the best action to take.) + error!("Unable to send: {err}"); + } + }, + ) else { + error!("Unable to create debouncer."); + return; + }; + if let Some(ref cfp) = current_filepath { + if let Err(err) = debounced_watcher.watch(cfp, RecursiveMode::NonRecursive) { + error!("Unable to watch file: {err}"); + return; + }; + } + + 'task: { + // Provide it a file to open. + if let Some(cfp) = ¤t_filepath { + let Some(cfp_str) = cfp.to_str() else { + let err = format!("Unable to convert file path {cfp:?} to string."); + error!("{err}"); + break 'task; + }; + queue_send!(from_ide_tx.send(EditorMessage { + id, + message: EditorMessageContents::CurrentFile(cfp_str.to_string(), None) + }), 'task); + // Note: it's OK to postpone the increment to here; if the + // `queue_send` exits before this runs, the message didn't get + // sent, so the ID wasn't used. + id += MESSAGE_ID_INCREMENT; + }; + + shutdown_only = false; + } + + // Now that the filewatcher is started, start the translation task then + // proceed to the filewatcher main loop. + actix_rt::spawn(async move { + translation_task( + FW.to_string(), + connection_id_raw_task, + FILEWATCHER_PATH_PREFIX, + app_state, + shutdown_only, + false, + to_ide_tx, + from_ide_rx, + to_client_tx, + from_client_rx, + ) + .await; + }); + + let mut is_closed = false; + 'task: loop { + select! { + // Process results produced by the file watcher. + Some(result) = watcher_rx.recv() => { + match result { + Err(err_vec) => { + for err in err_vec { + // Report errors locally and to the CodeChat + // Editor. + let msg = format!("Watcher error: {err}"); + error!("{msg}"); + // Send using an ID which indicates this isn't a + // response to a message received from the + // client. + send_response(&from_ide_tx, RESERVED_MESSAGE_ID, Err(msg)).await; + } + } + + Ok(debounced_event_vec) => { + for debounced_event in debounced_event_vec { + let is_modify = match debounced_event.event.kind { + // On OS X, we get a `Create` event when a + // file is modified. + EventKind::Create(_create_kind) => true, + // On Windows, the `_modify_kind` is `Any`; + // therefore; ignore it rather than trying + // to look at only content modifications. + EventKind::Modify(_modify_kind) => true, + _ => { + // TODO: handle delete. + info!("Unhandled watcher event: {debounced_event:?}."); + false + } + }; + if is_modify { + if debounced_event.event.paths.len() != 1 || + current_filepath.as_ref().is_none_or(|cfp| cfp != &debounced_event.event.paths[0]) + { + warn!("Modification to different file {}.", debounced_event.event.paths[0].to_string_lossy()); + } else { + let cfp = current_filepath.as_ref().unwrap(); + let Some(current_filepath_str) = cfp.to_str() else { + error!("Unable to convert path {cfp:?} to string."); + break 'task; + }; + + // Since the parents are identical, send an + // update. First, read the modified file. + let mut file_contents = String::new(); + let read_ret = match File::open(&cfp).await { + Ok(fc) => fc, + Err(err) => { + // We can't open the file -- it's been + // moved or deleted. Close the file. + error!("Unable to open file: {err}"); + break 'task; + } + } + .read_to_string(&mut file_contents) + .await; + + // Close the file if it can't be read as + // Unicode text. + if read_ret.is_err() { + error!("Unable to read '{}': {}", cfp.to_string_lossy(), read_ret.unwrap_err()); + break 'task; + } + + queue_send!(from_ide_tx.send(EditorMessage { + id, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: current_filepath_str.to_string(), + contents: Some(CodeChatForWeb { + metadata: SourceFileMetadata { + // The IDE doesn't need to provide this. + mode: "".to_string(), + }, + source: crate::processing::CodeMirrorDiffable::Plain(CodeMirror { + doc: file_contents, + doc_blocks: vec![], + }) + }), + cursor_position: None, + scroll_position: None, + }), + })); + id += MESSAGE_ID_INCREMENT; + } + } + } + } + } + } + + Some(m) = to_ide_rx.recv() => { + match m.message { + EditorMessageContents::Update(update_message_contents) => { + let result = 'process: { + // Check that the file path matches the current + // file. If `canonicalize` fails, then the files + // don't match. + if Some(Path::new(&update_message_contents.file_path).to_path_buf()) != current_filepath { + break 'process Err(format!( + "Update for file '{}' doesn't match current file '{current_filepath:?}'.", + update_message_contents.file_path + )); + } + // With code or a path, there's nothing to save. + let codechat_for_web = match update_message_contents.contents { + None => break 'process Ok(ResultOkTypes::Void), + Some(cfw) => cfw, + }; + + // Translate from the CodeChatForWeb format to + // the contents of a source file. + let CodeMirrorDiffable::Plain(plain) = codechat_for_web.source else { + error!("{}", "Diff not supported."); + break 'task; + }; + let cfp = current_filepath.as_ref().unwrap(); + // Unwrap the file, write to it, then rewatch + // it, in order to avoid a watch notification + // from this write. + if let Err(err) = debounced_watcher.unwatch(cfp) { + let msg = format!( + "Unable to unwatch file '{}': {err}.", + cfp.to_string_lossy() + ); + break 'process Err(msg); + } + // Save this string to a file. + if let Err(err) = fs::write(cfp.as_path(), plain.doc).await { + let msg = format!( + "Unable to save file '{}': {err}.", + cfp.to_string_lossy() + ); + break 'process Err(msg); + } + if let Err(err) = debounced_watcher.watch(cfp, RecursiveMode::NonRecursive) { + let msg = format!( + "Unable to watch file '{}': {err}.", + cfp.to_string_lossy() + ); + break 'process Err(msg); + } + Ok(ResultOkTypes::Void) + }; + send_response(&from_ide_tx, m.id, result).await; + } + + EditorMessageContents::CurrentFile(url_string, _is_text) => { + let result = match url_to_path(&url_string, FILEWATCHER_PATH_PREFIX) { + Err(err) => Err(err), + Ok(ref file_path) => 'err_exit: { + // We finally have the desired path! First, + // unwatch the old path. + if let Some(cfp) = ¤t_filepath { + if let Err(err) = debounced_watcher.unwatch(cfp) { + break 'err_exit Err(format!( + "Unable to unwatch file '{}': {err}.", + cfp.to_string_lossy() + )); + }; + } + // Update to the new path. + current_filepath = Some(file_path.to_path_buf()); + + // Watch the new file. + if let Err(err) = debounced_watcher.watch(file_path, RecursiveMode::NonRecursive) { + break 'err_exit Err(format!( + "Unable to watch file '{}': {err}.", + file_path.to_string_lossy() + )); + } + // Indicate there was no error in the + // `Result` message. + Ok(ResultOkTypes::Void) + } + }; + send_response(&from_ide_tx, m.id, result).await; + }, + + // Process a result, the respond to a message we sent. + EditorMessageContents::Result(message_result) => { + // Report errors to the log. + if let Err(err) = message_result { + error!("Error in message {}: {err}", m.id); + } + } + + EditorMessageContents::Closed => { + info!("Filewatcher closing"); + is_closed = true; + break; + } + + EditorMessageContents::LoadFile(_) => { + // We never have the requested file loaded in this "IDE". Intead, it's always on disk. + send_response(&from_ide_tx, m.id, Ok(ResultOkTypes::LoadFile(None))).await; + } + + EditorMessageContents::Opened(_) | + EditorMessageContents::OpenUrl(_) | + EditorMessageContents::ClientHtml(_) | + EditorMessageContents::RequestClose => { + let msg = format!("Client sent unsupported message type {m:?}"); + error!("{msg}"); + send_response(&from_ide_tx, m.id, Err(msg)).await; + } + } + } + + else => break + } + } + + #[allow(clippy::never_loop)] + loop { + if !is_closed { + queue_send!(from_ide_tx.send(EditorMessage { + id, + message: EditorMessageContents::Closed + })); + } + break; + } + info!("Watcher closed."); + }); + + match get_client_framework(get_test_mode(&req), "fw/ws", &connection_id_raw.to_string()) { + Ok(s) => Ok(HttpResponse::Ok().content_type(ContentType::html()).body(s)), + Err(err) => Err(error::ErrorBadRequest(err)), + } +} + +/// Define a websocket handler for the CodeChat Editor Client. +#[get("/fw/ws/{connection_id_raw}")] +pub async fn filewatcher_websocket( + connection_id_raw: web::Path, + req: HttpRequest, + body: web::Payload, + app_state: web::Data, +) -> Result { + client_websocket( + format!("{FW}{connection_id_raw}"), + req, + body, + app_state.client_queues.clone(), + ) + .await +} + +/// Return a unique ID for an IDE websocket connection. +pub fn get_connection_id_raw(app_state: &web::Data) -> u32 { + let mut connection_id_raw = app_state.filewatcher_next_connection_id.lock().unwrap(); + *connection_id_raw += 1; + *connection_id_raw +} + +// Tests +// ----- +#[cfg(test)] +mod tests { + use std::{ + fs, + path::{Path, PathBuf}, + str::FromStr, + time::Duration, + }; + + use actix_http::Request; + use actix_web::{ + App, + body::BoxBody, + dev::{Service, ServiceResponse}, + test, web, + }; + use assertables::assert_starts_with; + use dunce::simplified; + use path_slash::PathExt; + use pretty_assertions::assert_eq; + use tokio::{select, sync::mpsc::Receiver, time::sleep}; + use url::Url; + + use super::FW; + use crate::{ + cast, prep_test_dir, + processing::{ + CodeChatForWeb, CodeMirror, CodeMirrorDiffable, SourceFileMetadata, TranslationResults, + source_to_codechat_for_web, + }, + test_utils::{check_logger_errors, configure_testing_logger}, + webserver::{ + AppState, EditorMessage, EditorMessageContents, IdeType, ResultOkTypes, + UpdateMessageContents, WebsocketQueues, configure_app, drop_leading_slash, + make_app_data, send_response, tests::IP_PORT, + }, + }; + + async fn get_websocket_queues( + // A path to the temporary directory where the source file is located. + test_dir: &Path, + ) -> ( + WebsocketQueues, + impl Service, Error = actix_web::Error> + use<>, + ) { + let app_data = make_app_data(IP_PORT, None); + let app = test::init_service(configure_app(App::new(), &app_data)).await; + + // Load in a test source file to create a websocket. + let uri = format!("/fw/fsb/{}/test.py", test_dir.to_string_lossy()); + let req = test::TestRequest::get().uri(&uri).to_request(); + let resp = test::call_service(&app, req).await; + assert!(resp.status().is_success()); + // Even after the webpage is served, the websocket task hasn't started. + // Wait a bit for that. + sleep(Duration::from_millis(10)).await; + + // The web page has been served; fake the connected websocket by getting + // the appropriate tx/rx queues. + let app_state = resp.request().app_data::>().unwrap(); + let mut client_queues = app_state.client_queues.lock().unwrap(); + let connection_id_raw = *app_state.filewatcher_next_connection_id.lock().unwrap(); + assert_eq!(client_queues.len(), 1); + ( + client_queues + .remove(&format!("{FW}{connection_id_raw}")) + .unwrap(), + app, + ) + } + + async fn get_message(client_rx: &mut Receiver) -> EditorMessage { + select! { + data = client_rx.recv() => { + let m = data.unwrap(); + // For debugging, print out each message. + println!("{} - {:?}", m.id, m.message); + m + } + _ = sleep(Duration::from_secs(3)) => panic!("Timeout waiting for message") + } + } + + macro_rules! get_message_as { + ($client_rx: expr, $cast_type: ty) => {{ + let m = get_message(&mut $client_rx).await; + (m.id, cast!(m.message, $cast_type)) + }}; + ($client_rx: expr, $cast_type: ty, $( $tup: ident),*) => {{ + let m = get_message(&mut $client_rx).await; + (m.id, cast!(m.message, $cast_type, $($tup),*)) + }}; + } + + #[actix_web::test] + async fn test_websocket_opened_1() { + configure_testing_logger(); + let (temp_dir, test_dir) = prep_test_dir!(); + let (wq, app) = get_websocket_queues(&test_dir).await; + let from_client_tx = wq.from_websocket_tx; + let mut to_client_rx = wq.to_websocket_rx; + + // The initial web request for the Client framework produces a + // `CurrentFile`. + // + // Message ids: IDE - 3->6, Server - 4, Client - 2. + let (id, (url_string, is_text)) = get_message_as!( + to_client_rx, + EditorMessageContents::CurrentFile, + file_name, + is_text + ); + assert_eq!(id, 3.0); + assert_eq!(is_text, Some(true)); + // Acknowledge it. + send_response(&from_client_tx, id, Ok(ResultOkTypes::Void)).await; + + // Compute the path this message should contain. + let mut test_path = test_dir.clone(); + test_path.push("test.py"); + // The comparison below fails without this. + let test_path = test_path.canonicalize().unwrap(); + // The URL parser requires a valid origin. + let url = Url::parse(&format!("http://foo.com{url_string}")).unwrap(); + let url_segs: Vec<_> = url + .path_segments() + .unwrap() + .map(|s| urlencoding::decode(s).unwrap()) + .collect(); + let mut url_path = if cfg!(windows) { + PathBuf::new() + } else { + PathBuf::from_str("/").unwrap() + }; + url_path.push(PathBuf::from_str(&url_segs[3..].join("/")).unwrap()); + let url_path = url_path.canonicalize().unwrap(); + assert_eq!(url_path, test_path); + + // 2. After fetching the file, we should get an update. The Server sends + // a `LoadFile` to the IDE using message id 4; therefore, the `Update` + // is ID 7, and the next message is ID 10. + // + // Message ids: IDE - 6, Server - 4->10, Client - 2. + let uri = format!( + "/fw/fsc/1/{}/test.py", + drop_leading_slash(&test_dir.to_slash().unwrap()) + ); + let req = test::TestRequest::get().uri(&uri).to_request(); + let resp = test::call_service(&app, req).await; + assert!(resp.status().is_success()); + let (id, umc) = get_message_as!(to_client_rx, EditorMessageContents::Update); + assert_eq!(id, 7.0); + send_response(&from_client_tx, id, Ok(ResultOkTypes::Void)).await; + + // Check the contents. + let translation_results = source_to_codechat_for_web("", &"py".to_string(), false, false); + let codechat_for_web = cast!(translation_results, TranslationResults::CodeChat); + assert_eq!(umc.contents, Some(codechat_for_web)); + + // Report any errors produced when removing the temporary directory. + check_logger_errors(0); + temp_dir.close().unwrap(); + } + + #[actix_web::test] + async fn test_websocket_update_1() { + configure_testing_logger(); + let (temp_dir, test_dir) = prep_test_dir!(); + let (wq, app) = get_websocket_queues(&test_dir).await; + let from_client_tx = wq.from_websocket_tx; + let mut to_client_rx = wq.to_websocket_rx; + + // 1. The initial web request for the Client framework produces a + // `CurrentFile`. + // + // Message ids: IDE - 3->6, Server - 4, Client - 2. + let (id, (..)) = get_message_as!( + to_client_rx, + EditorMessageContents::CurrentFile, + file_name, + is_text + ); + assert_eq!(id, 3.0); + send_response(&from_client_tx, id, Ok(ResultOkTypes::Void)).await; + + // 2. After fetching the file, we should get an update. The Server sends + // a `LoadFile` to the IDE using message id 4; therefore, the `Update` + // is ID 7, and the next message is ID 10. + // + // Message ids: IDE - 6, Server - 4->10, Client - 2. + let mut file_path = test_dir.clone(); + file_path.push("test.py"); + let file_path = simplified(&file_path.canonicalize().unwrap()) + .to_str() + .unwrap() + .to_string(); + let uri = format!( + "/fw/fsc/1/{}/test.py", + drop_leading_slash(&test_dir.to_slash().unwrap()) + ); + let req = test::TestRequest::get().uri(&uri).to_request(); + let resp = test::call_service(&app, req).await; + assert!(resp.status().is_success()); + let (id, _) = get_message_as!(to_client_rx, EditorMessageContents::Update); + assert_eq!(id, 7.0); + send_response(&from_client_tx, id, Ok(ResultOkTypes::Void)).await; + + // 3. Send an update message with no contents. + // + // Message ids: IDE - 6, Server - 10, Client - 2->5. + from_client_tx + .send(EditorMessage { + id: 2.0, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: file_path.clone(), + contents: None, + cursor_position: None, + scroll_position: None, + }), + }) + .await + .unwrap(); + + // Check that it produces no error. + assert_eq!( + get_message_as!(to_client_rx, EditorMessageContents::Result), + (2.0, Ok(ResultOkTypes::Void)) + ); + + // 4. Send invalid messages. + // + // Message ids: IDE - 6, Server - 10, Client - 5->14. + for (id, msg) in [ + (5.0, EditorMessageContents::Opened(IdeType::VSCode(true))), + (8.0, EditorMessageContents::ClientHtml("".to_string())), + (11.0, EditorMessageContents::RequestClose), + ] { + from_client_tx + .send(EditorMessage { id, message: msg }) + .await + .unwrap(); + let (id_rx, msg_rx) = get_message_as!(to_client_rx, EditorMessageContents::Result); + assert_eq!(id, id_rx); + assert_starts_with!(cast!(&msg_rx, Err), "Client must not send this message."); + } + + // 5. Send an update message with no path. + // + // Message ids: IDE - 6, Server - 10, Client - 14->17. + from_client_tx + .send(EditorMessage { + id: 14.0, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: "".to_string(), + contents: Some(CodeChatForWeb { + metadata: SourceFileMetadata { + mode: "".to_string(), + }, + source: CodeMirrorDiffable::Plain(CodeMirror { + doc: "".to_string(), + doc_blocks: vec![], + }), + }), + cursor_position: None, + scroll_position: None, + }), + }) + .await + .unwrap(); + + // Check that it produces an error. + let (id, err_msg) = get_message_as!(to_client_rx, EditorMessageContents::Result); + assert_eq!(id, 14.0); + assert_starts_with!( + cast!(err_msg, Err), + "Update for file '' doesn't match current file" + ); + + // 6. Send an update message with unknown source language. + // + // Message ids: IDE - 6, Server - 10, Client - 17->20. + from_client_tx + .send(EditorMessage { + id: 17.0, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: file_path.clone(), + contents: Some(CodeChatForWeb { + metadata: SourceFileMetadata { + mode: "nope".to_string(), + }, + source: CodeMirrorDiffable::Plain(CodeMirror { + doc: "testing".to_string(), + doc_blocks: vec![], + }), + }), + cursor_position: None, + scroll_position: None, + }), + }) + .await + .unwrap(); + + // Check that it produces an error. + assert_eq!( + get_message_as!(to_client_rx, EditorMessageContents::Result), + ( + 17.0, + Err("Unable to translate to source: Invalid mode".to_string()) + ) + ); + + // 7. Send a valid message. + // + // Message ids: IDE - 6, Server - 10, Client - 20->23. + from_client_tx + .send(EditorMessage { + id: 20.0, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: file_path.clone(), + contents: Some(CodeChatForWeb { + metadata: SourceFileMetadata { + mode: "python".to_string(), + }, + source: CodeMirrorDiffable::Plain(CodeMirror { + doc: "testing()".to_string(), + doc_blocks: vec![], + }), + }), + cursor_position: None, + scroll_position: None, + }), + }) + .await + .unwrap(); + assert_eq!( + get_message_as!(to_client_rx, EditorMessageContents::Result), + (20.0, Ok(ResultOkTypes::Void)) + ); + + // Check that the requested file is written. + let mut s = fs::read_to_string(&file_path).unwrap(); + assert_eq!(s, "testing()"); + // Wait for the filewatcher to debounce this file write. + sleep(Duration::from_secs(1)).await; + + // 8. Change this file and verify that this produces an update. + // + // Message ids: IDE - 6->9, Server - 10, Client - 23. + s.push_str("123"); + fs::write(&file_path, s).unwrap(); + assert_eq!( + get_message_as!(to_client_rx, EditorMessageContents::Update), + ( + 6.0, + UpdateMessageContents { + file_path: file_path.clone(), + contents: Some(CodeChatForWeb { + metadata: SourceFileMetadata { + mode: "python".to_string(), + }, + source: CodeMirrorDiffable::Plain(CodeMirror { + doc: "testing()123".to_string(), + doc_blocks: vec![], + }), + }), + cursor_position: None, + scroll_position: None, + } + ) + ); + // Acknowledge this message. + send_response(&from_client_tx, 6.0, Ok(ResultOkTypes::Void)).await; + + // 9. Rename it and check for an close (the file watcher can't detect + // the destination file, so it's treated as the file is deleted). + // + // Message ids: IDE - 9->12, Server - 10, Client - 23. + let mut dest = PathBuf::from(&file_path).parent().unwrap().to_path_buf(); + dest.push("test2.py"); + fs::rename(file_path, dest.as_path()).unwrap(); + // Wait for the filewatcher to debounce this file write. + sleep(Duration::from_secs(1)).await; + let m = get_message(&mut to_client_rx).await; + assert_eq!(m.id, 9.0); + assert!(matches!(m.message, EditorMessageContents::Closed)); + send_response(&from_client_tx, 9.0, Ok(ResultOkTypes::Void)).await; + + // Each of the three invalid message types produces one error. + check_logger_errors(5); + // Report any errors produced when removing the temporary directory. + temp_dir.close().unwrap(); + } +} diff --git a/server/src/ide/vscode.rs b/server/src/ide/vscode.rs new file mode 100644 index 00000000..bbe9e83d --- /dev/null +++ b/server/src/ide/vscode.rs @@ -0,0 +1,284 @@ +use indoc::formatdoc; +// Copyright (C) 2025 Bryan A. Jones. +// +// This file is part of the CodeChat Editor. The CodeChat Editor is free +// software: you can redistribute it and/or modify it under the terms of the GNU +// General Public License as published by the Free Software Foundation, either +// version 3 of the License, or (at your option) any later version. +// +// The CodeChat Editor is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +// details. +// +// You should have received a copy of the GNU General Public License along with +// the CodeChat Editor. If not, see +// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). +/// `vscode.rs` -- Implement server-side functionality for the Visual Studio +/// Code IDE +/// ======================================================================== +// Modules +// ------- +#[cfg(test)] +pub mod tests; + +// Imports +// ------- +// ### Standard library +// None. +// +// ### Third-party +use actix_web::{ + HttpRequest, HttpResponse, + error::{Error, ErrorBadRequest}, + get, web, +}; +use log::{debug, error}; + +// ### Local +use crate::{ + queue_send, + translation::{CreateTranslationQueuesError, create_translation_queues, translation_task}, + webserver::{ + AppState, EditorMessage, EditorMessageContents, IdeType, ResultOkTypes, client_websocket, + escape_html, filesystem_endpoint, get_client_framework, get_server_url, html_wrapper, + send_response, + }, +}; + +// Globals +// ------- +const VSCODE_PATH_PREFIX: &[&str] = &["vsc", "fs"]; +const VSC: &str = "vsc-"; + +// Code +// ---- +#[get("/vsc/ws-ide/{connection_id_raw}")] +pub async fn vscode_ide_websocket( + connection_id_raw: web::Path, + req: HttpRequest, + body: web::Payload, + app_state: web::Data, +) -> Result { + let connection_id_raw = connection_id_raw.to_string(); + let connection_id_str = format!("{VSC}{connection_id_raw}"); + + let created_translation_queues_result = + create_translation_queues(connection_id_str.clone(), app_state.clone()); + let (mut from_ide_rx, to_ide_tx, from_client_rx, to_client_tx) = + match created_translation_queues_result { + Err(err) => match err { + CreateTranslationQueuesError::IdInUse(_) => { + return Err(ErrorBadRequest(err.to_string())); + } + CreateTranslationQueuesError::IdeInUse => { + return client_websocket( + connection_id_str.clone(), + req, + body, + app_state.ide_queues.clone(), + ) + .await; + } + }, + Ok(tqr) => ( + tqr.from_ide_rx, + tqr.to_ide_tx, + tqr.from_client_rx, + tqr.to_client_tx, + ), + }; + + let app_state_task = app_state.clone(); + actix_rt::spawn(async move { + let mut shutdown_only = true; + 'task: { + // Get the first message sent by the IDE. + let Some(first_message): std::option::Option = from_ide_rx.recv().await + else { + error!("{}", "IDE websocket received no data."); + break 'task; + }; + + // Make sure it's the `Opened` message. + let EditorMessageContents::Opened(ide_type) = first_message.message else { + let msg = format!("Unexpected message {first_message:?}"); + error!("{msg}"); + send_response(&to_ide_tx, first_message.id, Err(msg)).await; + + // Send a `Closed` message to shut down the websocket. + queue_send!(to_ide_tx.send(EditorMessage { id: 0.0, message: EditorMessageContents::Closed}), 'task); + break 'task; + }; + debug!("Received IDE Opened message."); + + // Ensure the IDE type (VSCode) is correct. + match ide_type { + IdeType::VSCode(is_self_hosted) => { + // Get the address for the server. + let port = app_state_task.port; + let address = match get_server_url(port).await { + Ok(address) => address, + Err(err) => { + error!("{err:?}"); + break 'task; + } + }; + if is_self_hosted { + // Send a response (successful) to the `Opened` message. + debug!( + "Sending response = OK to IDE Opened message, id {}.", + first_message.id + ); + send_response(&to_ide_tx, first_message.id, Ok(ResultOkTypes::Void)).await; + + // Send the HTML for the internal browser. + let client_html = formatdoc!( + r#" + + + + + + + + "# + ); + debug!("Sending ClientHtml message to IDE: {client_html}"); + queue_send!(to_ide_tx.send(EditorMessage { + id: 0.0, + message: EditorMessageContents::ClientHtml(client_html) + }), 'task); + + // Wait for the response. + let Some(message) = from_ide_rx.recv().await else { + error!("{}", "IDE websocket received no data."); + break 'task; + }; + + // Make sure it's the `Result` message with no errors. + let res = + // First, make sure the ID matches. + if message.id != 0.0 { + Err(format!("Unexpected message ID {}.", message.id)) + } else { + match message.message { + EditorMessageContents::Result(message_result) => match message_result { + Err(err) => Err(format!("Error in ClientHtml: {err}")), + Ok(result_ok) => + if let ResultOkTypes::Void = result_ok { + Ok(()) + } else { + Err(format!( + "Unexpected message LoadFile contents {result_ok:?}." + )) + } + }, + _ => Err(format!("Unexpected message {message:?}")), + } + }; + if let Err(err) = res { + error!("{err}"); + // Send a `Closed` message. + queue_send!(to_ide_tx.send(EditorMessage { + id: 1.0, + message: EditorMessageContents::Closed + }), 'task); + break 'task; + }; + } else { + // Open the Client in an external browser. + if let Err(err) = + webbrowser::open(&format!("{address}/vsc/cf/{connection_id_raw}")) + { + let msg = format!("Unable to open web browser: {err}"); + error!("{msg}"); + send_response(&to_ide_tx, first_message.id, Err(msg)).await; + + // Send a `Closed` message. + queue_send!(to_ide_tx.send(EditorMessage{ + id: 0.0, + message: EditorMessageContents::Closed + }), 'task); + break 'task; + } + // Send a response (successful) to the `Opened` message. + send_response(&to_ide_tx, first_message.id, Ok(ResultOkTypes::Void)).await; + } + } + _ => { + // This is the wrong IDE type. Report then error. + let msg = format!("Invalid IDE type: {ide_type:?}"); + error!("{msg}"); + send_response(&to_ide_tx, first_message.id, Err(msg)).await; + + // Close the connection. + queue_send!(to_ide_tx.send(EditorMessage { id: 0.0, message: EditorMessageContents::Closed}), 'task); + break 'task; + } + } + shutdown_only = false; + } + translation_task( + VSC.to_string(), + connection_id_raw, + VSCODE_PATH_PREFIX, + app_state_task, + shutdown_only, + true, + to_ide_tx, + from_ide_rx, + to_client_tx, + from_client_rx, + ) + .await; + }); + + // Move data between the IDE and the processing task via queues. The + // websocket connection between the client and the IDE will run in the + // endpoint for that connection. + client_websocket(connection_id_str, req, body, app_state.ide_queues.clone()).await +} + +/// Serve the Client Framework. +#[get("/vsc/cf/{connection_id}")] +pub async fn vscode_client_framework(connection_id: web::Path) -> HttpResponse { + HttpResponse::Ok().content_type("text/html").body( + // Send the HTML for the internal browser. + match get_client_framework(false, "vsc/ws-client", &connection_id) { + Ok(web_page) => web_page, + Err(html_string) => { + error!("{html_string}"); + html_wrapper(&escape_html(&html_string)) + } + }, + ) +} + +/// Define a websocket handler for the CodeChat Editor Client. +#[get("/vsc/ws-client/{connection_id}")] +pub async fn vscode_client_websocket( + connection_id: web::Path, + req: HttpRequest, + body: web::Payload, + app_state: web::Data, +) -> Result { + client_websocket( + format!("{VSC}{connection_id}"), + req, + body, + app_state.client_queues.clone(), + ) + .await +} + +// Respond to requests for the filesystem. +#[get("/vsc/fs/{connection_id}/{file_path:.*}")] +async fn serve_vscode_fs( + request_path: web::Path<(String, String)>, + req: HttpRequest, + app_state: web::Data, +) -> HttpResponse { + let (connection_id, file_path) = request_path.into_inner(); + filesystem_endpoint(format!("{VSC}{connection_id}"), file_path, &req, &app_state).await +} diff --git a/server/src/webserver/vscode/tests.rs b/server/src/ide/vscode/tests.rs similarity index 96% rename from server/src/webserver/vscode/tests.rs rename to server/src/ide/vscode/tests.rs index 83027af2..310f91df 100644 --- a/server/src/webserver/vscode/tests.rs +++ b/server/src/ide/vscode/tests.rs @@ -47,7 +47,8 @@ use tokio_tungstenite::{ tungstenite::{http::StatusCode, protocol::Message}, }; -use super::super::{EditorMessage, EditorMessageContents, IdeType, run_server, tests::IP_PORT}; +use crate::translation::{EolType, find_eol_type}; +use crate::webserver::{EditorMessage, EditorMessageContents, IdeType, run_server, tests::IP_PORT}; use crate::{ cast, processing::{ @@ -55,10 +56,7 @@ use crate::{ CodeMirrorDocBlockTransaction, SourceFileMetadata, StringDiff, }, test_utils::{_prep_test_dir, check_logger_errors, configure_testing_logger}, - webserver::{ - ResultOkTypes, UpdateMessageContents, drop_leading_slash, - vscode::{EolType, find_eol_type}, - }, + webserver::{ResultOkTypes, UpdateMessageContents, drop_leading_slash}, }; // Globals @@ -316,19 +314,19 @@ async fn test_vscode_ide_websocket3() { // The HTTP request produces a `LoadFile` message. // - // Message ids: IDE - 4, Server - 3->6, Client - 2. + // Message ids: IDE - 4, Server - 4->7, Client - 2. let em = read_message(&mut ws_ide).await; let msg = cast!(em.message, EditorMessageContents::LoadFile); // Compare these as strings -- we want to ensure the path separator is // correct for the current platform. assert_eq!(file_path.to_string_lossy(), msg.to_string_lossy()); - assert_eq!(em.id, 3.0); + assert_eq!(em.id, 4.0); // Reply to the `LoadFile` message -- the file isn't present. send_message( &mut ws_ide, &EditorMessage { - id: 3.0, + id: 4.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))), }, ) @@ -373,17 +371,17 @@ async fn test_vscode_ide_websocket3a() { // The HTTP request produces a `LoadFile` message. // - // Message ids: IDE - 4, Server - 3->6, Client - 2. + // Message ids: IDE - 4, Server - 4->7, Client - 2. let em = read_message(&mut ws_ide).await; cast!(em.message, EditorMessageContents::LoadFile); // Skip comparing the file names, due to the backslash encoding. - assert_eq!(em.id, 3.0); + assert_eq!(em.id, 4.0); // Reply to the `LoadFile` message -- the file isn't present. send_message( &mut ws_ide, &EditorMessage { - id: 3.0, + id: 4.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))), }, ) @@ -405,7 +403,7 @@ async fn test_vscode_ide_websocket8() { let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await; open_client(&mut ws_ide).await; - // Message ids: IDE - 4->7, Server - 3, Client - 2. + // Message ids: IDE - 4->7, Server - 4, Client - 2. let file_path = test_dir.join("only-in-ide.py"); let file_path_str = file_path.to_str().unwrap().to_string(); send_message( @@ -467,20 +465,20 @@ async fn test_vscode_ide_websocket8() { // This should produce a `LoadFile` message. // - // Message ids: IDE - 7, Server - 3->6, Client - 2. + // Message ids: IDE - 7, Server - 4->7, Client - 2. let em = read_message(&mut ws_ide).await; let msg = cast!(em.message, EditorMessageContents::LoadFile); assert_eq!( path::absolute(Path::new(&msg)).unwrap(), path::absolute(&file_path).unwrap() ); - assert_eq!(em.id, 3.0); + assert_eq!(em.id, 4.0); // Reply to the `LoadFile` message with the file's contents. send_message( &mut ws_ide, &EditorMessage { - id: 3.0, + id: 4.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(Some( "# testing".to_string(), )))), @@ -491,11 +489,11 @@ async fn test_vscode_ide_websocket8() { // This should also produce an `Update` message sent from the Server. // - // Message ids: IDE - 7, Server - 6->9, Client - 2. + // Message ids: IDE - 7, Server - 7->10, Client - 2. assert_eq!( read_message(&mut ws_client).await, EditorMessage { - id: 6.0, + id: 7.0, message: EditorMessageContents::Update(UpdateMessageContents { file_path: file_path_str.clone(), contents: Some(CodeChatForWeb { @@ -521,7 +519,7 @@ async fn test_vscode_ide_websocket8() { send_message( &mut ws_client, &EditorMessage { - id: 6.0, + id: 7.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)), }, ) @@ -532,7 +530,7 @@ async fn test_vscode_ide_websocket8() { assert_eq!( read_message(&mut ws_ide).await, EditorMessage { - id: 6.0, + id: 7.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)) } ); @@ -828,7 +826,7 @@ async fn test_vscode_ide_websocket4() { let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await; open_client(&mut ws_ide).await; - // Message ids: IDE - 4, Server - 3, Client - 2->5. + // Message ids: IDE - 4, Server - 4, Client - 2->5. let file_path_temp = fs::canonicalize(test_dir.join("test.py")).unwrap(); let file_path = simplified(&file_path_temp); let file_path_str = file_path.to_str().unwrap().to_string(); @@ -891,17 +889,17 @@ async fn test_vscode_ide_websocket4() { // This should produce a `LoadFile` message. // - // Message ids: IDE - 4, Server - 3->6, Client - 5. + // Message ids: IDE - 4, Server - 4->7, Client - 5. let em = read_message(&mut ws_ide).await; let msg = cast!(em.message, EditorMessageContents::LoadFile); assert_eq!(fs::canonicalize(&msg).unwrap(), file_path_temp); - assert_eq!(em.id, 3.0); + assert_eq!(em.id, 4.0); // Reply to the `LoadFile` message: the IDE doesn't have the file. send_message( &mut ws_ide, &EditorMessage { - id: 3.0, + id: 4.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))), }, ) @@ -910,11 +908,11 @@ async fn test_vscode_ide_websocket4() { // This should also produce an `Update` message sent from the Server. // - // Message ids: IDE - 4, Server - 6->9, Client - 5. + // Message ids: IDE - 4, Server - 7->10, Client - 5. assert_eq!( read_message(&mut ws_client).await, EditorMessage { - id: 6.0, + id: 7.0, message: EditorMessageContents::Update(UpdateMessageContents { file_path: file_path_str.clone(), contents: Some(CodeChatForWeb { @@ -940,7 +938,7 @@ async fn test_vscode_ide_websocket4() { send_message( &mut ws_client, &EditorMessage { - id: 6.0, + id: 7.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)), }, ) @@ -948,7 +946,7 @@ async fn test_vscode_ide_websocket4() { assert_eq!( read_message(&mut ws_ide).await, EditorMessage { - id: 6.0, + id: 7.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)), } ); @@ -970,20 +968,20 @@ async fn test_vscode_ide_websocket4() { // This should also produce a `LoadFile` message. // - // Message ids: IDE - 4, Server - 9->12, Client - 5. + // Message ids: IDE - 4, Server - 10->13, Client - 5. let em = read_message(&mut ws_ide).await; let msg = cast!(em.message, EditorMessageContents::LoadFile); assert_eq!( fs::canonicalize(&msg).unwrap(), fs::canonicalize(test_dir.join("toc.md")).unwrap() ); - assert_eq!(em.id, 9.0); + assert_eq!(em.id, 10.0); // Reply to the `LoadFile` message: the IDE doesn't have the file. send_message( &mut ws_ide, &EditorMessage { - id: 9.0, + id: 10.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))), }, ) @@ -992,7 +990,7 @@ async fn test_vscode_ide_websocket4() { // Send an update from the Client, which should produce a diff. // - // Message ids: IDE - 4, Server - 12, Client - 5->8. + // Message ids: IDE - 4, Server - 13, Client - 5->8. send_message( &mut ws_client, &EditorMessage { @@ -1073,7 +1071,7 @@ async fn test_vscode_ide_websocket4a() { let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await; open_client(&mut ws_ide).await; - // Message ids: IDE - 4, Server - 3, Client - 2->5. + // Message ids: IDE - 4, Server - 4, Client - 2->5. let hw = "helloworld.pdf"; let file_path_temp = fs::canonicalize(test_dir.join(hw)).unwrap(); let file_path = simplified(&file_path_temp); @@ -1144,17 +1142,17 @@ async fn test_vscode_ide_websocket4a() { // This should produce a `LoadFile` message. // - // Message ids: IDE - 4, Server - 3->6, Client - 5. + // Message ids: IDE - 4, Server - 4->7, Client - 5. let em = read_message(&mut ws_ide).await; let msg = cast!(em.message, EditorMessageContents::LoadFile); assert_eq!(fs::canonicalize(&msg).unwrap(), file_path_temp); - assert_eq!(em.id, 3.0); + assert_eq!(em.id, 4.0); // Reply to the `LoadFile` message: the IDE doesn't have the file. send_message( &mut ws_ide, &EditorMessage { - id: 3.0, + id: 4.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))), }, ) @@ -1174,7 +1172,7 @@ async fn test_vscode_ide_websocket4b() { let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await; open_client(&mut ws_ide).await; - // Message ids: IDE - 4, Server - 3, Client - 2->5. + // Message ids: IDE - 4, Server - 4, Client - 2->5. let hw = "helloworld.pdf"; let file_path_temp = fs::canonicalize(test_dir.join(hw)).unwrap(); let file_path = simplified(&file_path_temp); @@ -1258,17 +1256,17 @@ async fn test_vscode_ide_websocket4b() { // This should produce a `LoadFile` message. // - // Message ids: IDE - 4, Server - 3->6, Client - 5. + // Message ids: IDE - 4, Server - 4->7, Client - 5. let em = read_message(&mut ws_ide).await; let msg = cast!(em.message, EditorMessageContents::LoadFile); assert_eq!(fs::canonicalize(&msg).unwrap(), file_path_temp); - assert_eq!(em.id, 3.0); + assert_eq!(em.id, 4.0); // Reply to the `LoadFile` message: the IDE doesn't have the file. send_message( &mut ws_ide, &EditorMessage { - id: 3.0, + id: 4.0, message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))), }, ) diff --git a/server/src/lib.rs b/server/src/lib.rs index f1834d54..ee3f9193 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -20,8 +20,10 @@ /// TODO: Add the ability to use /// [plugins](https://zicklag.github.io/rust-tutorials/rust-plugins.html). pub mod capture; +pub mod ide; pub mod lexer; pub mod processing; +pub mod translation; pub mod webserver; #[cfg(test)] diff --git a/server/src/processing.rs b/server/src/processing.rs index 455c25bc..24694612 100644 --- a/server/src/processing.rs +++ b/server/src/processing.rs @@ -150,15 +150,18 @@ pub struct CodeMirrorDocBlockUpdate { /// in UTF-16 code units. pub from: usize, /// The starting character this doc block is anchored to after this update. - pub from_new: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub from_new: Option, /// The ending character this doc block is anchored to. - pub to: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub to: Option, /// `None` if the indent is unchanged. Since the indent may be many /// characters, use an `Option` here. #[serde(skip_serializing_if = "Option::is_none")] pub indent: Option, /// Delimiter. - pub delimiter: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub delimiter: Option, /// Contents, as a diff of the previous contents. pub contents: Vec, } @@ -442,12 +445,17 @@ fn code_mirror_to_code_doc_blocks(code_mirror: &CodeMirror) -> Vec contents: codemirror_doc_block.contents.to_string(), lines: 0, })); + let byte_index_prev = byte_index; // Translate `to`. byte_index += byte_index_of( &code_mirror.doc[byte_index..], codemirror_doc_block.to - utf16_index, ); utf16_index = codemirror_doc_block.to; + // Verify that everything between `from` and `to` is newlines. + for char in code_mirror.doc[byte_index_prev..byte_index].chars() { + assert_eq!(char, '\n'); + } } // See if there's a code block after the last doc block. @@ -937,6 +945,18 @@ impl<'a> TokenSource for CodeMirrorDocBlocksStruct<'a> { } } +fn none_if_eq(before: T, after: T) -> Option { + if before == after { None } else { Some(after) } +} + +fn none_if_eq_ref(before: &T, after: &T) -> Option { + if before == after { + None + } else { + Some(after.clone()) + } +} + /// Given two `CodeMirrorDocBlocks`, return a list of changes between them. pub fn diff_code_mirror_doc_blocks( before: &CodeMirrorDocBlockVec, @@ -973,16 +993,22 @@ pub fn diff_code_mirror_doc_blocks( change_specs.push(CodeMirrorDocBlockTransaction::Update( CodeMirrorDocBlockUpdate { from: prev_before_range_start_val.from, - from_new: prev_after_range_start_val.from, - to: prev_after_range_start_val.to, - indent: if prev_before_range_start_val.indent - == prev_after_range_start_val.indent - { - None - } else { - Some(prev_after_range_start_val.indent.clone()) - }, - delimiter: prev_after_range_start_val.delimiter.clone(), + from_new: none_if_eq( + prev_before_range_start_val.from, + prev_after_range_start_val.from, + ), + to: none_if_eq( + prev_before_range_start_val.to, + prev_after_range_start_val.to, + ), + indent: none_if_eq_ref( + &prev_before_range_start_val.indent, + &prev_after_range_start_val.indent, + ), + delimiter: none_if_eq_ref( + &prev_before_range_start_val.delimiter, + &prev_after_range_start_val.delimiter, + ), contents: diff_str( &prev_after_range_start_val.contents, &prev_after_range_start_val.contents, @@ -1019,14 +1045,10 @@ pub fn diff_code_mirror_doc_blocks( change_specs.push(CodeMirrorDocBlockTransaction::Update( CodeMirrorDocBlockUpdate { from: before_val.from, - from_new: after_val.from, - to: after_val.to, - indent: if before_val.indent == after_val.indent { - None - } else { - Some(after_val.indent.clone()) - }, - delimiter: after_val.delimiter.clone(), + from_new: none_if_eq(before_val.from, after_val.from), + to: none_if_eq(before_val.to, after_val.to), + indent: none_if_eq_ref(&before_val.indent, &after_val.indent), + delimiter: none_if_eq_ref(&before_val.delimiter, &after_val.delimiter), contents: diff_str(&before_val.contents, &after_val.contents), }, )); @@ -1070,23 +1092,47 @@ pub fn diff_code_mirror_doc_blocks( // therefore, these two doc blocks can no longer be distinguished, making it // impossible to apply the change to the second doc block. More generally, // this can occur with an insert before a series of blocks which immediately - // follow each other. Therefore, look for sequences of updates where - // `from_new` of a previous entry == `from` of the current entry and swap - // these sequences. + // follow each other. + // + // A similar problem occurs when multiple lines are inserted: the `from` of + // an earlier doc block can become the same as the `from` of a later doc + // block. For example, consider three doc blocks starting at lines 10, 15, + // and 20. Inserting 10 lines makes from first doc block's `from` value + // change to 20, which again violates the doc blocks invariant. + // + // Rather than search for this case (which would be computationally + // expensive), generalize: inserts (which increase the `from` value of a doc + // block, possibly making it identical to later `from` values) should be + // processed from the end of the document toward the beginning, while + // deletions which decrease the `from` value should be processed from the + // beginning of the document to its end. + // + // Doc block insertions and deletions carry the same challenges as textual + // insertions and deletions. Insertions must be performed end to beginning, + // while deletions must be performed beginning to end. + // + // Therefore, look for sequences of insertions (adds) or updates where + // `from_new` >Β `from` and swap these sequences. let mut immediate_sequence_start_index: Option = None; - for index in 1..change_specs.len() { - if let CodeMirrorDocBlockTransaction::Update(prev_update) = &change_specs[index - 1] - && let CodeMirrorDocBlockTransaction::Update(update) = &change_specs[index] - && prev_update.from_new == update.from - && prev_update.from < prev_update.from_new + for index in 0..change_specs.len() { + let is_add = matches!(&change_specs[index], CodeMirrorDocBlockTransaction::Add(_)); + let is_inserted_update = if let CodeMirrorDocBlockTransaction::Update(update) = + &change_specs[index] + && let Some(from_new) = update.from_new + && from_new > update.from { - // We've found two elements in a sequence. + true + } else { + false + }; + if is_add || is_inserted_update { + // This is an update produced by inserting lines. if immediate_sequence_start_index.is_none() { // This is the start of the sequence -- mark it. - immediate_sequence_start_index = Some(index - 1); + immediate_sequence_start_index = Some(index); } } else { - // These two elements aren't a sequence. + // This is not an update produced by an insertion. if let Some(prev_index) = immediate_sequence_start_index { // This is the end of a sequence. Reverse it. change_specs[prev_index..index].reverse(); diff --git a/server/src/processing/tests.rs b/server/src/processing/tests.rs index 6c8b7d94..75b61ed0 100644 --- a/server/src/processing/tests.rs +++ b/server/src/processing/tests.rs @@ -216,6 +216,21 @@ fn test_codemirror_to_code_doc_blocks_py() { build_doc_block("", "#", "") ] ); + + // Error -- instead of newlines, doc blocks replace something else. +} + +#[test] +#[should_panic] +fn test_codemirror_to_code_doc_blocks_error() { + run_test( + "python", + "a\n\n", + vec![ + build_codemirror_doc_block(0, 1, "", "#", ""), + build_codemirror_doc_block(2, 3, "", "#", ""), + ], + ); } #[test] @@ -908,10 +923,10 @@ fn test_diff_2() { vec![CodeMirrorDocBlockTransaction::Update( CodeMirrorDocBlockUpdate { from: 10, - from_new: 10, - to: 12, + from_new: None, + to: Some(12), indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![] } )] @@ -925,10 +940,10 @@ fn test_diff_2() { vec![CodeMirrorDocBlockTransaction::Update( CodeMirrorDocBlockUpdate { from: 10, - from_new: 10, - to: 11, + from_new: None, + to: None, indent: Some(" ".to_string()), - delimiter: "#".to_string(), + delimiter: None, contents: vec![] } )] @@ -942,10 +957,10 @@ fn test_diff_2() { vec![CodeMirrorDocBlockTransaction::Update( CodeMirrorDocBlockUpdate { from: 10, - from_new: 10, - to: 11, + from_new: None, + to: None, indent: None, - delimiter: "*".to_string(), + delimiter: Some("*".to_string()), contents: vec![] } )] @@ -959,10 +974,10 @@ fn test_diff_2() { vec![CodeMirrorDocBlockTransaction::Update( CodeMirrorDocBlockUpdate { from: 10, - from_new: 10, - to: 11, + from_new: None, + to: None, indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![StringDiff { from: 5, to: None, @@ -1004,10 +1019,10 @@ fn test_diff_2() { vec![ CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { from: 11, - from_new: 10, - to: 11, + from_new: Some(10), + to: Some(11), indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![] }), CodeMirrorDocBlockTransaction::Add(CodeMirrorDocBlock { @@ -1105,134 +1120,65 @@ fn test_diff_2() { )] ); - // Test inserts before adjacent doc blocks. - // - // First, with end adjacent doc blocks at the end of the document. + // Test ordering of inserts, deletes, and updates: two deletes, follow by three add/updates. let before = vec![ - build_codemirror_doc_block(10, 11, "", "#", "test1"), - build_codemirror_doc_block(11, 12, " ", "#", "test2"), - build_codemirror_doc_block(12, 13, "", "#", "test3"), + build_codemirror_doc_block(9, 10, "", "#", "test1"), + build_codemirror_doc_block(10, 11, "", "#", "test2"), + build_codemirror_doc_block(11, 12, "", "#", "test3"), + build_codemirror_doc_block(12, 13, "", "#", "test4"), + build_codemirror_doc_block(22, 23, "", "#", "test5"), ]; let after = vec![ - build_codemirror_doc_block(11, 12, "", "#", "test1"), - build_codemirror_doc_block(12, 13, " ", "#", "test2"), - build_codemirror_doc_block(13, 14, "", "#", "test3"), + build_codemirror_doc_block(8, 9, "", "#", "test1"), + build_codemirror_doc_block(10, 11, "", "#", "test3"), + build_codemirror_doc_block(13, 14, "", "#", "test4"), + build_codemirror_doc_block(14, 15, "", "#", "test4a"), + build_codemirror_doc_block(23, 24, "", "#", "test5"), ]; let ret = diff_code_mirror_doc_blocks(&before, &after); assert_eq!( ret, vec![ - // Order is important -- first, 12->13, then 11->12, 10->11 - // (reversed order). + // Order is important! Deletions are ordered beginning to end. CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 12, - from_new: 13, - to: 14, + from: 9, + from_new: Some(8), + to: Some(9), indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![] }), + CodeMirrorDocBlockTransaction::Delete(CodeMirrorDocBlockDelete { from: 10 }), CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { from: 11, - from_new: 12, - to: 13, + from_new: Some(10), + to: Some(11), indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![] }), + // Insertions are ordered end to beginning. CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 10, - from_new: 11, - to: 12, + from: 22, + from_new: Some(23), + to: Some(24), indent: None, - delimiter: "#".to_string(), - contents: vec![] - }), - ] - ); - - // Next, with end adjacent doc blocks not at the end of the document. - let before = vec![ - build_codemirror_doc_block(10, 11, "", "#", "test1"), - build_codemirror_doc_block(11, 12, " ", "#", "test2"), - build_codemirror_doc_block(13, 14, "", "#", "test3"), - ]; - let after = vec![ - build_codemirror_doc_block(11, 12, "", "#", "test1"), - build_codemirror_doc_block(12, 13, " ", "#", "test2"), - build_codemirror_doc_block(14, 15, "", "#", "test3"), - ]; - let ret = diff_code_mirror_doc_blocks(&before, &after); - assert_eq!( - ret, - vec![ - // Order is important -- 11->12, then 10->11 (reversed), then - // 13->14. - CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 11, - from_new: 12, - to: 13, - indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![] }), - CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 10, - from_new: 11, - to: 12, - indent: None, - delimiter: "#".to_string(), - contents: vec![] - }), - CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 13, - from_new: 14, + CodeMirrorDocBlockTransaction::Add(CodeMirrorDocBlock { + from: 14, to: 15, - indent: None, - delimiter: "#".to_string(), - contents: vec![] - }), - ] - ); - - // Test deletes before adjacent doc blocks. - let before = vec![ - build_codemirror_doc_block(10, 11, "", "#", "test1"), - build_codemirror_doc_block(11, 12, " ", "#", "test2"), - build_codemirror_doc_block(12, 13, "", "#", "test3"), - ]; - let after = vec![ - build_codemirror_doc_block(9, 10, "", "#", "test1"), - build_codemirror_doc_block(10, 11, " ", "#", "test2"), - build_codemirror_doc_block(11, 12, "", "#", "test3"), - ]; - let ret = diff_code_mirror_doc_blocks(&before, &after); - assert_eq!( - ret, - vec![ - // Order: no reversal needed. - CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 10, - from_new: 9, - to: 10, - indent: None, - delimiter: "#".to_string(), - contents: vec![] - }), - CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { - from: 11, - from_new: 10, - to: 11, - indent: None, + indent: "".to_string(), delimiter: "#".to_string(), - contents: vec![] + contents: "test4a".to_string() }), CodeMirrorDocBlockTransaction::Update(CodeMirrorDocBlockUpdate { from: 12, - from_new: 11, - to: 12, + from_new: Some(13), + to: Some(14), indent: None, - delimiter: "#".to_string(), + delimiter: None, contents: vec![] }), ] diff --git a/server/src/test_utils.rs b/server/src/test_utils.rs index c183d999..4c6200ff 100644 --- a/server/src/test_utils.rs +++ b/server/src/test_utils.rs @@ -49,9 +49,9 @@ macro_rules! cast { // For an enum containing a single value (the typical case). ($target: expr, $pat: path) => {{ // The if let exploits recent Rust compiler's smart pattern matching. - // Contrary to other solutions like `into_variant`` and friends, this - // one macro covers all ownership usage like` self``, `&self`` and `&mut - // self``. On the other hand` {into,as,as\_mut}\_{variant}\`\` solution + // Contrary to other solutions like `into_variant` and friends, this + // one macro covers all ownership usage like `self`, `&self` and `&mut + // self`. On the other hand `{into,as,as_mut}_{variant}` solution // usually needs 3 \* N method definitions where N is the number of // variants. if let $pat(a) = $target { @@ -63,7 +63,9 @@ macro_rules! cast { panic!("mismatch variant when cast to {}", stringify!($pat)); } }}; - // For an enum containing multiple values, return a tuple. I can't figure out how to automatically do this; for now, the caller must provide the correct number of tuple parameters. + // For an enum containing multiple values, return a tuple. I can't figure + // out how to automatically do this; for now, the caller must provide the + // correct number of tuple parameters. ($target: expr, $pat: path, $( $tup: ident),*) => {{ if let $pat($($tup,)*) = $target { ($($tup,)*) diff --git a/server/src/translation.rs b/server/src/translation.rs new file mode 100644 index 00000000..30e85324 --- /dev/null +++ b/server/src/translation.rs @@ -0,0 +1,999 @@ +// Copyright (C) 2025 Bryan A. Jones. +// +// This file is part of the CodeChat Editor. The CodeChat Editor is free +// software: you can redistribute it and/or modify it under the terms of the GNU +// General Public License as published by the Free Software Foundation, either +// version 3 of the License, or (at your option) any later version. +// +// The CodeChat Editor is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +// details. +// +// You should have received a copy of the GNU General Public License along with +// the CodeChat Editor. If not, see +// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). +/// `vscode.rs` -- Implement server-side functionality for the Visual Studio +/// Code IDE +/// ======================================================================== +/// +/// The IDE extension client (IDE for short) and the CodeChat Editor Client (or +/// Editor for short) exchange messages with each other, mediated by the +/// CodeChat Server. The Server forwards messages from one client to the other, +/// translating as necessary (for example, between source code and the Editor +/// format). This module implements the protocol for this forwarding and +/// translation logic; the actuation translation algoritms are implemented in +/// the processing module. +/// +/// Overview +/// -------- +/// +/// ### Architecture +/// +/// It uses a set of queues to decouple websocket protocol activity from the +/// core processing needed to translate source code between a CodeChat Editor +/// Client and an IDE client. The following diagram illustrates this approach: +/// +/// +/// +/// The queues use multiple-sender, single receiver (mpsc) types. The exception +/// to this pattern is the HTTP endpoint. This endpoint is invoked with each +/// HTTP request, rather than operating as a single, long-running task. It sends +/// the request to the processing task using an mpsc queue; this request +/// includes a one-shot channel which enables the request to return a response +/// to this specific request instance. The endpoint then returns the provided +/// response. +/// +/// ### Protocol +/// +/// The following diagrams formally define the forwarding and translation +/// protocol which this module implements. +/// +/// * The startup phase loads the Client framework into a browser: +/// +/// +/// sequenceDiagram +/// participant IDE +/// participant Server +/// participant Client +/// note over IDE, Client: Startup +/// IDE ->> Server: Opened(IdeType) +/// Server ->> IDE: Result(String: OK) +/// Server ->> IDE: ClientHtml(String: HTML or URL) +/// IDE ->> Server: Result(String: OK) +/// note over IDE, Client: Open browser (Client framework HTML or URL) +/// loop +/// Client -> Server: HTTP request(/static URL) +/// Server -> Client: HTTP response(/static data) +/// end +/// +/// +/// * If the current file in the IDE changes (including the initial startup, +/// when the change is from no file to the current file), or a link is +/// followed in the Client's iframe: +/// +/// +/// sequenceDiagram +/// participant IDE +/// participant Server +/// participant Client +/// alt IDE loads file +/// IDE ->> Client: CurrentFile(String: Path of main.py) +/// opt If Client document is dirty +/// Client ->> IDE: Update(String: contents of main.py) +/// IDE ->> Client: Response(OK) +/// end +/// Client ->> IDE: Response(OK) +/// else Client loads file +/// Client ->> IDE: CurrentFile(String: URL of main.py) +/// IDE ->> Client: Response(OK) +/// end +/// Client ->> Server: HTTP request(URL of main.py) +/// Server ->> IDE: LoadFile(String: path to main.py) +/// IDE ->> Server: Response(LoadFile(String: file contents of main.py)) +/// alt main.py is editable +/// Server ->> Client: HTTP response(contents of Client) +/// Server ->> Client: Update(String: contents of main.py) +/// Client ->> Server: Response(OK) +/// loop +/// Client ->> Server: HTTP request(URL of supporting file in main.py) +/// Server ->> IDE: LoadFile(String: path of supporting file) +/// alt Supporting file in IDE +/// IDE ->> Server: Response(LoadFile(contents of supporting file) +/// Server ->> Client: HTTP response(contents of supporting file) +/// else Supporting file not in IDE +/// IDE ->> Server: Response(LoadFile(None)) +/// Server ->> Client: HTTP response(contents of supporting file from /// filesystem) +/// end +/// end +/// else main.py not editable and not a project +/// Server ->> Client: HTTP response(contents of main.py) +/// else main.py not editable and is a project +/// Server ->> Client: HTTP response(contents of Client Simple Viewer) +/// Client ->> Server: HTTP request (URL?raw of main.py) +/// Server ->> Client: HTTP response(contents of main.py) +/// end +/// +/// +/// * If the current file's contents in the IDE are edited: +/// +/// +/// sequenceDiagram +/// participant IDE +/// participant Server +/// participant Client +/// IDE ->> Server: Update(String: new text contents) +/// alt Main file is editable +/// Server ->> Client: Update(String: new Client contents) +/// else Main file is not editable +/// Server ->> Client: Update(String: new text contents) +/// end +/// Client ->> IDE: Response(String: OK)
    +///     +/// +/// * If the current file's contents in the Client are edited, the Client +/// sends the IDE an `Update` with the revised contents. +/// +/// * When the PC goes to sleep then wakes up, the IDE client and the Editor +/// client both reconnect to the websocket URL containing their assigned ID. +/// +/// * If the Editor client or the IDE client are closed, they close their +/// websocket, which sends a `Close` message to the other websocket, causes +/// it to also close and ending the session. +/// +/// * If the server is stopped (or crashes), both clients shut down after +/// several reconnect retries. +/// +/// ### Editor-overlay filesystem +/// +/// When the Client displays a file provided by the IDE, that file may not exist +/// in the filesystem (a newly-created document), the IDE's content may be newer +/// than the filesystem content (an unsaved file), or the file may exist only in +/// the filesystem (for examples, images referenced by a file). The Client loads +/// files by sending HTTP requests to the Server with a URL which includes the +/// path to the desired file. Therefore, the Server must first ask the IDE if it +/// has the requested file; if so, it must deliver the IDE's file contents; if +/// not, it must load thee requested file from the filesystem. This process -- +/// fetching from the IDE if possible, then falling back to the filesystem -- +/// defines the editor-overlay filesystem. +/// +/// #### Message IDs +/// +/// The message system connects the IDE, Server, and Client; all three can serve +/// as the source or destination for a message. Any message sent should produce +/// a Response message in return. Therefore, we need globally unique IDs for +/// each message. To achieve this, the Server uses IDs that are multiples of 3 +/// (0, 3, 6, ...), the Client multiples of 3 + 1 (1, 4, 7, ...) and the IDE +/// multiples of 3 + 2 (2, 5, 8, ...). A double-precision floating point number +/// (the standard [numeric +/// type](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#number_type) +/// in JavaScript) has a 53-bit mantissa, meaning IDs won't wrap around for a +/// very long time. +// Imports +// ------- +// +// ### Standard library +use std::{cmp::min, collections::HashMap, ffi::OsStr, path::PathBuf}; + +// ### Third-party +use actix_web::web; +use lazy_static::lazy_static; +use log::{debug, error, warn}; +use regex::Regex; +use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::{fs::File, select, sync::mpsc}; + +// ### Local +use crate::webserver::{ + AppState, EditorMessage, EditorMessageContents, WebsocketQueues, send_response, +}; +use crate::{ + oneshot_send, + processing::{ + CodeChatForWeb, CodeMirror, CodeMirrorDiff, CodeMirrorDiffable, SourceFileMetadata, + TranslationResultsString, codechat_for_web_to_source, diff_code_mirror_doc_blocks, + diff_str, source_to_codechat_for_web_string, + }, + queue_send, + webserver::{ + INITIAL_MESSAGE_ID, MESSAGE_ID_INCREMENT, ProcessingTaskHttpRequest, ResultOkTypes, + SimpleHttpResponse, SimpleHttpResponseError, SyncState, UpdateMessageContents, + file_to_response, path_to_url, try_canonicalize, try_read_as_text, url_to_path, + }, +}; + +// Globals +// ------- +// The max length of a message to show in the console. +const MAX_MESSAGE_LENGTH: usize = 300; + +lazy_static! { + /// A regex to determine the type of the first EOL. See 'PROCESSINGS1. + pub static ref EOL_FINDER: Regex = Regex::new("[^\r\n]*(\r?\n)").unwrap(); + +} + +// Data structures +// --------------- +#[derive(Clone, Debug, PartialEq)] +pub enum EolType { + Lf, + Crlf, +} + +// Code +// ---- +pub fn find_eol_type(s: &str) -> EolType { + match EOL_FINDER.captures(s) { + // Assume a line type for strings with no newlines. + None => { + if cfg!(windows) { + EolType::Crlf + } else { + EolType::Lf + } + } + Some(captures) => match captures.get(1) { + None => panic!("No capture group!"), + Some(match_) => { + if match_.as_str() == "\n" { + EolType::Lf + } else { + EolType::Crlf + } + } + }, + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CreateTranslationQueuesError { + #[error("Connection ID {0} already in use.")] + IdInUse(String), + #[error("IDE queue already in use.")] + IdeInUse, +} + +pub struct CreatedTranslationQueues { + pub from_ide_rx: Receiver, + pub to_ide_tx: Sender, + pub from_client_rx: Receiver, + pub to_client_tx: Sender, +} + +pub fn create_translation_queues( + connection_id: String, + app_state: web::Data, +) -> Result { + // There are three cases for this `connection_id`: + // + // 1. It hasn't been used before. In this case, create the appropriate + // queues and start websocket and processing tasks. + // 2. It's in use, but was disconnected. In this case, re-use the queues + // and start the websocket task; the processing task is still running. + // 3. It's in use by another IDE. This is an error, but I don't have a way + // to detect it yet. + // + // Check case 3. + if app_state + .connection_id + .lock() + .unwrap() + .contains(&connection_id) + { + return Err(CreateTranslationQueuesError::IdInUse(connection_id)); + } + + // Now case 2. + if app_state + .ide_queues + .lock() + .unwrap() + .contains_key(&connection_id) + { + return Err(CreateTranslationQueuesError::IdeInUse); + } + + // Then this is case 1. Add the connection ID to the list of active + // connections. + let (from_ide_tx, from_ide_rx) = mpsc::channel(10); + let (to_ide_tx, to_ide_rx) = mpsc::channel(10); + assert!( + app_state + .ide_queues + .lock() + .unwrap() + .insert( + connection_id.clone(), + WebsocketQueues { + from_websocket_tx: from_ide_tx, + to_websocket_rx: to_ide_rx, + }, + ) + .is_none() + ); + let (from_client_tx, from_client_rx) = mpsc::channel(10); + let (to_client_tx, to_client_rx) = mpsc::channel(10); + assert!( + app_state + .client_queues + .lock() + .unwrap() + .insert( + connection_id.clone(), + WebsocketQueues { + from_websocket_tx: from_client_tx, + to_websocket_rx: to_client_rx, + }, + ) + .is_none() + ); + assert!( + app_state + .connection_id + .lock() + .unwrap() + .insert(connection_id.clone()) + ); + + Ok(CreatedTranslationQueues { + from_ide_rx, + to_ide_tx, + from_client_rx, + to_client_tx, + }) +} + +// This is the processing task for the Visual Studio Code IDE. It handles all +// the core logic to moving data between the IDE and the client. +#[allow(clippy::too_many_arguments)] +pub async fn translation_task( + connection_id_prefix: String, + connection_id_raw: String, + prefix: &'static [&'static str], + app_state_task: web::Data, + shutdown_only: bool, + allow_source_diffs: bool, + to_ide_tx: Sender, + mut from_ide_rx: Receiver, + to_client_tx: Sender, + mut from_client_rx: Receiver, +) { + // Start the processing task. + actix_rt::spawn(async move { + let connection_id = format!("{connection_id_prefix}{connection_id_raw}"); + if !shutdown_only { + // Use a [labeled block + // expression](https://doc.rust-lang.org/reference/expressions/loop-expr.html#labelled-block-expressions) + // to provide a way to exit the current task. + 'task: { + let mut current_file = PathBuf::new(); + let mut load_file_requests: HashMap = + HashMap::new(); + debug!("VSCode processing task started."); + + // Create a queue for HTTP requests fo communicate with this task. + let (from_http_tx, mut from_http_rx) = mpsc::channel(10); + app_state_task + .processing_task_queue_tx + .lock() + .unwrap() + .insert(connection_id.to_string(), from_http_tx); + + // Leave space for a server message during the init phase. + let mut id: f64 = INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT; + let mut source_code = String::new(); + let mut code_mirror_doc = String::new(); + // The initial state will be overwritten by the first `Update` or + // `LoadFile`, so this value doesn't matter. + let mut eol = EolType::Lf; + // Some means this contains valid HTML; None means don't use it + // (since it would have contained Markdown). + let mut code_mirror_doc_blocks = Some(Vec::new()); + let prefix_str = "/".to_string() + &prefix.join("/"); + // To send a diff from Server to Client or vice versa, we need to + // ensure they are in sync: + // + // 1. IDE update -> Server -> Client or Client update -> Server -> + // IDE: the Server and Client sync is pending. Client response + // -> Server -> IDE or IDE response -> Server -> Client: the + // Server and Client are synced. + // 2. IDE current file -> Server -> Client or Client current file + // -> Server -> IDE: Out of sync. + // + // It's only safe to send a diff when the most recent sync is + // achieved. So, we need to track the ID of the most recent IDE -> + // Client update or Client -> IDE update, if one is in flight. When + // complete, mark the connection as synchronized. Since all IDs are + // unique, we can use a single variable to store the ID. + // + // Currently, when the Client sends an update, mark the connection + // as out of sync, since the update contains not HTML in the doc + // blocks, but Markdown. When Turndown is moved from JavaScript to + // Rust, this can be changed, since both sides will have HTML in the + // doc blocks. + let mut sync_state = SyncState::OutOfSync; + loop { + select! { + // Look for messages from the IDE. + Some(ide_message) = from_ide_rx.recv() => { + let msg = format!("{:?}", ide_message.message); + debug!("Received IDE message id = {}, message = {}", ide_message.id, &msg[..min(MAX_MESSAGE_LENGTH, msg.len())]); + match ide_message.message { + // Handle messages that the IDE must not send. + EditorMessageContents::Opened(_) | + EditorMessageContents::OpenUrl(_) | + EditorMessageContents::LoadFile(_) | + EditorMessageContents::ClientHtml(_) => { + let msg = "IDE must not send this message."; + error!("{msg}"); + send_response(&to_ide_tx, ide_message.id, Err(msg.to_string())).await; + }, + + // Handle messages that are simply passed through. + EditorMessageContents::Closed | + EditorMessageContents::RequestClose => { + debug!("Forwarding it to the Client."); + queue_send!(to_client_tx.send(ide_message)) + }, + + // Pass a `Result` message to the Client, unless + // it's a `LoadFile` result. + EditorMessageContents::Result(ref result) => { + let is_loadfile = match result { + // See if this error was produced by a + // `LoadFile` result. + Err(_) => load_file_requests.contains_key(&ide_message.id.to_bits()), + Ok(result_ok) => match result_ok { + ResultOkTypes::Void => false, + ResultOkTypes::LoadFile(_) => true, + } + }; + // Pass the message to the client if this isn't + // a `LoadFile` result (the only type of result + // which the Server should handle). + if !is_loadfile { + debug!("Forwarding it to the Client."); + // If this was confirmation from the IDE + // that it received the latest update, then + // mark the IDE as synced. + if sync_state == SyncState::Pending(ide_message.id) { + sync_state = SyncState::InSync; + } + queue_send!(to_client_tx.send(ide_message)); + continue; + } + // Ensure there's an HTTP request for this + // `LoadFile` result. + let Some(http_request) = load_file_requests.remove(&ide_message.id.to_bits()) else { + error!("Error: no HTTP request found for LoadFile result ID {}.", ide_message.id); + break 'task; + }; + + // Take ownership of the result after sending it + // above (which requires ownership). + let EditorMessageContents::Result(result) = ide_message.message else { + error!("{}", "Not a result."); + break; + }; + // Get the file contents from a `LoadFile` + // result; otherwise, this is None. + let file_contents_option = match result { + Err(err) => { + error!("{err}"); + None + }, + Ok(result_ok) => match result_ok { + ResultOkTypes::Void => panic!("LoadFile result should not be void."), + ResultOkTypes::LoadFile(file_contents) => file_contents, + } + }; + + // Process the file contents. Since VSCode + // doesn't have a PDF viewer, determine if this + // is a PDF file. (TODO: look at the magic + // number also -- "%PDF"). + let use_pdf_js = http_request.file_path.extension() == Some(OsStr::new("pdf")); + let (simple_http_response, option_update, file_contents) = match file_contents_option { + Some(file_contents) => { + // If there are Windows newlines, replace + // with Unix; this is reversed when the + // file is sent back to the IDE. + eol = find_eol_type(&file_contents); + let file_contents = if use_pdf_js { file_contents } else { file_contents.replace("\r\n", "\n") }; + file_to_response(&http_request, ¤t_file, Some(file_contents), use_pdf_js).await + }, + None => { + // The file wasn't available in the IDE. + // Look for it in the filesystem. + match File::open(&http_request.file_path).await { + Err(err) => ( + SimpleHttpResponse::Err(SimpleHttpResponseError::Io(err)), + None, + None + ), + Ok(mut fc) => { + let option_file_contents = try_read_as_text(&mut fc).await; + let option_file_contents = if let Some(file_contents) = option_file_contents { + eol = find_eol_type(&file_contents); + let file_contents = if use_pdf_js { file_contents } else { file_contents.replace("\r\n", "\n") }; + Some(file_contents) + } else { + None + }; + // If this + // is a binary file (meaning we can't read + // the contents as UTF-8), send the + // contents as none to signal this isn't a + // text file. + file_to_response( + &http_request, + ¤t_file, + option_file_contents, + use_pdf_js, + ) + .await + } + } + } + }; + if let Some(update) = option_update { + let Some(ref tmp) = update.contents else { + error!("None."); + break; + }; + let CodeMirrorDiffable::Plain(ref plain) = tmp.source else { + error!("Not plain!"); + break; + }; + // We must clone here, since the original is + // placed in the TX queue. + source_code = file_contents.unwrap(); + code_mirror_doc = plain.doc.clone(); + code_mirror_doc_blocks = Some(plain.doc_blocks.clone()); + sync_state = SyncState::Pending(id); + + debug!("Sending Update to Client, id = {id}."); + queue_send!(to_client_tx.send(EditorMessage { + id, + message: EditorMessageContents::Update(update) + })); + id += MESSAGE_ID_INCREMENT; + } + debug!("Sending HTTP response."); + oneshot_send!(http_request.response_queue.send(simple_http_response)); + } + + // Handle the `Update` message. + EditorMessageContents::Update(update) => { + // Normalize the provided file name. + let result = match try_canonicalize(&update.file_path) { + Err(err) => Err(err), + Ok(clean_file_path) => { + match update.contents { + None => { + queue_send!(to_client_tx.send(EditorMessage { + id: ide_message.id, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), + contents: None, + cursor_position: update.cursor_position, + scroll_position: update.scroll_position, + }), + })); + Ok(ResultOkTypes::Void) + } + + Some(contents) => { + match contents.source { + CodeMirrorDiffable::Diff(_diff) => Err("TODO: support for updates with diffable sources.".to_string()), + CodeMirrorDiffable::Plain(code_mirror) => { + // If there are Windows newlines, replace + // with Unix; this is reversed when the + // file is sent back to the IDE. + eol = find_eol_type(&code_mirror.doc); + let doc_normalized_eols = code_mirror.doc.replace("\r\n", "\n"); + // Translate the file. + let (translation_results_string, _path_to_toc) = + source_to_codechat_for_web_string(&doc_normalized_eols, ¤t_file, false); + match translation_results_string { + TranslationResultsString::CodeChat(ccfw) => { + // Send the new translated contents. + debug!("Sending translated contents to Client."); + let CodeMirrorDiffable::Plain(ref ccfw_source_plain) = ccfw.source else { + error!("{}", "Unexpected diff value."); + break; + }; + // Send a diff if possible (only when the + // Client's contents are synced with the + // IDE). + let contents = Some( + if let Some(cmdb) = code_mirror_doc_blocks && + sync_state == SyncState::InSync { + let doc_diff = diff_str(&code_mirror_doc, &ccfw_source_plain.doc); + let code_mirror_diff = diff_code_mirror_doc_blocks(&cmdb, &ccfw_source_plain.doc_blocks); + CodeChatForWeb { + // Clone needed here, so we can copy it + // later. + metadata: ccfw.metadata.clone(), + source: CodeMirrorDiffable::Diff(CodeMirrorDiff { + doc: doc_diff, + doc_blocks: code_mirror_diff + }) + } + } else { + // We must make a clone to put in the TX + // queue; this allows us to keep the + // original below to use with the next + // diff. + ccfw.clone() + } + ); + queue_send!(to_client_tx.send(EditorMessage { + id: ide_message.id, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), + contents, + cursor_position: update.cursor_position, + scroll_position: update.scroll_position, + }), + })); + // Update to the latest code after + // computing diffs. To avoid ownership + // problems, re-define `ccfw_source_plain`. + let CodeMirrorDiffable::Plain(ccfw_source_plain) = ccfw.source else { + error!("{}", "Unexpected diff value."); + break; + }; + source_code = code_mirror.doc; + code_mirror_doc = ccfw_source_plain.doc; + code_mirror_doc_blocks = Some(ccfw_source_plain.doc_blocks); + // Mark the Client as unsynced until this + // is acknowledged. + sync_state = SyncState::Pending(ide_message.id); + Ok(ResultOkTypes::Void) + } + // TODO + TranslationResultsString::Binary => Err("TODO".to_string()), + TranslationResultsString::Err(err) => Err(format!("Error translating source to CodeChat: {err}").to_string()), + TranslationResultsString::Unknown => { + // Send the new raw contents. + debug!("Sending translated contents to Client."); + queue_send!(to_client_tx.send(EditorMessage { + id: ide_message.id, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), + contents: Some(CodeChatForWeb { + metadata: SourceFileMetadata { + // Since this is raw data, `mode` doesn't + // matter. + mode: "".to_string(), + }, + source: CodeMirrorDiffable::Plain(CodeMirror { + doc: code_mirror.doc, + doc_blocks: vec![] + }) + }), + cursor_position: update.cursor_position, + scroll_position: update.scroll_position, + }), + })); + Ok(ResultOkTypes::Void) + }, + TranslationResultsString::Toc(_) => { + Err("Error: source incorrectly recognized as a TOC.".to_string()) + } + } + } + } + } + } + } + }; + // If there's an error, then report it; + // otherwise, the message is passed to the + // Client, which will provide the result. + if let Err(err) = &result { + error!("{err}"); + send_response(&to_ide_tx, ide_message.id, result).await; + } + } + + // Update the current file; translate it to a URL + // then pass it to the Client. + EditorMessageContents::CurrentFile(file_path, _is_text) => { + debug!("Translating and forwarding it to the Client."); + match try_canonicalize(&file_path) { + Ok(clean_file_path) => { + queue_send!(to_client_tx.send(EditorMessage { + id: ide_message.id, + message: EditorMessageContents::CurrentFile( + path_to_url(&prefix_str, Some(&connection_id_raw), &clean_file_path), Some(true) + ) + })); + current_file = file_path.into(); + // Since this is a new file, mark it as + // unsynced. + sync_state = SyncState::OutOfSync; + } + Err(err) => { + let msg = format!( + "Unable to canonicalize file name {}: {err}", &file_path + ); + error!("{msg}"); + send_response(&to_client_tx, ide_message.id, Err(msg)).await; + } + } + } + } + }, + + // Handle HTTP requests. + Some(http_request) = from_http_rx.recv() => { + debug!("Received HTTP request for {:?} and sending LoadFile to IDE, id = {id}.", http_request.file_path); + // Convert the request into a `LoadFile` message. + queue_send!(to_ide_tx.send(EditorMessage { + id, + message: EditorMessageContents::LoadFile(http_request.file_path.clone()) + })); + // Store the ID and request, which are needed to send a + // response when the `LoadFile` result is received. + load_file_requests.insert(id.to_bits(), http_request); + id += MESSAGE_ID_INCREMENT; + } + + // Handle messages from the client. + Some(client_message) = from_client_rx.recv() => { + let msg = format!("{:?}", client_message.message); + debug!("Received Client message id = {}, message = {}", client_message.id, &msg[..min(MAX_MESSAGE_LENGTH, msg.len())]); + match client_message.message { + // Handle messages that the client must not send. + EditorMessageContents::Opened(_) | + EditorMessageContents::LoadFile(_) | + EditorMessageContents::RequestClose | + EditorMessageContents::ClientHtml(_) => { + let msg = "Client must not send this message."; + error!("{msg}"); + send_response(&to_client_tx, client_message.id, Err(msg.to_string())).await; + }, + + // Handle messages that are simply passed through. + EditorMessageContents::Closed | + EditorMessageContents::Result(_) => { + debug!("Forwarding it to the IDE."); + // If this result confirms that the Client + // received the most recent IDE update, then + // mark the documents as synced. + if sync_state == SyncState::Pending(client_message.id) { + sync_state = SyncState::InSync; + } + queue_send!(to_ide_tx.send(client_message)) + }, + + // Open a web browser when requested. + EditorMessageContents::OpenUrl(url) => { + // This doesn't work in Codespaces. TODO: send + // this back to the VSCode window, then call + // `vscode.env.openExternal(vscode.Uri.parse(url))`. + if let Err(err) = webbrowser::open(&url) { + let msg = format!("Unable to open web browser to URL {url}: {err}"); + error!("{msg}"); + send_response(&to_client_tx, client_message.id, Err(msg)).await; + } else { + send_response(&to_client_tx, client_message.id, Ok(ResultOkTypes::Void)).await; + } + }, + + // Handle the `Update` message. + EditorMessageContents::Update(update_message_contents) => { + debug!("Forwarding translation of it to the IDE."); + match try_canonicalize(&update_message_contents.file_path) { + Err(err) => { + let msg = format!( + "Unable to canonicalize file name {}: {err}", &update_message_contents.file_path + ); + error!("{msg}"); + send_response(&to_client_tx, client_message.id, Err(msg)).await; + continue; + } + Ok(clean_file_path) => { + let codechat_for_web = match update_message_contents.contents { + None => None, + Some(cfw) => match codechat_for_web_to_source( + &cfw) + { + Ok(result) => { + let ccfw = if sync_state == SyncState::InSync && allow_source_diffs { + Some(CodeChatForWeb { + metadata: cfw.metadata, + source: CodeMirrorDiffable::Diff(CodeMirrorDiff { + // Diff with correct EOLs, so that (for + // CRLF files as well as LF files) offsets + // are correct. + doc: diff_str(&eol_convert(source_code, &eol), &eol_convert(result.clone(), &eol)), + doc_blocks: vec![], + }), + }) + } else { + Some(CodeChatForWeb { + metadata: cfw.metadata, + source: CodeMirrorDiffable::Plain(CodeMirror { + // We must clone here, so that it can be + // placed in the TX queue. + doc: eol_convert(result.clone(), &eol), + doc_blocks: vec![], + }), + }) + }; + // Store the document with Unix-style EOLs + // (LFs). + source_code = result; + let CodeMirrorDiffable::Plain(cmd) = cfw.source else { + // TODO: support diffable! + error!("No diff!"); + break; + }; + code_mirror_doc = cmd.doc; + // TODO: instead of `cmd.doc_blocks`, use + // `None` to indicate that the doc blocks + // contain Markdown instead of HTML. + code_mirror_doc_blocks = None; + ccfw + }, + Err(message) => { + let msg = format!( + "Unable to translate to source: {message}" + ); + error!("{msg}"); + send_response(&to_client_tx, client_message.id, Err(msg)).await; + continue; + } + }, + }; + queue_send!(to_ide_tx.send(EditorMessage { + id: client_message.id, + message: EditorMessageContents::Update(UpdateMessageContents { + file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), + contents: codechat_for_web, + cursor_position: update_message_contents.cursor_position, + scroll_position: update_message_contents.scroll_position, + }) + })); + // Mark the IDE contents as out of sync + // until this message is received. + sync_state = SyncState::Pending(client_message.id); + } + } + }, + + // Update the current file; translate it to a URL + // then pass it to the IDE. + EditorMessageContents::CurrentFile(url_string, _is_text) => { + debug!("Forwarding translated path to IDE."); + let result = match url_to_path(&url_string, prefix) { + Err(err) => Err(format!("Unable to convert URL to path: {err}")), + Ok(file_path) => { + match file_path.to_str() { + None => Err("Unable to convert path to string.".to_string()), + Some(file_path_string) => { + // Use a [binary file + // sniffer](#binary-file-sniffer) to + // determine if the file is text or binary. + let is_text = if let Ok(mut fc) = File::open(&file_path).await { + try_read_as_text(&mut fc).await.is_some() + } else { + false + }; + queue_send!(to_ide_tx.send(EditorMessage { + id: client_message.id, + message: EditorMessageContents::CurrentFile(file_path_string.to_string(), Some(is_text)) + })); + current_file = file_path; + // Mark the IDE as out of sync, since this + // is a new file. + sync_state = SyncState::OutOfSync; + Ok(()) + } + } + } + }; + if let Err(msg) = result { + error!("{msg}"); + send_response(&to_client_tx, client_message.id, Err(msg)).await; + } + } + } + }, + + else => break + } + } + } + + debug!("VSCode processing task shutting down."); + if app_state_task + .processing_task_queue_tx + .lock() + .unwrap() + .remove(&connection_id) + .is_none() + { + error!( + "Unable to remove connection ID {connection_id} from processing task queue." + ); + } + if app_state_task + .client_queues + .lock() + .unwrap() + .remove(&connection_id) + .is_none() + { + error!("Unable to remove connection ID {connection_id} from client queues."); + } + if app_state_task + .ide_queues + .lock() + .unwrap() + .remove(&connection_id) + .is_none() + { + error!("Unable to remove connection ID {connection_id} from IDE queues."); + } + + from_ide_rx.close(); + from_ide_rx.close(); + + // Drain any remaining messages after closing the queue. + while let Some(m) = from_ide_rx.recv().await { + warn!("Dropped queued message {m:?}"); + } + while let Some(m) = from_client_rx.recv().await { + warn!("Dropped queued message {m:?}"); + } + debug!("VSCode processing task exited."); + } + }); +} + +// If a string is encoded using CRLFs (Windows style), convert it to LFs only +// (Unix style). +fn eol_convert(s: String, eol_type: &EolType) -> String { + if eol_type == &EolType::Crlf { + s.replace("\n", "\r\n") + } else { + s + } +} diff --git a/server/src/webserver.rs b/server/src/webserver.rs index 3838dff5..6281433e 100644 --- a/server/src/webserver.rs +++ b/server/src/webserver.rs @@ -17,10 +17,8 @@ /// ======================================================= // Submodules // ---------- -mod filewatcher; #[cfg(test)] pub mod tests; -mod vscode; // Imports // ------- @@ -77,15 +75,14 @@ use url::Url; // ### Local //use crate::capture::EventCapture; -use crate::processing::{ - CodeChatForWeb, TranslationResultsString, find_path_to_toc, source_to_codechat_for_web_string, -}; -use filewatcher::{ +use crate::ide::filewatcher::{ filewatcher_browser_endpoint, filewatcher_client_endpoint, filewatcher_root_fs_redirect, filewatcher_websocket, }; -use vscode::{ - serve_vscode_fs, vscode_client_framework, vscode_client_websocket, vscode_ide_websocket, +use crate::ide::vscode::vscode_ide_websocket; +use crate::ide::vscode::{serve_vscode_fs, vscode_client_framework, vscode_client_websocket}; +use crate::processing::{ + CodeChatForWeb, TranslationResultsString, find_path_to_toc, source_to_codechat_for_web_string, }; // Data structures @@ -96,26 +93,26 @@ use vscode::{ // server, and the CodeChat Editor Client /// Provide queues which send data to the IDE and the CodeChat Editor Client. #[derive(Debug)] -struct WebsocketQueues { - from_websocket_tx: Sender, - to_websocket_rx: Receiver, +pub struct WebsocketQueues { + pub from_websocket_tx: Sender, + pub to_websocket_rx: Receiver, } #[derive(Debug)] /// Since an `HttpResponse` doesn't implement `Send`, use this as a simply proxy /// for it. This is used to send a response to the HTTP task to an HTTP request /// made to that task. Send: String, response -struct ProcessingTaskHttpRequest { +pub struct ProcessingTaskHttpRequest { /// The URL provided by this request. - url: String, + pub url: String, /// The path of the file requested. - file_path: PathBuf, + pub file_path: PathBuf, /// Flags for this file: none, TOC, raw. flags: ProcessingTaskHttpRequestFlags, /// True if test mode is enabled. is_test_mode: bool, /// A queue to send the response back to the HTTP task. - response_queue: oneshot::Sender, + pub response_queue: oneshot::Sender, } #[derive(Debug, PartialEq)] @@ -131,7 +128,7 @@ enum ProcessingTaskHttpRequestFlags { /// Since an `HttpResponse` doesn't implement `Send`, use this as a proxy to /// cover all responses to serving a file. #[derive(Debug)] -enum SimpleHttpResponse { +pub enum SimpleHttpResponse { /// Return a 200 with the provided string as the HTML body. Ok(String), /// Return an error as the HTML body. @@ -147,7 +144,7 @@ enum SimpleHttpResponse { // definitive guide to error handling in // Rust](https://www.howtocodeit.com/articles/the-definitive-guide-to-rust-error-handling). #[derive(Debug, thiserror::Error)] -enum SimpleHttpResponseError { +pub enum SimpleHttpResponseError { #[error("Error opening file")] Io(#[from] io::Error), #[error("Project path {0:?} has no final component.")] @@ -166,19 +163,19 @@ enum SimpleHttpResponseError { /// Client, the IDE, and the CodeChat Editor Server. #[derive(Debug, Serialize, Deserialize, PartialEq, TS)] #[ts(export)] -struct EditorMessage { +pub struct EditorMessage { /// A value unique to this message; it's used to report results /// (success/failure) back to the sender. - id: f64, + pub id: f64, /// The actual message. - message: EditorMessageContents, + pub message: EditorMessageContents, } /// Define the data structure used to pass data between the CodeChat Editor /// Client, the CodeChat Editor IDE extension, and the CodeChat Editor Server. #[derive(Debug, Serialize, Deserialize, PartialEq, TS)] #[ts(export)] -enum EditorMessageContents { +pub enum EditorMessageContents { // #### These messages may be sent by either the IDE or the Client. /// This sends an update; any missing fields are unchanged. Valid /// destinations: IDE, Client. @@ -240,7 +237,7 @@ type MessageResult = Result< >; #[derive(Debug, Serialize, Deserialize, PartialEq, TS)] -enum ResultOkTypes { +pub enum ResultOkTypes { /// Most messages have no result. Void, /// The `LoadFile` message provides file contents, if available. This @@ -250,7 +247,7 @@ enum ResultOkTypes { /// Specify the type of IDE that this client represents. #[derive(Debug, Serialize, Deserialize, PartialEq, TS)] -enum IdeType { +pub enum IdeType { /// True if the CodeChat Editor will be hosted inside VSCode; false means it /// should be hosted in an external browser. VSCode(bool), @@ -260,24 +257,27 @@ enum IdeType { /// Contents of the `Update` message. #[derive(Debug, Serialize, Deserialize, PartialEq, TS)] -#[ts(export)] -struct UpdateMessageContents { +#[ts(export, optional_fields)] +pub struct UpdateMessageContents { /// The filesystem path to this file. This is only used by the IDE to /// determine which file to apply Update contents to. The Client stores then /// then sends it back to the IDE in `Update` messages. This helps deal with /// transition times when the IDE and Client have different files loaded, /// guaranteeing to updates are still applied to the correct file. - file_path: String, + pub file_path: String, /// The contents of this file. - contents: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option, /// The current cursor position in the file, where 0 = before the first /// character in the file and contents.length() = after the last character /// in the file. TODO: Selections are not yet supported. TODO: how to get a /// cursor location from within a doc block in the Client? - cursor_position: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cursor_position: Option, /// The normalized vertical scroll position in the file, where 0 = top and 1 /// = bottom. - scroll_position: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub scroll_position: Option, } /// ### Data structures used by the webserver @@ -287,21 +287,18 @@ struct UpdateMessageContents { pub struct AppState { /// Provide methods to control the server. server_handle: Mutex>, - /// The number of the next connection ID to assign. - connection_id: Mutex, + /// The number of the next connection ID to assign for the filewatcher. + pub filewatcher_next_connection_id: Mutex, /// The port this server listens on. - port: u16, + pub port: u16, /// For each connection ID, store a queue tx for the HTTP server to send /// requests to the processing task for that ID. - processing_task_queue_tx: Arc>>>, - /// For each (connection ID, requested URL) store channel to send the - /// matching response to the HTTP task. - filewatcher_client_queues: Arc>>, - /// For each connection ID, store the queues for the VSCode IDE. - vscode_ide_queues: Arc>>, - vscode_client_queues: Arc>>, + pub processing_task_queue_tx: Arc>>>, + /// For each connection ID, store the queues for the IDE and Client. + pub ide_queues: Arc>>, + pub client_queues: Arc>>, /// Connection IDs that are currently in use. - vscode_connection_id: Arc>>, + pub connection_id: Arc>>, /// The auth credentials if authentication is used. credentials: Option, } @@ -356,8 +353,8 @@ const REPLY_TIMEOUT: Duration = if cfg!(test) { /// this server. const WEBSOCKET_PING_DELAY: Duration = Duration::from_secs(2); -/// The initial value for a message ID. -const INITIAL_MESSAGE_ID: f64 = if cfg!(test) { +/// A message ID which won't be used by anything but a `Result` produced by an error not produced in response to a message. +pub const RESERVED_MESSAGE_ID: f64 = if cfg!(test) { // A simpler value when testing. 0.0 } else { @@ -365,12 +362,18 @@ const INITIAL_MESSAGE_ID: f64 = if cfg!(test) { // representable. This is -9007199254740991. -((1i64 << f64::MANTISSA_DIGITS) - 1) as f64 }; +/// The initial value for the server's message ID. +pub const INITIAL_MESSAGE_ID: f64 = RESERVED_MESSAGE_ID + 1.0; +// The initial value for a Client. +pub const INITIAL_CLIENT_MESSAGE_ID: f64 = INITIAL_MESSAGE_ID + 1.0; +// The initial value for an IDE. +pub const INITIAL_IDE_MESSAGE_ID: f64 = INITIAL_CLIENT_MESSAGE_ID + 1.0; /// The increment for a message ID. Since the Client, IDE, and Server all /// increment by this same amount but start at different values, this ensures /// that message IDs will be unique. (Given a mantissa of 53 bits plus a sign /// bit, 2^54 seconds = 574 million years before the message ID wraps around /// assuming an average of 1 message/second.) -const MESSAGE_ID_INCREMENT: f64 = 3.0; +pub const MESSAGE_ID_INCREMENT: f64 = 3.0; /// Synchronization state between the Client, Server, and IDE. #[derive(PartialEq)] @@ -471,35 +474,6 @@ async fn stop(app_state: web::Data) -> HttpResponse { HttpResponse::NoContent().finish() } -/// Assign an ID to a new connection. -#[get("/id")] -async fn connection_id_endpoint( - req: HttpRequest, - body: web::Payload, - app_state: web::Data, -) -> Result { - let (response, mut session, _msg_stream) = actix_ws::handle(&req, body)?; - actix_rt::spawn(async move { - if let Err(err) = session - .text(get_connection_id(&app_state).to_string()) - .await - { - error!("Unable to send connection ID: {err}"); - } - if let Err(err) = session.close(None).await { - error!("Unable to close connection: {err}"); - } - }); - Ok(response) -} - -/// Return a unique ID for an IDE websocket connection. -fn get_connection_id(app_state: &web::Data) -> u32 { - let mut connection_id = app_state.connection_id.lock().unwrap(); - *connection_id += 1; - *connection_id -} - // Get the `mode` query parameter to determine `is_test_mode`; default to // `false`. pub fn get_test_mode(req: &HttpRequest) -> bool { @@ -512,7 +486,7 @@ pub fn get_test_mode(req: &HttpRequest) -> bool { } // Return an instance of the Client. -fn get_client_framework( +pub fn get_client_framework( // True if the page should enable test mode for Clients it loads. is_test_mode: bool, // The URL prefix for a websocket connection to the Server. @@ -575,11 +549,11 @@ fn get_client_framework( /// is then serve it. Serve a CodeChat Editor Client webpage using the /// FileWatcher "IDE". pub async fn filesystem_endpoint( - request_path: web::Path<(String, String)>, + connection_id: String, + request_file_path: String, req: &HttpRequest, app_state: &web::Data, ) -> HttpResponse { - let (connection_id, request_file_path) = request_path.into_inner(); // On Windows, backslashes in the `request_file_path` will be treated as // path separators; however, HTTP does not treat them as path separators. // Therefore, re-encode them to prevent inconsistency between the way HTTP @@ -688,7 +662,7 @@ pub async fn filesystem_endpoint( // Use the provided HTTP request to look for the requested file, returning it as // an HTTP response. This should be called from within a processing task. -async fn make_simple_http_response( +pub async fn make_simple_http_response( // The HTTP request presented to the processing task. http_request: &ProcessingTaskHttpRequest, // Path to the file currently being edited. @@ -726,7 +700,7 @@ async fn make_simple_http_response( // Determine if the provided file is text or binary. If text, return it as a // Unicode string. If binary, return None. -async fn try_read_as_text(file: &mut File) -> Option { +pub async fn try_read_as_text(file: &mut File) -> Option { let mut file_contents = String::new(); // TODO: this is a rather crude way to detect if a file is binary. It's // probably slow for large file (the [underlying @@ -745,7 +719,7 @@ async fn try_read_as_text(file: &mut File) -> Option { // file contents itself (if it's not editable by the Client). If responding with // a Client, also return an Update message which will provided the contents for // the Client. -async fn file_to_response( +pub async fn file_to_response( // The HTTP request presented to the processing task. http_request: &ProcessingTaskHttpRequest, // Path to the file currently being edited. This path should be cleaned by @@ -1092,8 +1066,8 @@ fn make_simple_viewer(http_request: &ProcessingTaskHttpRequest, html: &str) -> S /// allowing the user to edit the plain text of the source code in the IDE, or /// make GUI-enhanced edits of the source code rendered by the CodeChat Editor /// Client. -async fn client_websocket( - connection_id: web::Path, +pub async fn client_websocket( + connection_id: String, req: HttpRequest, body: web::Payload, websocket_queues: Arc>>, @@ -1110,17 +1084,14 @@ async fn client_websocket( aggregated_msg_stream = aggregated_msg_stream.max_continuation_size(10_000_000); // Transfer the queues from the global state to this task. - let (from_websocket_tx, mut to_websocket_rx) = match websocket_queues - .lock() - .unwrap() - .remove(&connection_id.to_string()) - { - Some(queues) => (queues.from_websocket_tx.clone(), queues.to_websocket_rx), - None => { - error!("No websocket queues for connection id {connection_id}."); - return; - } - }; + let (from_websocket_tx, mut to_websocket_rx) = + match websocket_queues.lock().unwrap().remove(&connection_id) { + Some(queues) => (queues.from_websocket_tx.clone(), queues.to_websocket_rx), + None => { + error!("No websocket queues for connection id {connection_id}."); + return; + } + }; // Keep track of pending messages. let mut pending_messages: HashMap> = HashMap::new(); @@ -1438,23 +1409,22 @@ pub fn configure_logger(level: LevelFilter) -> Result<(), Box) -> web::Data { +pub fn make_app_data(port: u16, credentials: Option) -> web::Data { web::Data::new(AppState { server_handle: Mutex::new(None), - connection_id: Mutex::new(0), + filewatcher_next_connection_id: Mutex::new(0), port, processing_task_queue_tx: Arc::new(Mutex::new(HashMap::new())), - filewatcher_client_queues: Arc::new(Mutex::new(HashMap::new())), - vscode_ide_queues: Arc::new(Mutex::new(HashMap::new())), - vscode_client_queues: Arc::new(Mutex::new(HashMap::new())), - vscode_connection_id: Arc::new(Mutex::new(HashSet::new())), + ide_queues: Arc::new(Mutex::new(HashMap::new())), + client_queues: Arc::new(Mutex::new(HashMap::new())), + connection_id: Arc::new(Mutex::new(HashSet::new())), credentials, }) } // Configure the web application. I'd like to make this return an // `App`, but `AppEntry` is a private module. -fn configure_app(app: App, app_data: &web::Data) -> App +pub fn configure_app(app: App, app_data: &web::Data) -> App where T: ServiceFactory, { @@ -1488,7 +1458,7 @@ where // --------- // // Send a response to the client after processing a message from the client. -async fn send_response(client_tx: &Sender, id: f64, result: MessageResult) { +pub async fn send_response(client_tx: &Sender, id: f64, result: MessageResult) { if let Err(err) = client_tx .send(EditorMessage { id, @@ -1502,7 +1472,7 @@ async fn send_response(client_tx: &Sender, id: f64, result: Messa // Convert a URL referring to a file in the filesystem into the path to that // file. -fn url_to_path( +pub fn url_to_path( // The URL for the file. url_string: &str, // An array of URL path segments; the URL must start with these. They will @@ -1563,7 +1533,7 @@ fn url_to_path( // 2. If the file exists and if this is Windows, correct case based on the // actual file's naming (even though the filesystem is case-insensitive; // this makes comparisons in the TypeScript simpler). -fn try_canonicalize(file_path: &str) -> Result { +pub fn try_canonicalize(file_path: &str) -> Result { match PathBuf::from_str(file_path) { Err(err) => Err(format!( "Error: unable to parse file path {file_path}: {err}." @@ -1620,7 +1590,7 @@ pub fn path_to_url(prefix: &str, connection_id: Option<&str>, file_path: &Path) // Given a string (which is probably a pathname), drop the leading slash if it's // present. -fn drop_leading_slash(path_: &str) -> &str { +pub fn drop_leading_slash(path_: &str) -> &str { if path_.starts_with("/") { let mut chars = path_.chars(); chars.next(); @@ -1632,19 +1602,19 @@ fn drop_leading_slash(path_: &str) -> &str { // Given a `Path`, transform it into a displayable HTML string (with any // necessary escaping). -fn path_display(p: &Path) -> String { +pub fn path_display(p: &Path) -> String { escape_html(&simplified(p).to_string_lossy()) } // Return a Not Found (404) error with the provided HTML body. -fn html_not_found(msg: &str) -> HttpResponse { +pub fn html_not_found(msg: &str) -> HttpResponse { HttpResponse::NotFound() .content_type(ContentType::html()) .body(html_wrapper(msg)) } // Wrap the provided HTML body in DOCTYPE/html/head tags. -fn html_wrapper(body: &str) -> String { +pub fn html_wrapper(body: &str) -> String { formatdoc!( r#" @@ -1663,7 +1633,7 @@ fn html_wrapper(body: &str) -> String { // Given text, escape it so it formats correctly as HTML. This is a translation // of Python's `html.escape` function. -fn escape_html(unsafe_text: &str) -> String { +pub fn escape_html(unsafe_text: &str) -> String { unsafe_text .replace('&', "&") .replace('<', "<") diff --git a/server/src/webserver/filewatcher.rs b/server/src/webserver/filewatcher.rs deleted file mode 100644 index 6f22bf62..00000000 --- a/server/src/webserver/filewatcher.rs +++ /dev/null @@ -1,1111 +0,0 @@ -// Copyright (C) 2025 Bryan A. Jones. -// -// This file is part of the CodeChat Editor. The CodeChat Editor is free -// software: you can redistribute it and/or modify it under the terms of the GNU -// General Public License as published by the Free Software Foundation, either -// version 3 of the License, or (at your option) any later version. -// -// The CodeChat Editor is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -// details. -// -// You should have received a copy of the GNU General Public License along with -// the CodeChat Editor. If not, see -// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). -/// `filewatcher.rs` -- Implement the File Watcher "IDE" -/// ==================================================== -// Imports -// ------- -// -// ### Standard library -use std::{ - path::{Path, PathBuf}, - time::Duration, -}; - -// ### Third-party -use actix_web::{ - HttpRequest, HttpResponse, Responder, - error::Error, - get, - http::header::{self, ContentType}, - web, -}; -use dunce::simplified; -use indoc::formatdoc; -use lazy_static::lazy_static; -use log::{error, info, warn}; -use notify_debouncer_full::{ - DebounceEventResult, new_debouncer, - notify::{EventKind, RecursiveMode}, -}; -use regex::Regex; -use tokio::{ - fs::DirEntry, - fs::{self, File}, - io::AsyncReadExt, - select, - sync::mpsc, -}; -use urlencoding; -#[cfg(target_os = "windows")] -use win_partitions::win_api::get_logical_drive; - -// ### Local -use super::{ - AppState, EditorMessage, EditorMessageContents, UpdateMessageContents, WebsocketQueues, - client_websocket, escape_html, get_client_framework, get_connection_id, html_not_found, - html_wrapper, path_display, send_response, -}; -use crate::{ - oneshot_send, - processing::{ - TranslationResultsString, codechat_for_web_to_source, source_to_codechat_for_web_string, - }, - queue_send, - webserver::{ - ResultOkTypes, filesystem_endpoint, get_test_mode, make_simple_http_response, path_to_url, - url_to_path, - }, -}; - -// Globals -// ------- -lazy_static! { - /// Matches a bare drive letter. Only needed on Windows. - static ref DRIVE_LETTER_REGEX: Regex = Regex::new("^[a-zA-Z]:$").unwrap(); -} - -pub const FILEWATCHER_PATH_PREFIX: &[&str] = &["fw", "fsc"]; - -/// File browser endpoints -/// ---------------------- -/// -/// The file browser provides a very crude interface, allowing a user to select -/// a file from the local filesystem for editing. Long term, this should be -/// replaced by something better. -/// -/// Redirect from the root of the filesystem to the actual root path on this OS. -pub async fn filewatcher_root_fs_redirect() -> impl Responder { - HttpResponse::TemporaryRedirect() - .insert_header((header::LOCATION, "/fw/fsb/")) - .finish() -} - -/// Dispatch to support functions which serve either a directory listing, a -/// CodeChat Editor file, or a normal file. -/// -/// `fsb` stands for "FileSystem Browser" -- directories provide a simple -/// navigation GUI; files load the Client framework. -/// -/// Omit code coverage -- this is a temporary interface, until IDE integration -/// replaces this. -#[cfg(not(tarpaulin_include))] -#[get("/fw/fsb/{path:.*}")] -async fn filewatcher_browser_endpoint( - req: HttpRequest, - app_state: web::Data, - orig_path: web::Path, -) -> impl Responder { - #[cfg(not(target_os = "windows"))] - let fixed_path = orig_path.to_string(); - #[cfg(target_os = "windows")] - let mut fixed_path = orig_path.to_string(); - #[cfg(target_os = "windows")] - // On Windows, a path of `drive_letter:` needs a `/` appended. - if DRIVE_LETTER_REGEX.is_match(&fixed_path) { - fixed_path += "/"; - } else if fixed_path.is_empty() { - // If there's no drive letter yet, we will always use `dir_listing` to - // select a drive. - return dir_listing("", Path::new("")).await; - } - // All other cases (for example, `C:\a\path\to\file.txt`) are OK. - - // For Linux/OS X, prepend a slash, so that `a/path/to/file.txt` becomes - // `/a/path/to/file.txt`. - #[cfg(not(target_os = "windows"))] - let fixed_path = "/".to_string() + &fixed_path; - - // Handle any - // [errors](https://doc.rust-lang.org/std/fs/fn.canonicalize.html#errors). - let canon_path = match Path::new(&fixed_path).canonicalize() { - Ok(p) => p, - Err(err) => { - return html_not_found(&format!( - "

    The requested path {fixed_path} is not valid: {err}.

    " - )); - } - }; - if canon_path.is_dir() { - return dir_listing(orig_path.as_str(), &canon_path).await; - } else if canon_path.is_file() { - // Get an ID for this connection. - let connection_id = get_connection_id(&app_state); - actix_rt::spawn(async move { - processing_task(&canon_path, app_state, connection_id).await; - }); - return match get_client_framework(get_test_mode(&req), "fw/ws", &connection_id.to_string()) - { - Ok(s) => HttpResponse::Ok().content_type(ContentType::html()).body(s), - Err(err) => html_not_found(&format!("

    {}

    ", escape_html(&err))), - }; - } - - // It's not a directory or a file...we give up. For simplicity, don't handle - // symbolic links. - html_not_found(&format!( - "

    The requested path {} is not a directory or a file.

    ", - path_display(&canon_path) - )) -} - -/// ### Directory browser -/// -/// Create a web page listing all files and subdirectories of the provided -/// directory. -/// -/// Omit code coverage -- this is a temporary interface, until IDE integration -/// replaces this. -#[cfg(not(tarpaulin_include))] -async fn dir_listing(web_path: &str, dir_path: &Path) -> HttpResponse { - // Special case on Windows: list drive letters. - #[cfg(target_os = "windows")] - if dir_path == Path::new("") { - // List drive letters in Windows - let mut drive_html = String::new(); - let logical_drives = match get_logical_drive() { - Ok(v) => v, - Err(err) => return html_not_found(&format!("Unable to list drive letters: {err}.")), - }; - for drive_letter in logical_drives { - drive_html.push_str(&format!( - "
  • {drive_letter}:
    • \n" - )); - } - - return HttpResponse::Ok() - .content_type(ContentType::html()) - .body(html_wrapper(&formatdoc!( - " -

    Drives

    -
      - {drive_html} -
    - " - ))); - } - - // List each file/directory with appropriate links. - let mut unwrapped_read_dir = match fs::read_dir(dir_path).await { - Ok(p) => p, - Err(err) => { - return html_not_found(&format!( - "

    Unable to list the directory {}: {err}/

    ", - path_display(dir_path) - )); - } - }; - - // Get a listing of all files and directories - let mut files: Vec = Vec::new(); - let mut dirs: Vec = Vec::new(); - loop { - match unwrapped_read_dir.next_entry().await { - Ok(v) => { - if let Some(dir_entry) = v { - let file_type = match dir_entry.file_type().await { - Ok(x) => x, - Err(err) => { - return html_not_found(&format!( - "

    Unable to determine the type of {}: {err}.", - path_display(&dir_entry.path()), - )); - } - }; - if file_type.is_file() { - files.push(dir_entry); - } else { - // Group symlinks with dirs. - dirs.push(dir_entry); - } - } else { - break; - } - } - Err(err) => { - return html_not_found(&format!("

    Unable to read file in directory: {err}.")); - } - }; - } - // Sort them -- case-insensitive on Windows, normally on Linux/OS X. - #[cfg(target_os = "windows")] - let file_name_key = |a: &DirEntry| { - Ok::(a.file_name().into_string()?.to_lowercase()) - }; - #[cfg(not(target_os = "windows"))] - let file_name_key = |a: &DirEntry| a.file_name().into_string(); - files.sort_unstable_by_key(file_name_key); - dirs.sort_unstable_by_key(file_name_key); - - // Put this on the resulting webpage. List directories first. - let mut dir_html = String::new(); - for dir in dirs { - let dir_name = match dir.file_name().into_string() { - Ok(v) => v, - Err(err) => { - return html_not_found(&format!( - "

    Unable to decode directory name '{err:?}' as UTF-8." - )); - } - }; - let encoded_dir = urlencoding::encode(&dir_name); - dir_html += &format!( - "

  • {dir_name}
    • \n", - // If this is a raw drive letter, then the path already ends with a - // slash, such as `C:/`. Don't add a second slash in this case. - // Otherwise, add a slash to make `C:/foo` into `C:/foo/`. - // - // Likewise, the Linux root path of `/` already ends with a slash, - // while all other paths such a `/foo` don't. To detect this, look - // for an empty `web_path`. - if web_path.ends_with('/') || web_path.is_empty() { - "" - } else { - "/" - } - ); - } - - // List files second. - let mut file_html = String::new(); - for file in files { - let file_name = match file.file_name().into_string() { - Ok(v) => v, - Err(err) => { - return html_not_found( - &format!("

    Unable to decode file name {err:?} as UTF-8.",), - ); - } - }; - let encoded_file = urlencoding::encode(&file_name); - file_html += &formatdoc!( - r#" -

  • {file_name}
    • - "# - ); - } - let body = formatdoc!( - " -

    Directory {}

    -

    Subdirectories

    -
      - {dir_html} -
    -

    Files

    -
      - {file_html} -
    - ", - path_display(dir_path) - ); - - HttpResponse::Ok() - .content_type(ContentType::html()) - .body(html_wrapper(&body)) -} - -/// `fsc` stands for "FileSystem Client", and provides the Client contents from -/// the filesystem. -#[get("/fw/fsc/{connection_id}/{file_path:.*}")] -async fn filewatcher_client_endpoint( - request_path: web::Path<(String, String)>, - req: HttpRequest, - app_state: web::Data, -) -> HttpResponse { - filesystem_endpoint(request_path, &req, &app_state).await -} - -async fn processing_task(file_path: &Path, app_state: web::Data, connection_id: u32) { - // #### Filewatcher IDE - // - // This is a CodeChat Editor file. Start up the Filewatcher IDE tasks: - // - // 1. A task to watch for changes to the file, notifying the CodeChat - // Editor Client when the file should be reloaded. - // 2. A task to receive and respond to messages from the CodeChat Editor - // Client. - // - // First, allocate variables needed by these two tasks. - // - // The path to the currently open CodeChat Editor file. - let Ok(current_filepath) = file_path.to_path_buf().canonicalize() else { - error!("Unable to canonicalize path {file_path:?}."); - return; - }; - let mut current_filepath = Some(PathBuf::from(simplified(¤t_filepath))); - // #### The filewatcher task. - actix_rt::spawn(async move { - 'task: { - // Use a channel to send from the watcher (which runs in another - // thread) into this async (task) context. - let (watcher_tx, mut watcher_rx) = mpsc::channel(10); - // Watch this file. Use the debouncer, to avoid multiple - // notifications for the same file. This approach returns a result - // of either a working debouncer or any errors that occurred. The - // debouncer's scope needs live as long as this connection does; - // dropping it early means losing file change notifications. - let Ok(mut debounced_watcher) = new_debouncer( - Duration::from_secs(2), - None, - // Note that this runs in a separate thread created by the - // watcher, not in an async context. Therefore, use a blocking - // send. - move |result: DebounceEventResult| { - if let Err(err) = watcher_tx.blocking_send(result) { - // Note: we can't break here, since this runs in a - // separate thread. We have no way to shut down the task - // (which would be the best action to take.) - error!("Unable to send: {err}"); - } - }, - ) else { - error!("Unable to create debouncer."); - break 'task; - }; - if let Some(ref cfp) = current_filepath { - if let Err(err) = debounced_watcher.watch(cfp, RecursiveMode::NonRecursive) { - error!("Unable to watch file: {err}"); - break 'task; - }; - } - - // Create the queues for the websocket connection to communicate - // with this task. - let (from_websocket_tx, mut from_websocket_rx) = mpsc::channel(10); - let (to_websocket_tx, to_websocket_rx) = mpsc::channel(10); - app_state.filewatcher_client_queues.lock().unwrap().insert( - connection_id.to_string(), - WebsocketQueues { - from_websocket_tx, - to_websocket_rx, - }, - ); - - // Provide it a file to open. - let mut id: f64 = 0.0; - if let Some(cfp) = ¤t_filepath { - let url_pathbuf = path_to_url("/fw/fsc", Some(&connection_id.to_string()), cfp); - queue_send!(to_websocket_tx.send(EditorMessage { - id, - message: EditorMessageContents::CurrentFile(url_pathbuf, None) - }), 'task); - // Note: it's OK to postpone the increment to here; if the - // `queue_send` exits before this runs, the message didn't get - // sent, so the ID wasn't used. - id += 1.0; - }; - - // Create a queue for HTTP requests fo communicate with this task. - let (from_http_tx, mut from_http_rx) = mpsc::channel(10); - app_state - .processing_task_queue_tx - .lock() - .unwrap() - .insert(connection_id.to_string(), from_http_tx); - - loop { - select! { - // Process results produced by the file watcher. - Some(result) = watcher_rx.recv() => { - match result { - Err(err_vec) => { - for err in err_vec { - // Report errors locally and to the CodeChat - // Editor. - let msg = format!("Watcher error: {err}"); - error!("{msg}"); - // Send using ID 0 to indicate this isn't a - // response to a message received from the - // client. - send_response(&to_websocket_tx, 0.0, Err(msg)).await; - } - } - - Ok(debounced_event_vec) => { - for debounced_event in debounced_event_vec { - let is_modify = match debounced_event.event.kind { - // On OS X, we get a `Create` event when a - // file is modified. - EventKind::Create(_create_kind) => true, - // On Windows, the `_modify_kind` is `Any`; - // therefore; ignore it rather than trying - // to look at only content modifications. - EventKind::Modify(_modify_kind) => true, - _ => { - // TODO: handle delete. - info!("Watcher event: {debounced_event:?}."); - false - } - }; - if is_modify { - if debounced_event.event.paths.len() != 1 || - current_filepath.as_ref().is_none_or(|cfp| cfp != &debounced_event.event.paths[0]) - { - warn!("Modification to different file {}.", debounced_event.event.paths[0].to_string_lossy()); - } else { - let cfp = current_filepath.as_ref().unwrap(); - let result = 'process: { - // Since the parents are identical, send an - // update. First, read the modified file. - let mut file_contents = String::new(); - let read_ret = match File::open(&cfp).await { - Ok(fc) => fc, - Err(_err) => { - // We can't open the file -- it's been - // moved or deleted. Close the file. - break 'process Err(()); - } - } - .read_to_string(&mut file_contents) - .await; - - // Close the file if it can't be read as - // Unicode text. - if read_ret.is_err() { - error!("Unable to read '{}': {}", cfp.to_string_lossy(), read_ret.unwrap_err()); - break 'process Err(()); - } - - // Translate the file. - let (translation_results_string, _path_to_toc) = - source_to_codechat_for_web_string(&file_contents, cfp, false); - if let TranslationResultsString::CodeChat(cc) = translation_results_string { - let Some(current_filepath_str) = cfp.to_str() else { - error!("Unable to convert path {cfp:?} to string."); - break 'process Err(()); - }; - // Send the new contents. - Ok(EditorMessage { - id, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: current_filepath_str.to_string(), - contents: Some(cc), - cursor_position: None, - scroll_position: None, - }), - }) - } else { - break 'process Err(()); - } - }; - if let Ok(editor_message) = result { - queue_send!(to_websocket_tx.send(editor_message)); - id += 1.0; - } else { - // We can't open the file -- it's been - // moved or deleted. Close the file. - queue_send!(to_websocket_tx.send(EditorMessage { - id, - message: EditorMessageContents::Closed - })); - id += 1.0; - - // Unwatch it. - if let Err(err) = debounced_watcher.unwatch(cfp) { - error!( - "Unable to unwatch file '{}': {err}.", - cfp.to_string_lossy() - ); - } - current_filepath = None; - continue; - } - } - } - } - } - } - } - - Some(http_request) = from_http_rx.recv() => { - // If there's no current file, replace it with an empty - // file, which will still produce an error. - let empty_path = PathBuf::new(); - let cfp = current_filepath.as_ref().unwrap_or(&empty_path); - let (simple_http_response, option_update, _) = make_simple_http_response(&http_request, cfp, false).await; - if let Some(update) = option_update { - // Send the update to the client. - queue_send!(to_websocket_tx.send(EditorMessage { - id, - message: EditorMessageContents::Update(update) - })); - id += 1.0; - } - oneshot_send!(http_request.response_queue.send(simple_http_response)); - } - - Some(m) = from_websocket_rx.recv() => { - match m.message { - EditorMessageContents::Update(update_message_contents) => { - let result = 'process: { - // Check that the file path matches the - // current file. If `canonicalize` fails, - // then the files don't match. - if Some(Path::new(&update_message_contents.file_path).to_path_buf()) != current_filepath { - break 'process Err(format!( - "Update for file '{}' doesn't match current file '{current_filepath:?}'.", - update_message_contents.file_path - )); - } - // With code or a path, there's nothing to - // save. - let codechat_for_web = match update_message_contents.contents { - None => break 'process Ok(ResultOkTypes::Void), - Some(cfw) => cfw, - }; - - // Translate from the CodeChatForWeb format - // to the contents of a source file. - let file_contents = match codechat_for_web_to_source( - &codechat_for_web, - ) { - Ok(r) => r, - Err(message) => { - break 'process Err(format!( - "Unable to translate to source: {message}" - )); - } - }; - - let cfp = current_filepath.as_ref().unwrap(); - // Unwrap the file, write to it, then - // rewatch it, in order to avoid a watch - // notification from this write. - if let Err(err) = debounced_watcher.unwatch(cfp) { - let msg = format!( - "Unable to unwatch file '{}': {err}.", - cfp.to_string_lossy() - ); - break 'process Err(msg); - } - // Save this string to a file. - if let Err(err) = fs::write(cfp.as_path(), file_contents).await { - let msg = format!( - "Unable to save file '{}': {err}.", - cfp.to_string_lossy() - ); - break 'process Err(msg); - } - if let Err(err) = debounced_watcher.watch(cfp, RecursiveMode::NonRecursive) { - let msg = format!( - "Unable to watch file '{}': {err}.", - cfp.to_string_lossy() - ); - break 'process Err(msg); - } - Ok(ResultOkTypes::Void) - }; - send_response(&to_websocket_tx, m.id, result).await; - } - - EditorMessageContents::CurrentFile(url_string, _is_text) => { - let result = match url_to_path(&url_string, FILEWATCHER_PATH_PREFIX) { - Err(err) => Err(err), - Ok(ref file_path) => 'err_exit: { - // We finally have the desired path! First, - // unwatch the old path. - if let Some(cfp) = ¤t_filepath { - if let Err(err) = debounced_watcher.unwatch(cfp) { - break 'err_exit Err(format!( - "Unable to unwatch file '{}': {err}.", - cfp.to_string_lossy() - )); - } - }; - // Update to the new path. - current_filepath = Some(file_path.to_path_buf()); - - // Watch the new file. - if let Err(err) = debounced_watcher.watch(file_path, RecursiveMode::NonRecursive) { - break 'err_exit Err(format!( - "Unable to watch file '{}': {err}.", - file_path.to_string_lossy() - )); - } - - // Indicate there was no error in the - // `Result` message. - Ok(ResultOkTypes::Void) - } - }; - send_response(&to_websocket_tx, m.id, result).await; - }, - - // Process a result, the respond to a message we - // sent. - EditorMessageContents::Result(message_result) => { - // Report errors to the log. - if let Err(err) = message_result { - error!("Error in message {}: {err}", m.id); - } - } - - EditorMessageContents::Closed => { - info!("Filewatcher closing"); - break; - } - - EditorMessageContents::Opened(_) | - EditorMessageContents::OpenUrl(_) | - EditorMessageContents::LoadFile(_) | - EditorMessageContents::ClientHtml(_) | - EditorMessageContents::RequestClose => { - let msg = format!("Client sent unsupported message type {m:?}"); - error!("{msg}"); - send_response(&to_websocket_tx, m.id, Err(msg)).await; - } - } - } - - else => break - } - } - - from_websocket_rx.close(); - if app_state - .processing_task_queue_tx - .lock() - .unwrap() - .remove(&connection_id.to_string()) - .is_none() - { - error!( - "Unable to remove connection ID {connection_id} from processing task queues." - ); - } - // Drain any remaining messages after closing the queue. - while let Some(m) = from_websocket_rx.recv().await { - warn!("Dropped queued message {m:?}"); - } - } - - info!("Watcher closed."); - }); -} - -/// Define a websocket handler for the CodeChat Editor Client. -#[get("/fw/ws/{connection_id}")] -pub async fn filewatcher_websocket( - connection_id: web::Path, - req: HttpRequest, - body: web::Payload, - app_state: web::Data, -) -> Result { - client_websocket( - connection_id, - req, - body, - app_state.filewatcher_client_queues.clone(), - ) - .await -} - -// Tests -// ----- -#[cfg(test)] -mod tests { - use std::{ - fs, - path::{Path, PathBuf}, - str::FromStr, - time::Duration, - }; - - use actix_http::Request; - use actix_web::{ - App, - body::BoxBody, - dev::{Service, ServiceResponse}, - test, web, - }; - use assertables::assert_starts_with; - use dunce::simplified; - use path_slash::PathExt; - use tokio::{select, sync::mpsc::Receiver, time::sleep}; - use url::Url; - - use super::{ - super::{WebsocketQueues, configure_app, make_app_data}, - AppState, EditorMessage, EditorMessageContents, UpdateMessageContents, send_response, - }; - use crate::{ - cast, prep_test_dir, - processing::{ - CodeChatForWeb, CodeMirror, CodeMirrorDiffable, SourceFileMetadata, TranslationResults, - source_to_codechat_for_web, - }, - test_utils::{check_logger_errors, configure_testing_logger}, - webserver::{IdeType, ResultOkTypes, drop_leading_slash, tests::IP_PORT}, - }; - - async fn get_websocket_queues( - // A path to the temporary directory where the source file is located. - test_dir: &Path, - ) -> ( - WebsocketQueues, - impl Service, Error = actix_web::Error> + use<>, - ) { - let app_data = make_app_data(IP_PORT, None); - let app = test::init_service(configure_app(App::new(), &app_data)).await; - - // Load in a test source file to create a websocket. - let uri = format!("/fw/fsb/{}/test.py", test_dir.to_string_lossy()); - let req = test::TestRequest::get().uri(&uri).to_request(); - let resp = test::call_service(&app, req).await; - assert!(resp.status().is_success()); - // Even after the webpage is served, the websocket task hasn't started. - // Wait a bit for that. - sleep(Duration::from_millis(10)).await; - - // The web page has been served; fake the connected websocket by getting - // the appropriate tx/rx queues. - let app_state = resp.request().app_data::>().unwrap(); - let mut joint_editors = app_state.filewatcher_client_queues.lock().unwrap(); - let connection_id = *app_state.connection_id.lock().unwrap(); - assert_eq!(joint_editors.len(), 1); - ( - joint_editors.remove(&connection_id.to_string()).unwrap(), - app, - ) - } - - async fn get_message(client_rx: &mut Receiver) -> EditorMessage { - select! { - data = client_rx.recv() => { - let m = data.unwrap(); - // For debugging, print out each message. - println!("{} - {:?}", m.id, m.message); - m - } - _ = sleep(Duration::from_secs(3)) => panic!("Timeout waiting for message") - } - } - - macro_rules! get_message_as { - ($client_rx: expr_2021, $cast_type: ty) => {{ - let m = get_message(&mut $client_rx).await; - (m.id, cast!(m.message, $cast_type)) - }}; - ($client_rx: expr_2021, $cast_type: ty, $( $tup: ident),*) => {{ - let m = get_message(&mut $client_rx).await; - (m.id, cast!(m.message, $cast_type, $($tup),*)) - }}; - } - - #[actix_web::test] - async fn test_websocket_opened_1() { - configure_testing_logger(); - let (temp_dir, test_dir) = prep_test_dir!(); - let (je, app) = get_websocket_queues(&test_dir).await; - let ide_tx_queue = je.from_websocket_tx; - let mut client_rx = je.to_websocket_rx; - - // The initial web request for the Client framework produces a - // `CurrentFile`. - let (id, (url_string, is_text)) = get_message_as!( - client_rx, - EditorMessageContents::CurrentFile, - file_name, - is_text - ); - assert_eq!(id, 0.0); - assert_eq!(is_text, None); - - // Compute the path this message should contain. - let mut test_path = test_dir.clone(); - test_path.push("test.py"); - // The comparison below fails without this. - let test_path = test_path.canonicalize().unwrap(); - // The URL parser requires a valid origin. - let url = Url::parse(&format!("http://foo.com{url_string}")).unwrap(); - let url_segs: Vec<_> = url - .path_segments() - .unwrap() - .map(|s| urlencoding::decode(s).unwrap()) - .collect(); - let mut url_path = if cfg!(windows) { - PathBuf::new() - } else { - PathBuf::from_str("/").unwrap() - }; - url_path.push(PathBuf::from_str(&url_segs[3..].join("/")).unwrap()); - let url_path = url_path.canonicalize().unwrap(); - assert_eq!(url_path, test_path); - send_response(&ide_tx_queue, id, Ok(ResultOkTypes::Void)).await; - - // 2. After fetching the file, we should get an update. - let uri = format!( - "/fw/fsc/1/{}/test.py", - drop_leading_slash(&test_dir.to_slash().unwrap()) - ); - let req = test::TestRequest::get().uri(&uri).to_request(); - let resp = test::call_service(&app, req).await; - assert!(resp.status().is_success()); - let (id, umc) = get_message_as!(client_rx, EditorMessageContents::Update); - assert_eq!(id, 1.0); - send_response(&ide_tx_queue, id, Ok(ResultOkTypes::Void)).await; - - // Check the contents. - let translation_results = source_to_codechat_for_web("", &"py".to_string(), false, false); - let codechat_for_web = cast!(translation_results, TranslationResults::CodeChat); - assert_eq!(umc.contents, Some(codechat_for_web)); - - // Report any errors produced when removing the temporary directory. - check_logger_errors(0); - temp_dir.close().unwrap(); - } - - #[actix_web::test] - async fn test_websocket_update_1() { - configure_testing_logger(); - let (temp_dir, test_dir) = prep_test_dir!(); - let (je, app) = get_websocket_queues(&test_dir).await; - let ide_tx_queue = je.from_websocket_tx; - let mut client_rx = je.to_websocket_rx; - - // The initial web request for the Client framework produces a - // `CurrentFile`. - let (id, (..)) = get_message_as!( - client_rx, - EditorMessageContents::CurrentFile, - file_name, - is_text - ); - assert_eq!(id, 0.0); - send_response(&ide_tx_queue, 0.0, Ok(ResultOkTypes::Void)).await; - - // The follow-up web request for the file produces an `Update`. - let mut file_path = test_dir.clone(); - file_path.push("test.py"); - let file_path = simplified(&file_path.canonicalize().unwrap()) - .to_str() - .unwrap() - .to_string(); - let uri = format!( - "/fw/fsc/1/{}/test.py", - drop_leading_slash(&test_dir.to_slash().unwrap()) - ); - let req = test::TestRequest::get().uri(&uri).to_request(); - let resp = test::call_service(&app, req).await; - assert!(resp.status().is_success()); - let (id, _) = get_message_as!(client_rx, EditorMessageContents::Update); - assert_eq!(id, 1.0); - send_response(&ide_tx_queue, 1.0, Ok(ResultOkTypes::Void)).await; - - // 1. Send an update message with no contents. - ide_tx_queue - .send(EditorMessage { - id: 0.0, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: file_path.clone(), - contents: None, - cursor_position: None, - scroll_position: None, - }), - }) - .await - .unwrap(); - - // Check that it produces no error. - assert_eq!( - get_message_as!(client_rx, EditorMessageContents::Result), - (0.0, Ok(ResultOkTypes::Void)) - ); - - // 2. Send invalid messages. - for (id, msg) in [ - (1.0, EditorMessageContents::Opened(IdeType::VSCode(true))), - (2.0, EditorMessageContents::ClientHtml("".to_string())), - (3.0, EditorMessageContents::RequestClose), - ] { - ide_tx_queue - .send(EditorMessage { id, message: msg }) - .await - .unwrap(); - let (id_rx, msg_rx) = get_message_as!(client_rx, EditorMessageContents::Result); - assert_eq!(id, id_rx); - assert_starts_with!(cast!(&msg_rx, Err), "Client sent unsupported message type"); - } - - // 3. Send an update message with no path. - ide_tx_queue - .send(EditorMessage { - id: 4.0, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: "".to_string(), - contents: Some(CodeChatForWeb { - metadata: SourceFileMetadata { - mode: "".to_string(), - }, - source: CodeMirrorDiffable::Plain(CodeMirror { - doc: "".to_string(), - doc_blocks: vec![], - }), - }), - cursor_position: None, - scroll_position: None, - }), - }) - .await - .unwrap(); - - // Check that it produces an error. - let (id, err_msg) = get_message_as!(client_rx, EditorMessageContents::Result); - assert_eq!(id, 4.0); - assert_starts_with!( - cast!(err_msg, Err), - "Update for file '' doesn't match current file" - ); - - // 4. Send an update message with unknown source language. - ide_tx_queue - .send(EditorMessage { - id: 5.0, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: file_path.clone(), - contents: Some(CodeChatForWeb { - metadata: SourceFileMetadata { - mode: "nope".to_string(), - }, - source: CodeMirrorDiffable::Plain(CodeMirror { - doc: "testing".to_string(), - doc_blocks: vec![], - }), - }), - cursor_position: None, - scroll_position: None, - }), - }) - .await - .unwrap(); - - // Check that it produces an error. - assert_eq!( - get_message_as!(client_rx, EditorMessageContents::Result), - ( - 5.0, - Err("Unable to translate to source: Invalid mode".to_string()) - ) - ); - - // 5. Send a valid message. - ide_tx_queue - .send(EditorMessage { - id: 6.0, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: file_path.clone(), - contents: Some(CodeChatForWeb { - metadata: SourceFileMetadata { - mode: "python".to_string(), - }, - source: CodeMirrorDiffable::Plain(CodeMirror { - doc: "testing()".to_string(), - doc_blocks: vec![], - }), - }), - cursor_position: None, - scroll_position: None, - }), - }) - .await - .unwrap(); - assert_eq!( - get_message_as!(client_rx, EditorMessageContents::Result), - (6.0, Ok(ResultOkTypes::Void)) - ); - - // Check that the requested file is written. - let mut s = fs::read_to_string(&file_path).unwrap(); - assert_eq!(s, "testing()"); - // Wait for the filewatcher to debounce this file write. - sleep(Duration::from_secs(1)).await; - - // 6. Change this file and verify that this produces an update. - s.push_str("123"); - fs::write(&file_path, s).unwrap(); - assert_eq!( - get_message_as!(client_rx, EditorMessageContents::Update), - ( - 2.0, - UpdateMessageContents { - file_path: file_path.clone(), - contents: Some(CodeChatForWeb { - metadata: SourceFileMetadata { - mode: "python".to_string(), - }, - source: CodeMirrorDiffable::Plain(CodeMirror { - doc: "testing()123".to_string(), - doc_blocks: vec![], - }), - }), - cursor_position: None, - scroll_position: None, - } - ) - ); - // Acknowledge this message. - send_response(&ide_tx_queue, 2.0, Ok(ResultOkTypes::Void)).await; - - // 7. Rename it and check for an close (the file watcher can't detect - // the destination file, so it's treated as the file is deleted). - let mut dest = PathBuf::from(&file_path).parent().unwrap().to_path_buf(); - dest.push("test2.py"); - fs::rename(file_path, dest.as_path()).unwrap(); - assert_eq!( - client_rx.recv().await.unwrap(), - EditorMessage { - id: 3.0, - message: EditorMessageContents::Closed - } - ); - send_response(&ide_tx_queue, 3.0, Ok(ResultOkTypes::Void)).await; - - // 8. Load another file from the Client. - let mut new_file_path = test_dir.clone(); - new_file_path.push("test1.py"); - let new_uri = format!( - "http://localhost/fw/fsc/1/{}", - drop_leading_slash(&urlencoding::encode(&new_file_path.to_slash().unwrap())) - ); - ide_tx_queue - .send(EditorMessage { - id: 7.0, - message: EditorMessageContents::CurrentFile(new_uri.clone(), None), - }) - .await - .unwrap(); - assert_eq!( - get_message_as!(client_rx, EditorMessageContents::Result), - (7.0, Ok(ResultOkTypes::Void)) - ); - - // The follow-up web request for the file produces an `Update`. - let new_req = test::TestRequest::get().uri(&new_uri).to_request(); - let new_resp = test::call_service(&app, new_req).await; - assert!(new_resp.status().is_success()); - let (id, _) = get_message_as!(client_rx, EditorMessageContents::Update); - assert_eq!(id, 4.0); - send_response(&ide_tx_queue, 4.0, Ok(ResultOkTypes::Void)).await; - - // 9. Writes to this file should produce an update. - fs::write(&new_file_path, "testing 1").unwrap(); - get_message_as!(client_rx, EditorMessageContents::Update); - - // Each of the three invalid message types produces one error. - check_logger_errors(3); - // Report any errors produced when removing the temporary directory. - temp_dir.close().unwrap(); - } -} diff --git a/server/src/webserver/tests.rs b/server/src/webserver/tests.rs index 2a69000c..0702fdbd 100644 --- a/server/src/webserver/tests.rs +++ b/server/src/webserver/tests.rs @@ -26,7 +26,8 @@ use std::{ use assert_cmd::Command; use assertables::{assert_ends_with, assert_not_contains, assert_starts_with}; -use super::{filewatcher::FILEWATCHER_PATH_PREFIX, path_to_url, url_to_path}; +use super::{path_to_url, url_to_path}; +use crate::ide::filewatcher::FILEWATCHER_PATH_PREFIX; use crate::prep_test_dir; // Constants diff --git a/server/src/webserver/vscode.rs b/server/src/webserver/vscode.rs deleted file mode 100644 index 9b7a85a5..00000000 --- a/server/src/webserver/vscode.rs +++ /dev/null @@ -1,971 +0,0 @@ -/// Copyright (C) 2025 Bryan A. Jones. -/// -/// This file is part of the CodeChat Editor. The CodeChat Editor is free -/// software: you can redistribute it and/or modify it under the terms of the -/// GNU General Public License as published by the Free Software Foundation, -/// either version 3 of the License, or (at your option) any later version. -/// -/// The CodeChat Editor is distributed in the hope that it will be useful, but -/// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY -/// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -/// more details. -/// -/// You should have received a copy of the GNU General Public License along with -/// the CodeChat Editor. If not, see -/// [http://www.gnu.org/licenses](http://www.gnu.org/licenses). -/// -/// `vscode.rs` -- Implement server-side functionality for the Visual Studio -/// Code IDE -/// ======================================================================== -// Submodules -// ---------- -#[cfg(test)] -pub mod tests; - -// Imports -// ------- -// -// ### Standard library -use std::{cmp::min, collections::HashMap, ffi::OsStr, path::PathBuf}; - -// ### Third-party -use actix_web::{ - HttpRequest, HttpResponse, - error::{Error, ErrorBadRequest}, - get, web, -}; -use indoc::formatdoc; -use lazy_static::lazy_static; -use log::{debug, error, warn}; -use regex::Regex; -use tokio::{fs::File, select, sync::mpsc}; - -// ### Local -use super::{ - AppState, EditorMessage, EditorMessageContents, IdeType, WebsocketQueues, client_websocket, - get_client_framework, send_response, -}; -use crate::{ - oneshot_send, - processing::{ - CodeChatForWeb, CodeMirror, CodeMirrorDiff, CodeMirrorDiffable, SourceFileMetadata, - TranslationResultsString, codechat_for_web_to_source, diff_code_mirror_doc_blocks, - diff_str, source_to_codechat_for_web_string, - }, - queue_send, - webserver::{ - INITIAL_MESSAGE_ID, MESSAGE_ID_INCREMENT, ProcessingTaskHttpRequest, ResultOkTypes, - SimpleHttpResponse, SimpleHttpResponseError, SyncState, UpdateMessageContents, escape_html, - file_to_response, filesystem_endpoint, get_server_url, html_wrapper, path_to_url, - try_canonicalize, try_read_as_text, url_to_path, - }, -}; - -// Globals -// ------- -const VSCODE_PATH_PREFIX: &[&str] = &["vsc", "fs"]; -// The max length of a message to show in the console. -const MAX_MESSAGE_LENGTH: usize = 300; - -lazy_static! { - /// A regex to determine the type of the first EOL. See 'PROCESSINGS1. - pub static ref EOL_FINDER: Regex = Regex::new("[^\r\n]*(\r?\n)").unwrap(); - -} - -// Data structures -// --------------- -#[derive(Clone, Debug, PartialEq)] -pub enum EolType { - Lf, - Crlf, -} - -// Code -// ---- -pub fn find_eol_type(s: &str) -> EolType { - match EOL_FINDER.captures(s) { - // Assume a line type for strings with no newlines. - None => { - if cfg!(windows) { - EolType::Crlf - } else { - EolType::Lf - } - } - Some(captures) => match captures.get(1) { - None => panic!("No capture group!"), - Some(match_) => { - if match_.as_str() == "\n" { - EolType::Lf - } else { - EolType::Crlf - } - } - }, - } -} - -// This is the processing task for the Visual Studio Code IDE. It handles all -// the core logic to moving data between the IDE and the client. -#[get("/vsc/ws-ide/{connection_id}")] -pub async fn vscode_ide_websocket( - connection_id: web::Path, - req: HttpRequest, - body: web::Payload, - app_state: web::Data, -) -> Result { - let connection_id_str = connection_id.to_string(); - - // There are three cases for this `connection_id`: - // - // 1. It hasn't been used before. In this case, create the appropriate - // queues and start websocket and processing tasks. - // 2. It's in use, but was disconnected. In this case, re-use the queues - // and start the websocket task; the processing task is still running. - // 3. It's in use by another IDE. This is an error, but I don't have a way - // to detect it yet. - // - // Check case 3. - if app_state - .vscode_connection_id - .lock() - .unwrap() - .contains(&connection_id_str) - { - let msg = format!("Connection ID {connection_id_str} already in use."); - error!("{msg}"); - return Err(ErrorBadRequest(msg)); - } - - // Now case 2. - if app_state - .vscode_ide_queues - .lock() - .unwrap() - .contains_key(&connection_id_str) - { - return client_websocket( - connection_id, - req, - body, - app_state.vscode_ide_queues.clone(), - ) - .await; - } - - // Then this is case 1. Add the connection ID to the list of active - // connections. - let (from_ide_tx, mut from_ide_rx) = mpsc::channel(10); - let (to_ide_tx, to_ide_rx) = mpsc::channel(10); - assert!( - app_state - .vscode_ide_queues - .lock() - .unwrap() - .insert( - connection_id_str.clone(), - WebsocketQueues { - from_websocket_tx: from_ide_tx, - to_websocket_rx: to_ide_rx, - }, - ) - .is_none() - ); - let (from_client_tx, mut from_client_rx) = mpsc::channel(10); - let (to_client_tx, to_client_rx) = mpsc::channel(10); - assert!( - app_state - .vscode_client_queues - .lock() - .unwrap() - .insert( - connection_id_str.clone(), - WebsocketQueues { - from_websocket_tx: from_client_tx, - to_websocket_rx: to_client_rx, - }, - ) - .is_none() - ); - app_state - .vscode_connection_id - .lock() - .unwrap() - .insert(connection_id_str.clone()); - - // Clone variables owned by the processing task. - let connection_id_task = connection_id_str; - let app_state_task = app_state.clone(); - - // Start the processing task. - actix_rt::spawn(async move { - // Use a [labeled block - // expression](https://doc.rust-lang.org/reference/expressions/loop-expr.html#labelled-block-expressions) - // to provide a way to exit the current task. - 'task: { - let mut current_file = PathBuf::new(); - let mut load_file_requests: HashMap = HashMap::new(); - debug!("VSCode processing task started."); - - // Get the first message sent by the IDE. - let Some(first_message): std::option::Option = from_ide_rx.recv().await - else { - error!("{}", "IDE websocket received no data."); - break 'task; - }; - - // Make sure it's the `Opened` message. - let EditorMessageContents::Opened(ide_type) = first_message.message else { - let msg = format!("Unexpected message {first_message:?}"); - error!("{msg}"); - send_response(&to_ide_tx, first_message.id, Err(msg)).await; - - // Send a `Closed` message to shut down the websocket. - queue_send!(to_ide_tx.send(EditorMessage { id: 0.0, message: EditorMessageContents::Closed}), 'task); - break 'task; - }; - debug!("Received IDE Opened message."); - - // Ensure the IDE type (VSCode) is correct. - match ide_type { - IdeType::VSCode(is_self_hosted) => { - // Get the address for the server. - let port = app_state_task.port; - let address = match get_server_url(port).await { - Ok(address) => address, - Err(err) => { - error!("{err:?}"); - break 'task; - } - }; - if is_self_hosted { - // Send a response (successful) to the `Opened` message. - debug!( - "Sending response = OK to IDE Opened message, id {}.", - first_message.id - ); - send_response(&to_ide_tx, first_message.id, Ok(ResultOkTypes::Void)).await; - - // Send the HTML for the internal browser. - let client_html = formatdoc!( - r#" - - - - - - - - "# - ); - debug!("Sending ClientHtml message to IDE: {client_html}"); - queue_send!(to_ide_tx.send(EditorMessage { - id: 0.0, - message: EditorMessageContents::ClientHtml(client_html) - }), 'task); - - // Wait for the response. - let Some(message) = from_ide_rx.recv().await else { - error!("{}", "IDE websocket received no data."); - break 'task; - }; - - // Make sure it's the `Result` message with no errors. - let res = - // First, make sure the ID matches. - if message.id != 0.0 { - Err(format!("Unexpected message ID {}.", message.id)) - } else { - match message.message { - EditorMessageContents::Result(message_result) => match message_result { - Err(err) => Err(format!("Error in ClientHtml: {err}")), - Ok(result_ok) => - if let ResultOkTypes::Void = result_ok { - Ok(()) - } else { - Err(format!( - "Unexpected message LoadFile contents {result_ok:?}." - )) - } - }, - _ => Err(format!("Unexpected message {message:?}")), - } - }; - if let Err(err) = res { - error!("{err}"); - // Send a `Closed` message. - queue_send!(to_ide_tx.send(EditorMessage { - id: 1.0, - message: EditorMessageContents::Closed - }), 'task); - break 'task; - }; - } else { - // Open the Client in an external browser. - if let Err(err) = - webbrowser::open(&format!("{address}/vsc/cf/{connection_id_task}")) - { - let msg = format!("Unable to open web browser: {err}"); - error!("{msg}"); - send_response(&to_ide_tx, first_message.id, Err(msg)).await; - - // Send a `Closed` message. - queue_send!(to_ide_tx.send(EditorMessage{ - id: 0.0, - message: EditorMessageContents::Closed - }), 'task); - break 'task; - } - // Send a response (successful) to the `Opened` message. - send_response(&to_ide_tx, first_message.id, Ok(ResultOkTypes::Void)).await; - } - } - _ => { - // This is the wrong IDE type. Report then error. - let msg = format!("Invalid IDE type: {ide_type:?}"); - error!("{msg}"); - send_response(&to_ide_tx, first_message.id, Err(msg)).await; - - // Close the connection. - queue_send!(to_ide_tx.send(EditorMessage { id: 0.0, message: EditorMessageContents::Closed}), 'task); - break 'task; - } - } - - // Create a queue for HTTP requests fo communicate with this task. - let (from_http_tx, mut from_http_rx) = mpsc::channel(10); - app_state_task - .processing_task_queue_tx - .lock() - .unwrap() - .insert(connection_id_task.to_string(), from_http_tx); - - // All further messages are handled in the main loop. - let mut id: f64 = INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT; - let mut source_code = String::new(); - let mut code_mirror_doc = String::new(); - // The initial state will be overwritten by the first `Update` or - // `LoadFile`, so this value doesn't matter. - let mut eol = EolType::Lf; - // Some means this contains valid HTML; None means don't use it - // (since it would have contained Markdown). - let mut code_mirror_doc_blocks = Some(Vec::new()); - // To send a diff from Server to Client or vice versa, we need to - // ensure they are in sync: - // - // 1. IDE update -> Server -> Client or Client update -> Server -> - // IDE: the Server and Client sync is pending. Client response - // -> Server -> IDE or IDE response -> Server -> Client: the - // Server and Client are synced. - // 2. IDE current file -> Server -> Client or Client current file - // -> Server -> IDE: Out of sync. - // - // It's only safe to send a diff when the most recent sync is - // achieved. So, we need to track the ID of the most recent IDE -> - // Client update or Client -> IDE update, if one is in flight. When - // complete, mark the connection as synchronized. Since all IDs are - // unique, we can use a single variable to store the ID. - // - // Currently, when the Client sends an update, mark the connection - // as out of sync, since the update contains not HTML in the doc - // blocks, but Markdown. When Turndown is moved from JavaScript to - // Rust, this can be changed, since both sides will have HTML in the - // doc blocks. - let mut sync_state = SyncState::OutOfSync; - loop { - select! { - // Look for messages from the IDE. - Some(ide_message) = from_ide_rx.recv() => { - let msg = format!("{:?}", ide_message.message); - debug!("Received IDE message id = {}, message = {}", ide_message.id, &msg[..min(MAX_MESSAGE_LENGTH, msg.len())]); - match ide_message.message { - // Handle messages that the IDE must not send. - EditorMessageContents::Opened(_) | - EditorMessageContents::OpenUrl(_) | - EditorMessageContents::LoadFile(_) | - EditorMessageContents::ClientHtml(_) => { - let msg = "IDE must not send this message."; - error!("{msg}"); - send_response(&to_ide_tx, ide_message.id, Err(msg.to_string())).await; - }, - - // Handle messages that are simply passed through. - EditorMessageContents::Closed | - EditorMessageContents::RequestClose => { - debug!("Forwarding it to the Client."); - queue_send!(to_client_tx.send(ide_message)) - }, - - // Pass a `Result` message to the Client, unless - // it's a `LoadFile` result. - EditorMessageContents::Result(ref result) => { - let is_loadfile = match result { - // See if this error was produced by a - // `LoadFile` result. - Err(_) => load_file_requests.contains_key(&ide_message.id.to_bits()), - Ok(result_ok) => match result_ok { - ResultOkTypes::Void => false, - ResultOkTypes::LoadFile(_) => true, - } - }; - // Pass the message to the client if this isn't - // a `LoadFile` result (the only type of result - // which the Server should handle). - if !is_loadfile { - debug!("Forwarding it to the Client."); - // If this was confirmation from the IDE - // that it received the latest update, then - // mark the IDE as synced. - if sync_state == SyncState::Pending(ide_message.id) { - sync_state = SyncState::InSync; - } - queue_send!(to_client_tx.send(ide_message)); - continue; - } - // Ensure there's an HTTP request for this - // `LoadFile` result. - let Some(http_request) = load_file_requests.remove(&ide_message.id.to_bits()) else { - error!("Error: no HTTP request found for LoadFile result ID {}.", ide_message.id); - break 'task; - }; - - // Take ownership of the result after sending it - // above (which requires ownership). - let EditorMessageContents::Result(result) = ide_message.message else { - error!("{}", "Not a result."); - break; - }; - // Get the file contents from a `LoadFile` - // result; otherwise, this is None. - let file_contents_option = match result { - Err(err) => { - error!("{err}"); - None - }, - Ok(result_ok) => match result_ok { - ResultOkTypes::Void => panic!("LoadFile result should not be void."), - ResultOkTypes::LoadFile(file_contents) => file_contents, - } - }; - - // Process the file contents. Since VSCode - // doesn't have a PDF viewer, determine if this - // is a PDF file. (TODO: look at the magic - // number also -- "%PDF"). - let use_pdf_js = http_request.file_path.extension() == Some(OsStr::new("pdf")); - let (simple_http_response, option_update, file_contents) = match file_contents_option { - Some(file_contents) => { - // If there are Windows newlines, replace - // with Unix; this is reversed when the - // file is sent back to the IDE. - eol = find_eol_type(&file_contents); - let file_contents = if use_pdf_js { file_contents } else { file_contents.replace("\r\n", "\n") }; - file_to_response(&http_request, ¤t_file, Some(file_contents), use_pdf_js).await - }, - None => { - // The file wasn't available in the IDE. - // Look for it in the filesystem. - match File::open(&http_request.file_path).await { - Err(err) => ( - SimpleHttpResponse::Err(SimpleHttpResponseError::Io(err)), - None, - None - ), - Ok(mut fc) => { - let option_file_contents = try_read_as_text(&mut fc).await; - let option_file_contents = if let Some(file_contents) = option_file_contents { - eol = find_eol_type(&file_contents); - let file_contents = if use_pdf_js { file_contents } else { file_contents.replace("\r\n", "\n") }; - Some(file_contents) - } else { - None - }; - // If this - // is a binary file (meaning we can't read - // the contents as UTF-8), send the - // contents as none to signal this isn't a - // text file. - file_to_response( - &http_request, - ¤t_file, - option_file_contents, - use_pdf_js, - ) - .await - } - } - } - }; - if let Some(update) = option_update { - let Some(ref tmp) = update.contents else { - error!("None."); - break; - }; - let CodeMirrorDiffable::Plain(ref plain) = tmp.source else { - error!("Not plain!"); - break; - }; - // We must clone here, since the original is - // placed in the TX queue. - source_code = file_contents.unwrap(); - code_mirror_doc = plain.doc.clone(); - code_mirror_doc_blocks = Some(plain.doc_blocks.clone()); - sync_state = SyncState::Pending(id); - - debug!("Sending Update to Client, id = {id}."); - queue_send!(to_client_tx.send(EditorMessage { - id, - message: EditorMessageContents::Update(update) - })); - id += MESSAGE_ID_INCREMENT; - } - debug!("Sending HTTP response."); - oneshot_send!(http_request.response_queue.send(simple_http_response)); - } - - // Handle the `Update` message. - EditorMessageContents::Update(update) => { - // Normalize the provided file name. - let result = match try_canonicalize(&update.file_path) { - Err(err) => Err(err), - Ok(clean_file_path) => { - match update.contents { - None => Err("TODO: support for updates without contents.".to_string()), - Some(contents) => { - match contents.source { - CodeMirrorDiffable::Diff(_diff) => Err("TODO: support for updates with diffable sources.".to_string()), - CodeMirrorDiffable::Plain(code_mirror) => { - // If there are Windows newlines, replace - // with Unix; this is reversed when the - // file is sent back to the IDE. - eol = find_eol_type(&code_mirror.doc); - let doc_normalized_eols = code_mirror.doc.replace("\r\n", "\n"); - // Translate the file. - let (translation_results_string, _path_to_toc) = - source_to_codechat_for_web_string(&doc_normalized_eols, ¤t_file, false); - match translation_results_string { - TranslationResultsString::CodeChat(ccfw) => { - // Send the new translated contents. - debug!("Sending translated contents to Client."); - let CodeMirrorDiffable::Plain(ref ccfw_source_plain) = ccfw.source else { - error!("{}", "Unexpected diff value."); - break; - }; - // Send a diff if possible (only when the - // Client's contents are synced with the - // IDE). - let contents = Some( - if let Some(cmdb) = code_mirror_doc_blocks && - sync_state == SyncState::InSync { - let doc_diff = diff_str(&code_mirror_doc, &ccfw_source_plain.doc); - let code_mirror_diff = diff_code_mirror_doc_blocks(&cmdb, &ccfw_source_plain.doc_blocks); - CodeChatForWeb { - // Clone needed here, so we can copy it - // later. - metadata: ccfw.metadata.clone(), - source: CodeMirrorDiffable::Diff(CodeMirrorDiff { - doc: doc_diff, - doc_blocks: code_mirror_diff - }) - } - } else { - // We must make a clone to put in the TX - // queue; this allows us to keep the - // original below to use with the next - // diff. - ccfw.clone() - } - ); - queue_send!(to_client_tx.send(EditorMessage { - id: ide_message.id, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), - contents, - cursor_position: None, - scroll_position: None, - }), - })); - // Update to the latest code after - // computing diffs. To avoid ownership - // problems, re-define `ccfw_source_plain`. - let CodeMirrorDiffable::Plain(ccfw_source_plain) = ccfw.source else { - error!("{}", "Unexpected diff value."); - break; - }; - source_code = code_mirror.doc; - code_mirror_doc = ccfw_source_plain.doc; - code_mirror_doc_blocks = Some(ccfw_source_plain.doc_blocks); - // Mark the Client as unsynced until this - // is acknowledged. - sync_state = SyncState::Pending(ide_message.id); - Ok(ResultOkTypes::Void) - } - // TODO - TranslationResultsString::Binary => Err("TODO".to_string()), - TranslationResultsString::Err(err) => Err(format!("Error translating source to CodeChat: {err}").to_string()), - TranslationResultsString::Unknown => { - // Send the new raw contents. - debug!("Sending translated contents to Client."); - queue_send!(to_client_tx.send(EditorMessage { - id: ide_message.id, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), - contents: Some(CodeChatForWeb { - metadata: SourceFileMetadata { - // Since this is raw data, `mode` doesn't - // matter. - mode: "".to_string(), - }, - source: CodeMirrorDiffable::Plain(CodeMirror { - doc: code_mirror.doc, - doc_blocks: vec![] - }) - }), - cursor_position: None, - scroll_position: None, - }), - })); - Ok(ResultOkTypes::Void) - }, - TranslationResultsString::Toc(_) => { - Err("Error: source incorrectly recognized as a TOC.".to_string()) - } - } - } - } - } - } - } - }; - // If there's an error, then report it; - // otherwise, the message is passed to the - // Client, which will provide the result. - if let Err(err) = &result { - error!("{err}"); - send_response(&to_ide_tx, ide_message.id, result).await; - } - } - - // Update the current file; translate it to a URL - // then pass it to the Client. - EditorMessageContents::CurrentFile(file_path, _is_text) => { - debug!("Translating and forwarding it to the Client."); - match try_canonicalize(&file_path) { - Ok(clean_file_path) => { - queue_send!(to_client_tx.send(EditorMessage { - id: ide_message.id, - message: EditorMessageContents::CurrentFile( - path_to_url("/vsc/fs", Some(&connection_id_task), &clean_file_path), Some(true) - ) - })); - current_file = file_path.into(); - // Since this is a new file, mark it as - // unsynced. - sync_state = SyncState::OutOfSync; - } - Err(err) => { - let msg = format!( - "Unable to canonicalize file name {}: {err}", &file_path - ); - error!("{msg}"); - send_response(&to_client_tx, ide_message.id, Err(msg)).await; - } - } - } - } - }, - - // Handle HTTP requests. - Some(http_request) = from_http_rx.recv() => { - debug!("Received HTTP request for {:?} and sending LoadFile to IDE, id = {id}.", http_request.file_path); - // Convert the request into a `LoadFile` message. - queue_send!(to_ide_tx.send(EditorMessage { - id, - message: EditorMessageContents::LoadFile(http_request.file_path.clone()) - })); - // Store the ID and request, which are needed to send a - // response when the `LoadFile` result is received. - load_file_requests.insert(id.to_bits(), http_request); - id += MESSAGE_ID_INCREMENT; - } - - // Handle messages from the client. - Some(client_message) = from_client_rx.recv() => { - let msg = format!("{:?}", client_message.message); - debug!("Received Client message id = {}, message = {}", client_message.id, &msg[..min(MAX_MESSAGE_LENGTH, msg.len())]); - match client_message.message { - // Handle messages that the client must not send. - EditorMessageContents::Opened(_) | - EditorMessageContents::LoadFile(_) | - EditorMessageContents::RequestClose | - EditorMessageContents::ClientHtml(_) => { - let msg = "Client must not send this message."; - error!("{msg}"); - send_response(&to_client_tx, client_message.id, Err(msg.to_string())).await; - }, - - // Handle messages that are simply passed through. - EditorMessageContents::Closed | - EditorMessageContents::Result(_) => { - debug!("Forwarding it to the IDE."); - // If this result confirms that the Client - // received the most recent IDE update, then - // mark the documents as synced. - if sync_state == SyncState::Pending(client_message.id) { - sync_state = SyncState::InSync; - } - queue_send!(to_ide_tx.send(client_message)) - }, - - // Open a web browser when requested. - EditorMessageContents::OpenUrl(url) => { - // This doesn't work in Codespaces. TODO: send - // this back to the VSCode window, then call - // `vscode.env.openExternal(vscode.Uri.parse(url))`. - if let Err(err) = webbrowser::open(&url) { - let msg = format!("Unable to open web browser to URL {url}: {err}"); - error!("{msg}"); - send_response(&to_client_tx, client_message.id, Err(msg)).await; - } else { - send_response(&to_client_tx, client_message.id, Ok(ResultOkTypes::Void)).await; - } - }, - - // Handle the `Update` message. - EditorMessageContents::Update(update_message_contents) => { - debug!("Forwarding translation of it to the IDE."); - match try_canonicalize(&update_message_contents.file_path) { - Err(err) => { - let msg = format!( - "Unable to canonicalize file name {}: {err}", &update_message_contents.file_path - ); - error!("{msg}"); - send_response(&to_client_tx, client_message.id, Err(msg)).await; - continue; - } - Ok(clean_file_path) => { - let codechat_for_web = match update_message_contents.contents { - None => None, - Some(cfw) => match codechat_for_web_to_source( - &cfw) - { - Ok(result) => { - let ccfw = if sync_state == SyncState::InSync { - Some(CodeChatForWeb { - metadata: cfw.metadata, - source: CodeMirrorDiffable::Diff(CodeMirrorDiff { - // Diff with correct EOLs, so that (for - // CRLF files as well as LF files) offsets - // are correct. - doc: diff_str(&eol_convert(source_code, &eol), &eol_convert(result.clone(), &eol)), - doc_blocks: vec![], - }), - }) - } else { - Some(CodeChatForWeb { - metadata: cfw.metadata, - source: CodeMirrorDiffable::Plain(CodeMirror { - // We must clone here, so that it can be - // placed in the TX queue. - doc: eol_convert(result.clone(), &eol), - doc_blocks: vec![], - }), - }) - }; - // Store the document with Unix-style EOLs - // (LFs). - source_code = result; - let CodeMirrorDiffable::Plain(cmd) = cfw.source else { - // TODO: support diffable! - error!("No diff!"); - break; - }; - code_mirror_doc = cmd.doc; - // TODO: instead of `cmd.doc_blocks`, use - // `None` to indicate that the doc blocks - // contain Markdown instead of HTML. - code_mirror_doc_blocks = None; - ccfw - }, - Err(message) => { - let msg = format!( - "Unable to translate to source: {message}" - ); - error!("{msg}"); - send_response(&to_client_tx, client_message.id, Err(msg)).await; - continue; - } - }, - }; - queue_send!(to_ide_tx.send(EditorMessage { - id: client_message.id, - message: EditorMessageContents::Update(UpdateMessageContents { - file_path: clean_file_path.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(), - contents: codechat_for_web, - cursor_position: update_message_contents.cursor_position, - scroll_position: update_message_contents.scroll_position, - }) - })); - // Mark the IDE contents as out of sync - // until this message is received. - sync_state = SyncState::Pending(client_message.id); - } - } - }, - - // Update the current file; translate it to a URL - // then pass it to the IDE. - EditorMessageContents::CurrentFile(url_string, _is_text) => { - debug!("Forwarding translated path to IDE."); - let result = match url_to_path(&url_string, VSCODE_PATH_PREFIX) { - Err(err) => Err(format!("Unable to convert URL to path: {err}")), - Ok(file_path) => { - match file_path.to_str() { - None => Err("Unable to convert path to string.".to_string()), - Some(file_path_string) => { - // Use a [binary file - // sniffer](#binary-file-sniffer) to - // determine if the file is text or binary. - let is_text = if let Ok(mut fc) = File::open(&file_path).await { - try_read_as_text(&mut fc).await.is_some() - } else { - false - }; - queue_send!(to_ide_tx.send(EditorMessage { - id: client_message.id, - message: EditorMessageContents::CurrentFile(file_path_string.to_string(), Some(is_text)) - })); - current_file = file_path; - // Mark the IDE as out of sync, since this - // is a new file. - sync_state = SyncState::OutOfSync; - Ok(()) - } - } - } - }; - if let Err(msg) = result { - error!("{msg}"); - send_response(&to_client_tx, client_message.id, Err(msg)).await; - } - } - } - }, - - else => break - } - } - - debug!("VSCode processing task shutting down."); - if app_state_task - .processing_task_queue_tx - .lock() - .unwrap() - .remove(&connection_id_task) - .is_none() - { - error!( - "Unable to remove connection ID {connection_id_task} from processing task queue." - ); - } - if app_state_task - .vscode_client_queues - .lock() - .unwrap() - .remove(&connection_id_task) - .is_none() - { - error!("Unable to remove connection ID {connection_id_task} from client queues."); - } - if app_state_task - .vscode_ide_queues - .lock() - .unwrap() - .remove(&connection_id_task) - .is_none() - { - error!("Unable to remove connection ID {connection_id_task} from IDE queues."); - } - - from_ide_rx.close(); - from_ide_rx.close(); - - // Drain any remaining messages after closing the queue. - while let Some(m) = from_ide_rx.recv().await { - warn!("Dropped queued message {m:?}"); - } - while let Some(m) = from_client_rx.recv().await { - warn!("Dropped queued message {m:?}"); - } - debug!("VSCode processing task exited."); - } - }); - - // Move data between the IDE and the processing task via queues. The - // websocket connection between the client and the IDE will run in the - // endpoint for that connection. - client_websocket( - connection_id, - req, - body, - app_state.vscode_ide_queues.clone(), - ) - .await -} - -pub fn get_vscode_client_framework(connection_id: &str) -> String { - // Send the HTML for the internal browser. - match get_client_framework(false, "vsc/ws-client", connection_id) { - Ok(web_page) => web_page, - Err(html_string) => { - error!("{html_string}"); - html_wrapper(&escape_html(&html_string)) - } - } -} - -/// Serve the Client Framework. -#[get("/vsc/cf/{connection_id}")] -pub async fn vscode_client_framework(connection_id: web::Path) -> HttpResponse { - HttpResponse::Ok() - .content_type("text/html") - .body(get_vscode_client_framework(&connection_id)) -} - -/// Define a websocket handler for the CodeChat Editor Client. -#[get("/vsc/ws-client/{connection_id}")] -pub async fn vscode_client_websocket( - connection_id: web::Path, - req: HttpRequest, - body: web::Payload, - app_state: web::Data, -) -> Result { - client_websocket( - connection_id, - req, - body, - app_state.vscode_client_queues.clone(), - ) - .await -} - -// Respond to requests for the filesystem. -#[get("/vsc/fs/{connection_id}/{file_path:.*}")] -async fn serve_vscode_fs( - request_path: web::Path<(String, String)>, - req: HttpRequest, - app_state: web::Data, -) -> HttpResponse { - filesystem_endpoint(request_path, &req, &app_state).await -} - -// If a string is encoded using CRLFs (Windows style), convert it to LFs only -// (Unix style). -fn eol_convert(s: String, eol_type: &EolType) -> String { - if eol_type == &EolType::Crlf { - s.replace("\n", "\r\n") - } else { - s - } -} diff --git a/server/tests/fixtures/webserver/filewatcher/tests/test_websocket_opened_1/test.py b/server/tests/fixtures/ide/filewatcher/tests/test_websocket_opened_1/test.py similarity index 100% rename from server/tests/fixtures/webserver/filewatcher/tests/test_websocket_opened_1/test.py rename to server/tests/fixtures/ide/filewatcher/tests/test_websocket_opened_1/test.py diff --git a/server/tests/fixtures/webserver/filewatcher/tests/test_websocket_opened_2/test.py b/server/tests/fixtures/ide/filewatcher/tests/test_websocket_opened_2/test.py similarity index 100% rename from server/tests/fixtures/webserver/filewatcher/tests/test_websocket_opened_2/test.py rename to server/tests/fixtures/ide/filewatcher/tests/test_websocket_opened_2/test.py diff --git a/server/tests/fixtures/webserver/filewatcher/tests/test_websocket_timeout/test.py b/server/tests/fixtures/ide/filewatcher/tests/test_websocket_timeout/test.py similarity index 100% rename from server/tests/fixtures/webserver/filewatcher/tests/test_websocket_timeout/test.py rename to server/tests/fixtures/ide/filewatcher/tests/test_websocket_timeout/test.py diff --git a/server/tests/fixtures/webserver/filewatcher/tests/test_websocket_update_1/test.py b/server/tests/fixtures/ide/filewatcher/tests/test_websocket_update_1/test.py similarity index 100% rename from server/tests/fixtures/webserver/filewatcher/tests/test_websocket_update_1/test.py rename to server/tests/fixtures/ide/filewatcher/tests/test_websocket_update_1/test.py diff --git a/server/tests/fixtures/webserver/filewatcher/tests/test_websocket_update_1/test1.py b/server/tests/fixtures/ide/filewatcher/tests/test_websocket_update_1/test1.py similarity index 100% rename from server/tests/fixtures/webserver/filewatcher/tests/test_websocket_update_1/test1.py rename to server/tests/fixtures/ide/filewatcher/tests/test_websocket_update_1/test1.py diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket3a/test.py b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket3a/test.py similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket3a/test.py rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket3a/test.py diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4/test.py b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4/test.py similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4/test.py rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4/test.py diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4/toc.md b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4/toc.md similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4/toc.md rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4/toc.md diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4a/black.png b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4a/black.png similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4a/black.png rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4a/black.png diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4a/helloworld.pdf b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4a/helloworld.pdf similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4a/helloworld.pdf rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4a/helloworld.pdf diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4b/black.png b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4b/black.png similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4b/black.png rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4b/black.png diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4b/helloworld.pdf b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4b/helloworld.pdf similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4b/helloworld.pdf rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4b/helloworld.pdf diff --git a/server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4b/toc.md b/server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4b/toc.md similarity index 100% rename from server/tests/fixtures/webserver/vscode/tests/test_vscode_ide_websocket4b/toc.md rename to server/tests/fixtures/ide/vscode/tests/test_vscode_ide_websocket4b/toc.md