Fix: translate EOLs when loading a file from disk.

Author: bjones1

docs/changelog.md

@@ -23,6 +23,7 @@ Changelog
     *   Fix indexing in diffs for characters that use more than one UTF-16 code
         unit, such as 😄,👉🏿,👨‍👦, and 🇺🇳.
     *   Fix data corruption with adjacent doc blocks.
+    *   Translate line endings when loading a file from disk.
 *   v0.1.23, 2025-Jul-24
     *   Correct diff errors in IDE with CRLF line endings.
     *   Upgrade to newest release of MathJax, TinyMCE.

server/src/webserver.rs

@@ -701,6 +701,8 @@ async fn make_simple_http_response(
     // If this file is currently being edited, this is the body of an `Update`
     // message to send.
     Option<UpdateMessageContents>,
+    // The resulting file contents, if this is a CodeChat Editor file
+    Option<String>
 ) {
     // Convert the provided URL back into a file name.
     let file_path = &http_request.file_path;
@@ -710,6 +712,7 @@ async fn make_simple_http_response(
         Err(err) => (
             SimpleHttpResponse::Err(SimpleHttpResponseError::Io(err)),
             None,
+            None
         ),
         Ok(mut fc) => {
             let file_contents = try_read_as_text(&mut fc).await;
@@ -719,7 +722,7 @@ async fn make_simple_http_response(
             file_to_response(
                 http_request,
                 current_filepath,
-                file_contents.as_deref(),
+                file_contents,
                 use_pdf_js,
             )
             .await
@@ -755,7 +758,7 @@ async fn file_to_response(
     // `try_canonicalize`.
     current_filepath: &Path,
     // Contents of this file, if it's text; None if it was binary data.
-    file_contents: Option<&str>,
+    file_contents: Option<String>,
     // True to use the PDF.js viewer for this file.
     use_pdf_js: bool,
 ) -> (
@@ -765,6 +768,8 @@ async fn file_to_response(
     // populate the Client with the parsed `file_contents`. In all other cases,
     // return None.
     Option<UpdateMessageContents>,
+    // The `file_contents` if this is a
+    Option<String>
 ) {
     // Use a lossy conversion, since this is UI display, not filesystem access.
     let file_path = &http_request.file_path;
@@ -774,6 +779,7 @@ async fn file_to_response(
                 file_path.to_path_buf(),
             )),
             None,
+            None
         );
     };
     let name = escape_html(&file_name.to_string_lossy());
@@ -791,6 +797,7 @@ async fn file_to_response(
                 codechat_editor_js_name,
             )),
             None,
+            None
         );
     };
     let codechat_editor_css_name = format!("CodeChatEditor{js_test_suffix}.css");
@@ -800,17 +807,18 @@ async fn file_to_response(
                 codechat_editor_css_name,
             )),
             None,
+            file_contents
         );
     };
 
     // Compare these files, since both have been canonicalized by
     // `try_canonical`.
     let is_current_file = file_path == current_filepath;
     let is_toc = http_request.flags == ProcessingTaskHttpRequestFlags::Toc;
-    let (translation_results_string, path_to_toc) = if let Some(file_contents_text) = file_contents
+    let (translation_results_string, path_to_toc) = if let Some(ref file_contents_text) = file_contents
     {
         if is_current_file || is_toc {
-            source_to_codechat_for_web_string(file_contents_text, file_path, is_toc)
+            source_to_codechat_for_web_string(&file_contents_text, file_path, is_toc)
         } else {
             // If this isn't the current file, then don't parse it.
             (TranslationResultsString::Unknown, None)
@@ -852,6 +860,7 @@ async fn file_to_response(
                     file_name,
                 ))),
                 None,
+                None
             );
         };
         return (
@@ -871,13 +880,14 @@ async fn file_to_response(
                 },
             ),
             None,
+            None
         );
     }
 
     let codechat_for_web = match translation_results_string {
         // The file type is binary. Ask the HTTP server to serve it raw.
         TranslationResultsString::Binary => return
-            (SimpleHttpResponse::Bin(file_path.to_path_buf()), None)
+            (SimpleHttpResponse::Bin(file_path.to_path_buf()), None, None)
         ,
         // The file type is unknown. Serve it raw.
         TranslationResultsString::Unknown => {
@@ -887,11 +897,12 @@ async fn file_to_response(
                     mime_guess::from_path(file_path).first_or_text_plain(),
                 ),
                 None,
+                None
             );
         }
         // Report a lexer error.
         TranslationResultsString::Err(err_string) => {
-            return (SimpleHttpResponse::Err(SimpleHttpResponseError::LexerError(err_string)), None);
+            return (SimpleHttpResponse::Err(SimpleHttpResponseError::LexerError(err_string)), None, None);
         }
         // This is a CodeChat file. The following code wraps the CodeChat for
         // web results in a CodeChat Editor Client webpage.
@@ -919,6 +930,7 @@ async fn file_to_response(
                     </html>"#,
                 )),
                 None,
+                None
             );
         }
     };
@@ -940,6 +952,7 @@ async fn file_to_response(
                 file_path.to_path_buf(),
             )),
             None,
+            None
         );
     };
     let dir = path_display(raw_dir);
@@ -949,6 +962,7 @@ async fn file_to_response(
                 file_path.to_path_buf(),
             )),
             None,
+            None
         );
     };
     // Build and return the webpage.
@@ -996,6 +1010,7 @@ async fn file_to_response(
             cursor_position: None,
             scroll_position: None,
         }),
+        file_contents
     )
 }
 

server/src/webserver/filewatcher.rs

@@ -534,7 +534,7 @@ async fn processing_task(file_path: &Path, app_state: web::Data<AppState>, conne
                         // file, which will still produce an error.
                         let empty_path = PathBuf::new();
                         let cfp = current_filepath.as_ref().unwrap_or(&empty_path);
-                        let (simple_http_response, option_update) = make_simple_http_response(&http_request, cfp, false).await;
+                        let (simple_http_response, option_update, _) = make_simple_http_response(&http_request, cfp, false).await;
                         if let Some(update) = option_update {
                             // Send the update to the client.
                             queue_send!(to_websocket_tx.send(EditorMessage {

server/src/webserver/vscode.rs

@@ -55,9 +55,9 @@ use crate::{
     queue_send,
     webserver::{
         INITIAL_MESSAGE_ID, MESSAGE_ID_INCREMENT, ProcessingTaskHttpRequest, ResultOkTypes,
-        SyncState, UpdateMessageContents, escape_html, file_to_response, filesystem_endpoint,
-        get_server_url, html_wrapper, make_simple_http_response, path_to_url, try_canonicalize,
-        try_read_as_text, url_to_path,
+        SimpleHttpResponse, SimpleHttpResponseError, SyncState, UpdateMessageContents, escape_html,
+        file_to_response, filesystem_endpoint, get_server_url, html_wrapper, path_to_url,
+        try_canonicalize, try_read_as_text, url_to_path,
     },
 };
 
@@ -454,21 +454,47 @@ pub async fn vscode_ide_websocket(
                                 // is a PDF file. (TODO: look at the magic
                                 // number also -- "%PDF").
                                 let use_pdf_js = http_request.file_path.extension() == Some(OsStr::new("pdf"));
-                                let (simple_http_response, option_update) = match file_contents_option {
+                                let (simple_http_response, option_update, file_contents) = match file_contents_option {
                                     Some(file_contents) => {
                                         // If there are Windows newlines, replace
                                         // with Unix; this is reversed when the
                                         // file is sent back to the IDE.
                                         eol = find_eol_type(&file_contents);
-                                        let file_contents = file_contents.replace("\r\n", "\n");
-                                        let ret = file_to_response(&http_request, &current_file, Some(&file_contents), use_pdf_js).await;
-                                        source_code = file_contents;
-                                        ret
+                                        let file_contents = if use_pdf_js { file_contents } else { file_contents.replace("\r\n", "\n") };
+                                        file_to_response(&http_request, &current_file, Some(file_contents), use_pdf_js).await
                                     },
                                     None => {
                                         // The file wasn't available in the IDE.
                                         // Look for it in the filesystem.
-                                        make_simple_http_response(&http_request, &current_file, use_pdf_js).await
+                                        match File::open(&http_request.file_path).await {
+                                            Err(err) => (
+                                                SimpleHttpResponse::Err(SimpleHttpResponseError::Io(err)),
+                                                None,
+                                                None
+                                            ),
+                                            Ok(mut fc) => {
+                                                let option_file_contents = try_read_as_text(&mut fc).await;
+                                                let option_file_contents = if let Some(file_contents) = option_file_contents {
+                                                    eol = find_eol_type(&file_contents);
+                                                    let file_contents = if use_pdf_js { file_contents } else { file_contents.replace("\r\n", "\n") };
+                                                    Some(file_contents)
+                                                } else {
+                                                    None
+                                                };
+                                                // <a id="binary-file-sniffer"></a>If this
+                                                // is a binary file (meaning we can't read
+                                                // the contents as UTF-8), send the
+                                                // contents as none to signal this isn't a
+                                                // text file.
+                                                file_to_response(
+                                                    &http_request,
+                                                    &current_file,
+                                                    option_file_contents,
+                                                    use_pdf_js,
+                                                )
+                                                .await
+                                            }
+                                        }
                                     }
                                 };
                                 if let Some(update) = option_update {
@@ -482,6 +508,7 @@ pub async fn vscode_ide_websocket(
                                     };
                                     // We must clone here, since the original is
                                     // placed in the TX queue.
+                                    source_code = file_contents.unwrap();
                                     code_mirror_doc = plain.doc.clone();
                                     code_mirror_doc_blocks = Some(plain.doc_blocks.clone());
                                     sync_state = SyncState::Pending(id);

server/src/webserver/vscode/tests.rs

@@ -831,6 +831,7 @@ async fn test_vscode_ide_websocket4() {
     // Message ids: IDE - 4, Server - 3, Client - 2->5.
     let file_path_temp = fs::canonicalize(test_dir.join("test.py")).unwrap();
     let file_path = simplified(&file_path_temp);
+    let file_path_str = file_path.to_str().unwrap().to_string();
     send_message(
         &mut ws_client,
         &EditorMessage {
@@ -849,10 +850,7 @@ async fn test_vscode_ide_websocket4() {
         read_message(&mut ws_ide).await,
         EditorMessage {
             id: 2.0,
-            message: EditorMessageContents::CurrentFile(
-                file_path.to_str().unwrap().to_string(),
-                Some(true)
-            )
+            message: EditorMessageContents::CurrentFile(file_path_str.clone(), Some(true))
         }
     );
 
@@ -917,7 +915,7 @@ async fn test_vscode_ide_websocket4() {
         EditorMessage {
             id: 6.0,
             message: EditorMessageContents::Update(UpdateMessageContents {
-                file_path: file_path.to_str().unwrap().to_string(),
+                file_path: file_path_str.clone(),
                 contents: Some(CodeChatForWeb {
                     metadata: SourceFileMetadata {
                         mode: "python".to_string(),
@@ -954,6 +952,76 @@ async fn test_vscode_ide_websocket4() {
         }
     );
 
+    // Send an update from the Client, which should produce a diff.
+    //
+    // Message ids: IDE - 4, Server - 9, Client - 5->8.
+    send_message(
+        &mut ws_client,
+        &EditorMessage {
+            id: 5.0,
+            message: EditorMessageContents::Update(UpdateMessageContents {
+                file_path: file_path_str.clone(),
+                contents: Some(CodeChatForWeb {
+                    metadata: SourceFileMetadata {
+                        mode: "python".to_string(),
+                    },
+                    source: CodeMirrorDiffable::Plain(CodeMirror {
+                        doc: "More\n".to_string(),
+                        doc_blocks: vec![CodeMirrorDocBlock {
+                            from: 5,
+                            to: 6,
+                            indent: "".to_string(),
+                            delimiter: "#".to_string(),
+                            contents: "test.py".to_string(),
+                        }],
+                    }),
+                }),
+                cursor_position: None,
+                scroll_position: None,
+            }),
+        },
+    )
+    .await;
+    assert_eq!(
+        read_message(&mut ws_ide).await,
+        EditorMessage {
+            id: 5.0,
+            message: EditorMessageContents::Update(UpdateMessageContents {
+                file_path: file_path_str.clone(),
+                contents: Some(CodeChatForWeb {
+                    metadata: SourceFileMetadata {
+                        mode: "python".to_string(),
+                    },
+                    source: CodeMirrorDiffable::Diff(CodeMirrorDiff {
+                        doc: vec![StringDiff {
+                            from: 0,
+                            to: None,
+                            insert: format!("More{}", if cfg!(windows) { "\r\n" } else { "\n" }),
+                        }],
+                        doc_blocks: vec![],
+                    }),
+                }),
+                cursor_position: None,
+                scroll_position: None,
+            })
+        }
+    );
+    send_message(
+        &mut ws_ide,
+        &EditorMessage {
+            id: 5.0,
+            message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+        },
+    )
+    .await;
+    assert_eq!(
+        read_message(&mut ws_client).await,
+        EditorMessage {
+            id: 5.0,
+            message: EditorMessageContents::Result(Ok(ResultOkTypes::Void))
+        }
+    );
+
     check_logger_errors(0);
     // Report any errors produced when removing the temporary directory.
     temp_dir.close().unwrap();