Store comments as u8s instead of Strings (#839)

Author: reese

librubyfmt/src/comment_block.rs

@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+use std::io::BufRead;
 use std::ops::Range;
 
 use crate::line_tokens::ConcreteLineToken;
@@ -7,19 +9,19 @@ use crate::util::get_indent;
 #[derive(Clone, Debug)]
 pub struct CommentBlock {
     span: Range<LineNumber>,
-    comments: Vec<String>,
+    comments: Vec<Cow<'static, [u8]>>,
 }
 
 impl CommentBlock {
-    pub fn new(span: Range<LineNumber>, comments: Vec<String>) -> Self {
+    pub fn new(span: Range<LineNumber>, comments: Vec<Cow<'static, [u8]>>) -> Self {
         CommentBlock { span, comments }
     }
 
     pub fn following_line_number(&self) -> LineNumber {
         self.span.end
     }
 
-    pub fn add_line(&mut self, line: String) {
+    pub fn add_line(&mut self, line: Cow<'static, [u8]>) {
         self.span.end += 1;
         self.comments.push(line);
     }
@@ -30,7 +32,7 @@ impl CommentBlock {
 
         comments.into_iter().enumerate().flat_map(move |(i, c)| {
             if c.is_empty() {
-                // Empty strings represent blank lines
+                // Empty vecs represent blank lines
                 // If this is a trailing empty comment (at the end), keep it as an empty Comment token
                 // to bypass the HardNewLine deduplication logic. Otherwise convert to just HardNewLine.
                 if i == len - 1 {
@@ -53,12 +55,18 @@ impl CommentBlock {
     }
 
     pub fn apply_spaces(mut self, indent_depth: ColNumber) -> Self {
+        if indent_depth == 0 {
+            return self;
+        }
+
         let indent = get_indent(indent_depth as usize);
         for comment in &mut self.comments {
-            // Ignore empty strings -- these represent blank lines between
+            // Ignore empty vecs -- these represent blank lines between
             // groups of comments
-            if !comment.is_empty() && !comment.starts_with("=begin") {
-                comment.insert_str(0, &indent);
+            if !comment.is_empty() && !comment.starts_with(b"=begin") {
+                comment
+                    .to_mut()
+                    .splice(0..0, indent.as_bytes().iter().copied());
             }
         }
         self

librubyfmt/src/file_comments.rs

@@ -1,9 +1,11 @@
+use std::borrow::Cow;
+use std::io::BufRead;
+
 use memchr::memchr_iter;
 
 use crate::comment_block::CommentBlock;
 use crate::parser_state::line_difference_requires_newline;
 use crate::types::{LineNumber, SourceOffset};
-use crate::util::u8_to_string;
 
 /// A vector of offsets in the source code where lines start, which
 /// we use to detect what line a given offset is one.
@@ -51,7 +53,7 @@ impl LineIndex {
 pub struct FileComments {
     start_of_file_contiguous_comment_lines: Option<CommentBlock>,
     /// A list of comments, sorted in order by `LineNumber`
-    other_comments: Vec<(LineNumber, String)>,
+    other_comments: Vec<(LineNumber, Vec<u8>)>,
     /// Sorted list of line numbers that contain Ruby code (not comments/blank)
     lines_with_ruby: Vec<LineNumber>,
     last_lineno: LineNumber,
@@ -96,7 +98,7 @@ impl FileComments {
         for comment in comments {
             file_comments.push_comment(
                 line_index.get_line_number(comment.location().start_offset()) as u64,
-                u8_to_string(comment.text().trim_ascii_end()),
+                comment.text().trim_ascii_end().to_vec(),
             );
             file_comments
                 .comment_start_offsets
@@ -146,7 +148,7 @@ impl FileComments {
     /// Add a new comment. If the beginning of this file is a comment block,
     /// each of those comment lines must be pushed before any other line, or
     /// the end of the block from the start of the file will be incorrectly calculated.
-    fn push_comment(&mut self, line_number: u64, l: String) {
+    fn push_comment(&mut self, line_number: u64, l: Vec<u8>) {
         match (
             &mut self.start_of_file_contiguous_comment_lines,
             line_number,
@@ -158,10 +160,10 @@ impl FileComments {
                      otherwise we won't know where the last line is",
                 );
                 self.start_of_file_contiguous_comment_lines =
-                    Some(CommentBlock::new(1..2, vec![l]));
+                    Some(CommentBlock::new(1..2, vec![l.into()]));
             }
             (Some(sled), _) if sled.following_line_number() == line_number => {
-                sled.add_line(l);
+                sled.add_line(l.into());
             }
             _ => {
                 debug_assert!(
@@ -220,21 +222,22 @@ impl FileComments {
             .other_comments
             .partition_point(|(ln, _)| *ln <= line_number);
 
-        let mut comment_block_with_spaces: Vec<String> = Vec::new();
+        let mut comment_block_with_spaces = Vec::new();
         let mut last_line = None;
 
         if line_difference_requires_newline(
             self.other_comments.first().unwrap().0,
             starting_line_number,
         ) {
-            comment_block_with_spaces.push(String::new());
+            comment_block_with_spaces.push(b"".into());
         }
 
         for (index, comment_contents) in self.other_comments.drain(..split_point) {
+            let comment_contents: Cow<'_, [u8]> = Cow::Owned(comment_contents);
             if let Some(last_line) = last_line
                 && line_difference_requires_newline(index, last_line)
             {
-                comment_block_with_spaces.push(String::new());
+                comment_block_with_spaces.push(b"".into());
             }
             let line_count = comment_contents.lines().count() as u64;
             last_line = Some(index + line_count - 1);
@@ -243,7 +246,7 @@ impl FileComments {
 
         if line_number > last_line.unwrap() + 1 {
             last_line = Some(line_number);
-            comment_block_with_spaces.push(String::new());
+            comment_block_with_spaces.push(b"".into());
         }
 
         Some((

librubyfmt/src/line_tokens.rs

@@ -69,7 +69,7 @@ pub enum ConcreteLineToken<'src> {
     },
     SingleSlash,
     Comment {
-        contents: String,
+        contents: Cow<'src, [u8]>,
     },
     Delim {
         contents: &'static str,
@@ -127,7 +127,7 @@ impl<'src> ConcreteLineToken<'src> {
                 Cow::Owned(s) => Cow::Owned(s.into_bytes()),
             },
             Self::SingleSlash => Cow::Borrowed(b"\\"),
-            Self::Comment { contents } => Cow::Owned(contents.into()),
+            Self::Comment { contents } => contents,
             Self::Delim { contents } => Cow::Borrowed(contents.as_bytes()),
             Self::End => Cow::Borrowed(b"end"),
             Self::HeredocClose { symbol } => Cow::Owned(symbol.into()),
@@ -157,9 +157,11 @@ impl<'src> ConcreteLineToken<'src> {
             MethodName { name: op } => op.len(),
             DirectPart { part } => part.len(),
             LTStringContent { content } => content.len(),
-            Comment { contents }
-            | HeredocClose { symbol: contents }
-            | RawHeredocContent { content: contents } => contents.len(),
+            Comment { contents } => contents.len(),
+            HeredocClose { symbol: contents } | RawHeredocContent { content: contents } => {
+                contents.len()
+            }
+
             HardNewLine | Comma | Space | Dot | OpenSquareBracket | CloseSquareBracket
             | OpenCurlyBracket | CloseCurlyBracket | OpenParen | CloseParen | SingleSlash
             | DoubleQuote => 1,

librubyfmt/src/parser_state.rs

@@ -471,7 +471,7 @@ impl<'src> ParserState<'src> {
                 && let Some(comments) = self.comments_to_insert.as_mut()
                 && comments.line_count() > 0
             {
-                comments.add_line("".to_string());
+                comments.add_line(b"".into());
             }
             self.on_line(self.current_orig_line_number + 1);
         }

librubyfmt/src/render_queue_writer.rs

@@ -65,14 +65,12 @@ impl<'src> RenderQueueWriter<'src> {
                     contents,
                 }) => {
                     if !contents.is_empty() {
-                        let new_contents = format!(
-                            "{}{}",
-                            get_indent(accum.additional_indent as usize * 2),
-                            contents
-                        );
+                        let indent = get_indent(accum.additional_indent as usize * 2);
+                        let mut new_contents = indent.as_bytes().to_vec();
+                        new_contents.extend_from_slice(contents);
                         next_token = ConcreteLineTokenAndTargets::ConcreteLineToken(
                             ConcreteLineToken::Comment {
-                                contents: new_contents,
+                                contents: new_contents.into(),
                             },
                         )
                     }

librubyfmt/src/util.rs

@@ -17,14 +17,10 @@ pub fn get_indent(depth: usize) -> Cow<'static, str> {
     }
 }
 
-pub fn u8_to_str(arr: &[u8]) -> &str {
+fn u8_to_str(arr: &[u8]) -> &str {
     std::str::from_utf8(arr).unwrap()
 }
 
-pub fn u8_to_string(arr: &[u8]) -> String {
-    u8_to_str(arr).to_string()
-}
-
 pub fn loc_to_str(loc: Location<'_>) -> &str {
     u8_to_str(loc.as_slice())
 }