Simplify the way character references are resolved (#643)

* Simplify the way character references are resolved

Signed-off-by: Simon Wülker <[email protected]>

* Remove CharRefTokenizer::finish_none

Signed-off-by: Simon Wülker <[email protected]>

---------

Signed-off-by: Simon Wülker <[email protected]>

Author: simonwuelker

html5ever/src/tokenizer/char_ref/mod.rs

@@ -28,7 +28,7 @@ pub(super) struct CharRef {
 pub(super) enum Status {
     Stuck,
     Progress,
-    Done,
+    Done(CharRef),
 }
 
 #[derive(Debug)]
@@ -43,7 +43,6 @@ enum State {
 
 pub(super) struct CharRefTokenizer {
     state: State,
-    result: Option<CharRef>,
     is_consumed_in_attribute: bool,
 
     num: u32,
@@ -56,12 +55,18 @@ pub(super) struct CharRefTokenizer {
     name_len: usize,
 }
 
+impl CharRef {
+    const EMPTY: CharRef = CharRef {
+        chars: ['\0', '\0'],
+        num_chars: 0,
+    };
+}
+
 impl CharRefTokenizer {
     pub(super) fn new(is_consumed_in_attribute: bool) -> CharRefTokenizer {
         CharRefTokenizer {
             is_consumed_in_attribute,
             state: State::Begin,
-            result: None,
             num: 0,
             num_too_big: false,
             seen_digit: false,
@@ -72,12 +77,6 @@ impl CharRefTokenizer {
         }
     }
 
-    // A CharRefTokenizer can only tokenize one character reference,
-    // so this method consumes the tokenizer.
-    pub(super) fn get_result(self) -> CharRef {
-        self.result.expect("get_result called before done")
-    }
-
     fn name_buf(&self) -> &StrTendril {
         self.name_buf_opt
             .as_ref()
@@ -90,20 +89,11 @@ impl CharRefTokenizer {
             .expect("name_buf missing in named character reference")
     }
 
-    fn finish_none(&mut self) -> Status {
-        self.result = Some(CharRef {
-            chars: ['\0', '\0'],
-            num_chars: 0,
-        });
-        Status::Done
-    }
-
     fn finish_one(&mut self, c: char) -> Status {
-        self.result = Some(CharRef {
+        Status::Done(CharRef {
             chars: [c, '\0'],
             num_chars: 1,
-        });
-        Status::Done
+        })
     }
 }
 
@@ -113,10 +103,6 @@ impl CharRefTokenizer {
         tokenizer: &Tokenizer<Sink>,
         input: &BufferQueue,
     ) -> Status {
-        if self.result.is_some() {
-            return Status::Done;
-        }
-
         debug!("char ref tokenizer stepping in state {:?}", self.state);
         match self.state {
             State::Begin => self.do_begin(tokenizer, input),
@@ -144,7 +130,7 @@ impl CharRefTokenizer {
                 self.state = State::Octothorpe;
                 Status::Progress
             },
-            Some(_) => self.finish_none(),
+            Some(_) => Status::Done(CharRef::EMPTY),
             None => Status::Stuck,
         }
     }
@@ -228,7 +214,7 @@ impl CharRefTokenizer {
 
         input.push_front(unconsume);
         tokenizer.emit_error(Borrowed("Numeric character reference without digits"));
-        self.finish_none()
+        Status::Done(CharRef::EMPTY)
     }
 
     fn finish_numeric<Sink: TokenSink>(&mut self, tokenizer: &Tokenizer<Sink>) -> Status {
@@ -331,7 +317,7 @@ impl CharRefTokenizer {
                     _ => (),
                 }
                 self.unconsume_name(input);
-                self.finish_none()
+                Status::Done(CharRef::EMPTY)
             },
 
             Some((c1, c2)) => {
@@ -379,15 +365,14 @@ impl CharRefTokenizer {
 
                 if unconsume_all {
                     self.unconsume_name(input);
-                    self.finish_none()
+                    Status::Done(CharRef::EMPTY)
                 } else {
                     input.push_front(StrTendril::from_slice(&self.name_buf()[name_len..]));
                     tokenizer.ignore_lf.set(false);
-                    self.result = Some(CharRef {
+                    Status::Done(CharRef {
                         chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()],
                         num_chars: if c2 == 0 { 1 } else { 2 },
-                    });
-                    Status::Done
+                    })
                 }
             },
         }
@@ -411,34 +396,42 @@ impl CharRefTokenizer {
             _ => (),
         }
         self.unconsume_name(input);
-        self.finish_none()
+        Status::Done(CharRef::EMPTY)
     }
 
     pub(super) fn end_of_file<Sink: TokenSink>(
         &mut self,
         tokenizer: &Tokenizer<Sink>,
         input: &BufferQueue,
-    ) {
-        while self.result.is_none() {
-            match self.state {
-                State::Begin => drop(self.finish_none()),
-                State::Numeric(_) if !self.seen_digit => {
-                    self.unconsume_numeric(tokenizer, input);
-                },
+    ) -> CharRef {
+        loop {
+            let status = match self.state {
+                State::Begin => Status::Done(CharRef::EMPTY),
+                State::Numeric(_) if !self.seen_digit => self.unconsume_numeric(tokenizer, input),
                 State::Numeric(_) | State::NumericSemicolon => {
                     tokenizer.emit_error(Borrowed("EOF in numeric character reference"));
-                    self.finish_numeric(tokenizer);
+                    self.finish_numeric(tokenizer)
                 },
-                State::Named => drop(self.finish_named(tokenizer, input, None)),
+                State::Named => self.finish_named(tokenizer, input, None),
                 State::BogusName => {
                     self.unconsume_name(input);
-                    self.finish_none();
+                    Status::Done(CharRef::EMPTY)
                 },
                 State::Octothorpe => {
                     input.push_front(StrTendril::from_slice("#"));
                     tokenizer.emit_error(Borrowed("EOF after '#' in character reference"));
-                    self.finish_none();
+                    Status::Done(CharRef::EMPTY)
+                },
+            };
+
+            match status {
+                Status::Done(char_ref) => {
+                    return char_ref;
+                },
+                Status::Stuck => {
+                    return CharRef::EMPTY;
                 },
+                Status::Progress => {},
             }
         }
     }

html5ever/src/tokenizer/mod.rs

@@ -108,7 +108,7 @@ pub struct Tokenizer<Sink> {
 
     /// Tokenizer for character references, if we're tokenizing
     /// one at the moment.
-    char_ref_tokenizer: RefCell<Option<Box<CharRefTokenizer>>>,
+    char_ref_tokenizer: RefCell<Option<CharRefTokenizer>>,
 
     /// Current input character.  Just consumed, may reconsume.
     current_char: Cell<char>,
@@ -558,11 +558,14 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
         }
     }
 
-    fn consume_char_ref(&self) {
-        *self.char_ref_tokenizer.borrow_mut() = Some(Box::new(CharRefTokenizer::new(matches!(
-            self.state.get(),
-            states::AttributeValue(_)
-        ))));
+    fn start_consuming_character_reference(&self) {
+        debug_assert!(
+            self.char_ref_tokenizer.borrow().is_none(),
+            "Nested character references are impossible"
+        );
+
+        let is_in_attribute = matches!(self.state.get(), states::AttributeValue(_));
+        *self.char_ref_tokenizer.borrow_mut() = Some(CharRefTokenizer::new(is_in_attribute));
     }
 
     fn emit_eof(&self) {
@@ -651,7 +654,7 @@ macro_rules! go (
     ( $me:ident : reconsume $s:ident $k1:expr           ) => ({ $me.reconsume.set(true); go!($me: to $s $k1);     });
     ( $me:ident : reconsume $s:ident $k1:ident $k2:expr ) => ({ $me.reconsume.set(true); go!($me: to $s $k1 $k2); });
 
-    ( $me:ident : consume_char_ref             ) => ({ $me.consume_char_ref(); return ProcessResult::Continue;         });
+    ( $me:ident : consume_char_ref             ) => ({ $me.start_consuming_character_reference(); return ProcessResult::Continue;         });
 
     // We have a default next state after emitting a tag, but the sink can override.
     ( $me:ident : emit_tag $s:ident ) => ({
@@ -1660,22 +1663,18 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
     }
 
     fn step_char_ref_tokenizer(&self, input: &BufferQueue) -> ProcessResult<Sink::Handle> {
-        // FIXME HACK: Take and replace the tokenizer so we don't
-        // double-mut-borrow self.  This is why it's boxed.
-        let mut tok = self.char_ref_tokenizer.take().unwrap();
-        let outcome = tok.step(self, input);
-
-        let progress = match outcome {
-            char_ref::Status::Done => {
-                self.process_char_ref(tok.get_result());
+        let mut char_ref_tokenizer = self.char_ref_tokenizer.borrow_mut();
+        let progress = match char_ref_tokenizer.as_mut().unwrap().step(self, input) {
+            char_ref::Status::Done(char_ref) => {
+                self.process_char_ref(char_ref);
+                *char_ref_tokenizer = None;
                 return ProcessResult::Continue;
             },
 
             char_ref::Status::Stuck => ProcessResult::Suspend,
             char_ref::Status::Progress => ProcessResult::Continue,
         };
 
-        *self.char_ref_tokenizer.borrow_mut() = Some(tok);
         progress
     }
 
@@ -1712,9 +1711,8 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
         let input = BufferQueue::default();
         match self.char_ref_tokenizer.take() {
             None => (),
-            Some(mut tok) => {
-                tok.end_of_file(self, &input);
-                self.process_char_ref(tok.get_result());
+            Some(mut tokenizer) => {
+                self.process_char_ref(tokenizer.end_of_file(self, &input));
             },
         }