Assorted documentation

Author: Richard Dodd

markup5ever/lib.rs

@@ -13,6 +13,20 @@ extern crate string_cache;
 extern crate phf;
 pub extern crate tendril;
 
+/// Create a [`SmallCharSet`], with each space-separated number stored in the set.
+///
+/// # Examples
+///
+/// ```
+/// # #[macro_use] extern crate markup5ever;
+/// # fn main() {
+/// let set = small_char_set!(12 54 42);
+/// assert_eq!(set.bits,
+///            0b00000000_01000000_00000100_00000000_00000000_00000000_00010000_00000000);
+/// # }
+/// ```
+///
+/// [`SmallCharSet`]: struct.SmallCharSet.html
 #[macro_export]
 macro_rules! small_char_set ( ($($e:expr)+) => (
     $ crate ::SmallCharSet {

markup5ever/util/buffer_queue.rs

@@ -7,6 +7,17 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+//! The [`BufferQueue`] struct and helper types.
+//!
+//! This type is designed for the efficient parsing of string data, especially where many
+//! significant characters are from the ascii range 0-63. This includes, for example, important
+//! characters in xml/html parsing.
+//!
+//! Good and predictable performance is achieved by avoiding allocation where possible (a.k.a. zero
+//! copy).
+//!
+//! [`BufferQueue`]: struct.BufferQueue.html
+
 
 use std::collections::VecDeque;
 
@@ -15,15 +26,24 @@ use tendril::StrTendril;
 pub use self::SetResult::{FromSet, NotFromSet};
 use util::smallcharset::SmallCharSet;
 
-/// Result from `pop_except_from`.
+/// Result from [`pop_except_from`] containing either a character from a [`SmallCharSet`], or a
+/// string buffer of characters not from the set.
+///
+/// [`pop_except_from`]: struct.BufferQueue.html#method.pop_except_from
+/// [`SmallCharSet`]: ../struct.SmallCharSet.html
 #[derive(PartialEq, Eq, Debug)]
 pub enum SetResult {
+    /// A character from the `SmallCharSet`.
     FromSet(char),
+    /// A block of text containing no characters from the `SmallCharSet`.
     NotFromSet(StrTendril),
 }
 
-/// A queue of owned string buffers, which supports incrementally
-/// consuming characters.
+/// A queue of owned string buffers, which supports incrementally consuming characters.
+///
+/// Internally it uses [`VecDeque`] and has the same complexity properties.
+///
+/// [`VecDeque`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html
 pub struct BufferQueue {
     /// Buffers to process.
     buffers: VecDeque<StrTendril>,

markup5ever/util/smallcharset.rs

@@ -7,23 +7,54 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+//! This module contains a single struct [`SmallCharSet`]. See its documentation for details.
+//!
+//! [`SmallCharSet`]: struct.SmallCharSet.html
 
 
 /// Represents a set of "small characters", those with Unicode scalar
 /// values less than 64.
+///
+/// This is stored as a bitmap, with 1 bit for each value.
 pub struct SmallCharSet {
     pub bits: u64,
 }
 
 impl SmallCharSet {
+    /// Checks whether a character (u8 value below 64) is stored in the SmallCharSet.
+    ///
+    /// # Examples
+    ///
+    /// ```ignore
+    /// # use markup5ever::SmallCharSet;
+    /// let set = SmallCharSet {
+    ///     bits: 0b00000000_01000000_00000100_00000000_00000000_00000000_00010000_00000000
+    /// };
+    /// assert!(set.contains(64));
+    /// assert!(set.contains(b'6')); // `b'6'` is the same as 64u8
+    /// ```
     #[inline]
     fn contains(&self, n: u8) -> bool {
         0 != (self.bits & (1 << (n as usize)))
     }
 
-    /// Count the number of bytes of characters at the beginning
-    /// of `buf` which are not in the set.
-    /// See `tokenizer::buffer_queue::pop_except_from`.
+    /// Count the number of bytes of characters at the beginning of `buf` which are not in the set.
+    ///
+    /// This functionality is used in [`BufferQueue::pop_except_from`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[macro_use] extern crate markup5ever;
+    /// # fn main() {
+    /// let set = small_char_set!(48 49 50); // '0' '1' '2'
+    /// // `test` is 4 chars, 😁 is 4 chars, then we meet a character in the set
+    /// let test_str = "test😁01232afd";
+    /// assert_eq!(set.nonmember_prefix_len(test_str), 8);
+    /// # }
+    /// ```
+    ///
+    /// [`BufferQueue::pop_except_from`]: buffer_queue/struct.BufferQueue.html#method.pop_except_from
     pub fn nonmember_prefix_len(&self, buf: &str) -> u32 {
         let mut n = 0;
         for b in buf.bytes() {