Auto merge of #328 - derekdreery:docs, r=jdm

Assorted documentation

I'm just trying to learn how this lib works, and I'm looking at source to work it out. Hopefully if I doc as I go, others that come after me will benefit :)

Author: bors-servo

markup5ever/build.rs

@@ -98,6 +98,18 @@ fn named_entities_to_phf(from: &Path, to: &Path) {
     }
 
     let mut file = File::create(to).unwrap();
+    writeln!(&mut file, r#"
+/// A map of entity names to their codepoints. The second codepoint will
+/// be 0 if the entity contains a single codepoint. Entities have their preceeding '&' removed.
+///
+/// # Examples
+///
+/// ```
+/// use markup5ever::data::NAMED_ENTITIES;
+///
+/// assert_eq!(NAMED_ENTITIES.get("gt;").unwrap(), &(62, 0));
+/// ```
+"#).unwrap();
     write!(&mut file, "pub static NAMED_ENTITIES: Map<&'static str, (u32, u32)> = ").unwrap();
     phf_map.build(&mut file).unwrap();
     write!(&mut file, ";\n").unwrap();

markup5ever/data/mod.rs

@@ -6,6 +6,7 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
+//! Data that is known at compile-time and hard-coded into the binary.
 use phf::Map;
 
 /// The spec replaces most characters in the ISO-2022 C1 control code range

markup5ever/lib.rs

@@ -13,6 +13,20 @@ extern crate string_cache;
 extern crate phf;
 pub extern crate tendril;
 
+/// Create a [`SmallCharSet`], with each space-separated number stored in the set.
+///
+/// # Examples
+///
+/// ```
+/// # #[macro_use] extern crate markup5ever;
+/// # fn main() {
+/// let set = small_char_set!(12 54 42);
+/// assert_eq!(set.bits,
+///            0b00000000_01000000_00000100_00000000_00000000_00000000_00010000_00000000);
+/// # }
+/// ```
+///
+/// [`SmallCharSet`]: struct.SmallCharSet.html
 #[macro_export]
 macro_rules! small_char_set ( ($($e:expr)+) => (
     $ crate ::SmallCharSet {

markup5ever/rcdom.rs

@@ -84,6 +84,7 @@ pub struct Node {
 }
 
 impl Node {
+    /// Create a new node from its contents
     fn new(data: NodeData) -> Rc<Self> {
         Rc::new(Node {
             data: data,
@@ -99,12 +100,15 @@ pub type Handle = Rc<Node>;
 /// Weak reference to a DOM node, used for parent pointers.
 pub type WeakHandle = Weak<Node>;
 
+/// Append a parentless node to another nodes' children
 fn append(new_parent: &Handle, child: Handle) {
     let previous_parent = child.parent.replace(Some(Rc::downgrade(new_parent)));
+    // Invariant: child cannot have existing parent
     assert!(previous_parent.is_none());
     new_parent.children.borrow_mut().push(child);
 }
 
+/// If the node has a parent, get it and this node's position in its children
 fn get_parent_and_index(target: &Handle) -> Option<(Handle, usize)> {
     if let Some(weak) = target.parent.take() {
         let parent = weak.upgrade().expect("dangling weak pointer");

markup5ever/serialize.rs

@@ -6,35 +6,68 @@
 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
+//! Traits for serializing elements. The serializer expects the data to be xml-like (with a name,
+//! and optional children, attrs, text, comments, doctypes, and [processing instructions]). It uses
+//! the visitor pattern, where the serializer and the serializable objects are decoupled and
+//! implement their own traits.
+//!
+//! [processing instructions]: https://en.wikipedia.org/wiki/Processing_Instruction
 
 use QualName;
 use std::io;
 
 //§ serializing-html-fragments
+/// Used as a parameter to `serialize`, telling it if we want to skip the parent.
 #[derive(Clone, PartialEq)]
 pub enum TraversalScope {
+    /// Include the parent node when serializing.
     IncludeNode,
+    /// Only serialize the children of the node, treating any provided qualified name as the
+    /// parent while serializing.
+    ///
+    /// This is used in the implementation of [`html5ever::serialize::serialize`]
+    ///
+    /// [`html5ever::serialize::serialize`]: ../../html5ever/serialize/fn.serialize.html
     ChildrenOnly(Option<QualName>)
 }
 
+/// Types that can be serialized (according to the xml-like scheme in `Serializer`) implement this
+/// trait.
 pub trait Serialize {
+    /// Take the serializer and call its methods to serialize this type. The type will dictate
+    /// which methods are called and with what parameters.
     fn serialize<S>(&self, serializer: &mut S, traversal_scope: TraversalScope) -> io::Result<()>
     where S: Serializer;
 }
 
+/// Types that are capable of serializing implement this trait
 pub trait Serializer {
+    /// Serialize the start of an element, for example `<div class="test">`.
     fn start_elem<'a, AttrIter>(&mut self, name: QualName, attrs: AttrIter) -> io::Result<()>
     where AttrIter: Iterator<Item=AttrRef<'a>>;
 
+    /// Serialize the end of an element, for example `</div>`.
     fn end_elem(&mut self, name: QualName) -> io::Result<()>;
 
+    /// Serialize a plain text node.
     fn write_text(&mut self, text: &str) -> io::Result<()>;
 
+    /// Serialize a comment node, for example `<!-- comment -->`.
     fn write_comment(&mut self, text: &str) -> io::Result<()>;
 
+    /// Serialize a doctype node, for example `<!doctype html>`.
     fn write_doctype(&mut self, name: &str) -> io::Result<()>;
 
+    /// Serialize a processing instruction node, for example
+    /// `<?xml-stylesheet type="text/xsl" href="style.xsl"?>`.
     fn write_processing_instruction(&mut self, target: &str, data: &str) -> io::Result<()>;
 }
 
+/// A type alias for an attribute name and value (e.g. the `class="test"` in `<div class="test">`
+/// is represented as `(<QualName of type class>, "test")`.
+///
+/// This is used in [`Serializer::start_elem`] where the value being serialized must supply an
+/// iterator over the attributes for the current element
+///
+/// [`Serializer::start_elem`]: trait.Serializer.html#tymethod.start_elem
 pub type AttrRef<'a> = (&'a QualName, &'a str);

markup5ever/util/buffer_queue.rs

@@ -7,6 +7,17 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+//! The `BufferQueue` struct and helper types.
+//!
+//! This type is designed for the efficient parsing of string data, especially where many
+//! significant characters are from the ascii range 0-63. This includes, for example, important
+//! characters in xml/html parsing.
+//!
+//! Good and predictable performance is achieved by avoiding allocation where possible (a.k.a. zero
+//! copy).
+//!
+//! [`BufferQueue`]: struct.BufferQueue.html
+
 
 use std::collections::VecDeque;
 
@@ -15,39 +26,53 @@ use tendril::StrTendril;
 pub use self::SetResult::{FromSet, NotFromSet};
 use util::smallcharset::SmallCharSet;
 
-/// Result from `pop_except_from`.
+/// Result from [`pop_except_from`] containing either a character from a [`SmallCharSet`], or a
+/// string buffer of characters not from the set.
+///
+/// [`pop_except_from`]: struct.BufferQueue.html#method.pop_except_from
+/// [`SmallCharSet`]: ../struct.SmallCharSet.html
 #[derive(PartialEq, Eq, Debug)]
 pub enum SetResult {
+    /// A character from the `SmallCharSet`.
     FromSet(char),
+    /// A string buffer containing no characters from the `SmallCharSet`.
     NotFromSet(StrTendril),
 }
 
-/// A queue of owned string buffers, which supports incrementally
-/// consuming characters.
+/// A queue of owned string buffers, which supports incrementally consuming characters.
+///
+/// Internally it uses [`VecDeque`] and has the same complexity properties.
+///
+/// [`VecDeque`]: https://doc.rust-lang.org/std/collections/struct.VecDeque.html
 pub struct BufferQueue {
     /// Buffers to process.
     buffers: VecDeque<StrTendril>,
 }
 
 impl BufferQueue {
     /// Create an empty BufferQueue.
+    #[inline]
     pub fn new() -> BufferQueue {
         BufferQueue {
             buffers: VecDeque::with_capacity(16),
         }
     }
 
     /// Returns whether the queue is empty.
+    #[inline]
     pub fn is_empty(&self) -> bool {
         self.buffers.is_empty()
     }
 
-    /// Get the tendril at the beginning of the queue.
+    /// Get the buffer at the beginning of the queue.
+    #[inline]
     pub fn pop_front(&mut self) -> Option<StrTendril> {
         self.buffers.pop_front()
     }
 
     /// Add a buffer to the beginning of the queue.
+    ///
+    /// If the buffer is empty, it will be skipped.
     pub fn push_front(&mut self, buf: StrTendril) {
         if buf.len32() == 0 {
             return;
@@ -56,20 +81,25 @@ impl BufferQueue {
     }
 
     /// Add a buffer to the end of the queue.
+    ///
+    /// If the buffer is empty, it will be skipped.
     pub fn push_back(&mut self, buf: StrTendril) {
         if buf.len32() == 0 {
             return;
         }
         self.buffers.push_back(buf);
     }
 
-    /// Look at the next available character, if any.
+    /// Look at the next available character without removing it, if the queue is not empty.
     pub fn peek(&self) -> Option<char> {
-        // Invariant: all buffers in the queue are non-empty.
+        debug_assert!(self.buffers.iter().skip_while(|el| el.len32() != 0).next().is_none(),
+                      "invariant \"all buffers in the queue are non-empty\" failed");
         self.buffers.front().map(|b| b.chars().next().unwrap())
     }
 
-    /// Get the next character, if one is available.
+    /// Get the next character if one is available, removing it from the queue.
+    ///
+    /// This function manages the buffers, removing them as they become empty.
     pub fn next(&mut self) -> Option<char> {
         let (result, now_empty) = match self.buffers.front_mut() {
             None => (None, false),
@@ -87,9 +117,32 @@ impl BufferQueue {
     }
 
     /// Pops and returns either a single character from the given set, or
-    /// a `StrTendril` of characters none of which are in the set.  The set
-    /// is represented as a bitmask and so can only contain the first 64
-    /// ASCII characters.
+    /// a buffer of characters none of which are in the set.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[macro_use] extern crate markup5ever;
+    /// # #[macro_use] extern crate tendril;
+    /// # fn main() {
+    /// use markup5ever::buffer_queue::{BufferQueue, SetResult};
+    ///
+    /// let mut queue = BufferQueue::new();
+    /// queue.push_back(format_tendril!(r#"<some_tag attr="text">SomeText</some_tag>"#));
+    /// let set = small_char_set!(b'<' b'>' b' ' b'=' b'"' b'/');
+    /// let tag = format_tendril!("some_tag");
+    /// let attr = format_tendril!("attr");
+    /// let attr_val = format_tendril!("text");
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::FromSet('<')));
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::NotFromSet(tag)));
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::FromSet(' ')));
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::NotFromSet(attr)));
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::FromSet('=')));
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::FromSet('"')));
+    /// assert_eq!(queue.pop_except_from(set), Some(SetResult::NotFromSet(attr_val)));
+    /// // ...
+    /// # }
+    /// ```
     pub fn pop_except_from(&mut self, set: SmallCharSet) -> Option<SetResult> {
         let (result, now_empty) = match self.buffers.front_mut() {
             None => (None, false),
@@ -117,12 +170,29 @@ impl BufferQueue {
         result
     }
 
-    // Check if the next characters are an ASCII case-insensitive match for
-    // `pat`, which must be non-empty.
-    //
-    // If so, consume them and return Some(true).
-    // If they do not match, return Some(false).
-    // If not enough characters are available to know, return None.
+    /// Consume bytes matching the pattern, using a custom comparison function `eq`.
+    ///
+    /// Returns `Some(true)` if there is a match, `Some(false)` if there is no match, or `None` if
+    /// it wasn't possible to know (more data is needed).
+    ///
+    /// The custom comparison function is used elsewhere to compare ascii-case-insensitively.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # extern crate markup5ever;
+    /// # #[macro_use] extern crate tendril;
+    /// # fn main() {
+    /// use markup5ever::buffer_queue::{BufferQueue};
+    ///
+    /// let mut queue = BufferQueue::new();
+    /// queue.push_back(format_tendril!("testtext"));
+    /// let test_str = "test";
+    /// assert_eq!(queue.eat("test", |&a, &b| a == b), Some(true));
+    /// assert_eq!(queue.eat("text", |&a, &b| a == b), Some(true));
+    /// assert!(queue.is_empty());
+    /// # }
+    /// ```
     pub fn eat<F: Fn(&u8, &u8) -> bool>(&mut self, pat: &str, eq: F) -> Option<bool> {
         let mut buffers_exhausted = 0;
         let mut consumed_from_last = 0;

markup5ever/util/smallcharset.rs

@@ -7,23 +7,55 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+//! This module contains a single struct [`SmallCharSet`]. See its documentation for details.
+//!
+//! [`SmallCharSet`]: struct.SmallCharSet.html
 
 
 /// Represents a set of "small characters", those with Unicode scalar
 /// values less than 64.
+///
+/// This is stored as a bitmap, with 1 bit for each value.
+#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)]
 pub struct SmallCharSet {
     pub bits: u64,
 }
 
 impl SmallCharSet {
+    /// Checks whether a character (u8 value below 64) is stored in the SmallCharSet.
+    ///
+    /// # Examples
+    ///
+    /// ```ignore
+    /// # use markup5ever::SmallCharSet;
+    /// let set = SmallCharSet {
+    ///     bits: 0b00000000_01000000_00000100_00000000_00000000_00000000_00010000_00000000
+    /// };
+    /// assert!(set.contains(64));
+    /// assert!(set.contains(b'6')); // `b'6'` is the same as 64u8
+    /// ```
     #[inline]
     fn contains(&self, n: u8) -> bool {
         0 != (self.bits & (1 << (n as usize)))
     }
 
-    /// Count the number of bytes of characters at the beginning
-    /// of `buf` which are not in the set.
-    /// See `tokenizer::buffer_queue::pop_except_from`.
+    /// Count the number of bytes of characters at the beginning of `buf` which are not in the set.
+    ///
+    /// This functionality is used in [`BufferQueue::pop_except_from`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #[macro_use] extern crate markup5ever;
+    /// # fn main() {
+    /// let set = small_char_set!(48 49 50); // '0' '1' '2'
+    /// // `test` is 4 chars, 😁 is 4 chars, then we meet a character in the set
+    /// let test_str = "test😁01232afd";
+    /// assert_eq!(set.nonmember_prefix_len(test_str), 8);
+    /// # }
+    /// ```
+    ///
+    /// [`BufferQueue::pop_except_from`]: buffer_queue/struct.BufferQueue.html#method.pop_except_from
     pub fn nonmember_prefix_len(&self, buf: &str) -> u32 {
         let mut n = 0;
         for b in buf.bytes() {