Add an option on the writer to specify encoding scheme

Only Utf8 and Utf8WithBom are supported, as encoding_rs doesn't and
likely won't support other encodings.

Author: dralley

Changelog.md

@@ -42,7 +42,9 @@
 - [#450]: Added support of asynchronous [tokio](https://tokio.rs/) readers
 - [#455]: Change return type of all `read_to_end*` methods to return a span between tags
 - [#455]: Added `Reader::read_text` method to return a raw content (including markup) between tags
-
+- [#458]: Added an `EncodingScheme` configuration option to `Writer` to allow writing documents
+  with a BOM. Currently UTF-8 is the only supported encoding however it could be extended to cover
+  others in the future.
 
 ### Bug Fixes
 
@@ -186,11 +188,13 @@
 - [#440]: Removed `Deserializer::from_slice` and `quick_xml::de::from_slice` methods because deserializing from a byte
   array cannot guarantee borrowing due to possible copying while decoding.
 
-- [#455]: Removed `Reader::read_text_into` which is only not a better wrapper over match on `Event::Text`
+- [#455]: Removed `Reader::read_text_into` which is just a thin wrapper around match on `Event::Text`
 
 - [#456]: Reader and writer stuff grouped under `reader` and `writer` modules.
   You still can use re-exported definitions from a crate root
 
+- [#458]: Made the `Writer::write()` method non-public as writing random bytes to a document is not generally useful.
+
 ### New Tests
 
 - [#9]: Added tests for incorrect nested tags in input
@@ -234,7 +238,7 @@
 [#450]: https://github.com/tafia/quick-xml/pull/450
 [#455]: https://github.com/tafia/quick-xml/pull/455
 [#456]: https://github.com/tafia/quick-xml/pull/456
-
+[#458]: https://github.com/tafia/quick-xml/pull/458
 
 ## 0.23.0 -- 2022-05-08
 

src/encoding.rs

@@ -9,6 +9,15 @@ use encoding_rs::{Encoding, UTF_16BE, UTF_16LE, UTF_8};
 use crate::Error;
 use crate::Result;
 
+/// Unicode "byte order mark" encoded as UTF-8
+pub(crate) const UTF8_BOM: &[u8] = &[0xEF, 0xBB, 0xBF];
+/// Unicode "byte order mark" encoded as UTF-16 with little-endian byte order
+#[allow(dead_code)]
+pub(crate) const UTF16_LE_BOM: &[u8] = &[0xFF, 0xFE];
+/// Unicode "byte order mark" encoded as UTF-16 with big-endian byte order
+#[allow(dead_code)]
+pub(crate) const UTF16_BE_BOM: &[u8] = &[0xFE, 0xFF];
+
 /// Decoder of byte slices into strings.
 ///
 /// If feature `encoding` is enabled, this encoding taken from the `"encoding"`
@@ -62,7 +71,7 @@ impl Decoder {
     ///
     /// If you instead want to use XML declared encoding, use the `encoding` feature
     pub fn decode_with_bom_removal<'b>(&self, bytes: &'b [u8]) -> Result<Cow<'b, str>> {
-        let bytes = if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
+        let bytes = if bytes.starts_with(UTF8_BOM) {
             &bytes[3..]
         } else {
             bytes
@@ -131,11 +140,11 @@ pub fn decode_with_bom_removal<'b>(bytes: &'b [u8]) -> Result<Cow<'b, str>> {
 
 #[cfg(feature = "encoding")]
 fn split_at_bom<'b>(bytes: &'b [u8], encoding: &'static Encoding) -> (&'b [u8], &'b [u8]) {
-    if encoding == UTF_8 && bytes.starts_with(&[0xEF, 0xBB, 0xBF]) {
+    if encoding == UTF_8 && bytes.starts_with(UTF8_BOM) {
         bytes.split_at(3)
-    } else if encoding == UTF_16LE && bytes.starts_with(&[0xFF, 0xFE]) {
+    } else if encoding == UTF_16LE && bytes.starts_with(UTF16_LE_BOM) {
         bytes.split_at(2)
-    } else if encoding == UTF_16BE && bytes.starts_with(&[0xFE, 0xFF]) {
+    } else if encoding == UTF_16BE && bytes.starts_with(UTF16_BE_BOM) {
         bytes.split_at(2)
     } else {
         (&[], bytes)
@@ -172,9 +181,9 @@ fn remove_bom<'b>(bytes: &'b [u8], encoding: &'static Encoding) -> &'b [u8] {
 pub fn detect_encoding(bytes: &[u8]) -> Option<&'static Encoding> {
     match bytes {
         // with BOM
-        _ if bytes.starts_with(&[0xFE, 0xFF]) => Some(UTF_16BE),
-        _ if bytes.starts_with(&[0xFF, 0xFE]) => Some(UTF_16LE),
-        _ if bytes.starts_with(&[0xEF, 0xBB, 0xBF]) => Some(UTF_8),
+        _ if bytes.starts_with(UTF16_BE_BOM) => Some(UTF_16BE),
+        _ if bytes.starts_with(UTF16_LE_BOM) => Some(UTF_16LE),
+        _ if bytes.starts_with(UTF8_BOM) => Some(UTF_8),
 
         // without BOM
         _ if bytes.starts_with(&[0x00, b'<', 0x00, b'?']) => Some(UTF_16BE), // Some BE encoding, for example, UTF-16 or ISO-10646-UCS-2

src/writer.rs

@@ -1,9 +1,30 @@
 //! Contains high-level interface for an events-based XML emitter.
 
+use crate::encoding::UTF8_BOM;
 use crate::errors::{Error, Result};
 use crate::events::{attributes::Attribute, BytesCData, BytesStart, BytesText, Event};
 use std::io::Write;
 
+/// Writer-side encoding schemes supported by quick-xml.
+///
+/// Currently, `quick-xml` only supports UTF-8 as an output encoding as the `encoding_rs`
+/// library does not provide encoders for any other encodings. If you need to write UTF-16
+/// encoded XML, consider writing the XML with a UTF-8 encoding and then re-encoding the file.
+#[derive(Clone, Debug)]
+pub enum EncodingScheme {
+    /// UTF-8 text with no "BOM". This is the default, and recommended value.
+    Utf8,
+    /// UTF-8 with a "BOM" identifier. The standard recommends against this but some software
+    /// struggles to detect the encoding properly if it is not present.
+    Utf8WithBom,
+}
+
+impl Default for EncodingScheme {
+    fn default() -> Self {
+        Self::Utf8
+    }
+}
+
 /// XML writer.
 ///
 /// Writes XML `Event`s to a `Write` implementor.
@@ -57,6 +78,8 @@ pub struct Writer<W: Write> {
     /// underlying writer
     writer: W,
     indent: Option<Indentation>,
+    encoding: EncodingScheme,
+    first_write: bool,
 }
 
 impl<W: Write> Writer<W> {
@@ -65,14 +88,34 @@ impl<W: Write> Writer<W> {
         Writer {
             writer: inner,
             indent: None,
+            encoding: EncodingScheme::default(),
+            first_write: false,
         }
     }
 
-    /// Creates a Writer with configured whitespace indents from a generic Write
+    /// Creates a Writer from a generic Write implementor with configured whitespace indents
     pub fn new_with_indent(inner: W, indent_char: u8, indent_size: usize) -> Writer<W> {
         Writer {
             writer: inner,
             indent: Some(Indentation::new(indent_char, indent_size)),
+            encoding: EncodingScheme::default(),
+            first_write: true,
+        }
+    }
+
+    /// Creates a Writer from a generic Write implementor with configured whitespace indents and a
+    /// specified encoding scheme.
+    pub fn new_with_indent_and_encoding(
+        inner: W,
+        indent_char: u8,
+        indent_size: usize,
+        encoding_scheme: EncodingScheme,
+    ) -> Writer<W> {
+        Writer {
+            writer: inner,
+            indent: Some(Indentation::new(indent_char, indent_size)),
+            encoding: encoding_scheme,
+            first_write: true,
         }
     }
 
@@ -129,7 +172,15 @@ impl<W: Write> Writer<W> {
 
     /// Writes bytes
     #[inline]
-    pub fn write(&mut self, value: &[u8]) -> Result<()> {
+    pub(crate) fn write(&mut self, value: &[u8]) -> Result<()> {
+        // The BOM should be the very first thing written to the file, but it should only be written once
+        if self.first_write {
+            match self.encoding {
+                EncodingScheme::Utf8WithBom => self.writer.write_all(UTF8_BOM)?,
+                _ => (),
+            }
+            self.first_write = false;
+        }
         self.writer.write_all(value).map_err(Error::Io)
     }
 
@@ -579,4 +630,23 @@ mod indentation {
 </outer>"#
         );
     }
+
+    #[test]
+    fn write_utf8_with_bom() {
+        let mut buffer = Vec::new();
+        let mut writer =
+            Writer::new_with_indent_and_encoding(&mut buffer, b' ', 4, EncodingScheme::Utf8WithBom);
+
+        writer
+            .create_element("paired")
+            .with_attribute(("attr1", "value1"))
+            .with_attribute(("attr2", "value2"))
+            .write_text_content(BytesText::new("text"))
+            .expect("failure");
+
+        assert_eq!(
+            &buffer,
+            "\u{FEFF}<paired attr1=\"value1\" attr2=\"value2\">text</paired>".as_bytes()
+        );
+    }
 }