Add Parser::from_bytes, with BOM detection and Content-Type charset.

Author: SimonSapin

src/driver.rs

@@ -13,11 +13,13 @@ use tokenizer::{Attribute, Tokenizer, TokenizerOpts};
 use tree_builder::{TreeBuilderOpts, TreeBuilder, TreeSink};
 
 use std::borrow::Cow;
+use std::mem;
 
+use encoding::{self, EncodingRef};
 use string_cache::QualName;
 use tendril;
-use tendril::StrTendril;
-use tendril::stream::{TendrilSink, Utf8LossyDecoder};
+use tendril::{StrTendril, ByteTendril};
+use tendril::stream::{TendrilSink, Utf8LossyDecoder, LossyDecoder};
 
 /// All-encompassing options struct for the parser.
 #[derive(Clone, Default)]
@@ -30,13 +32,25 @@ pub struct ParseOpts {
 }
 
 /// Parse an HTML document
+///
+/// The returned value implements `tendril::TendrilSink`
+/// so that Unicode input may be provided incrementally,
+/// or all at once with the `one` method.
+///
+/// If your input is bytes, use `Parser::from_utf8` or `Parser::from_bytes`.
 pub fn parse_document<Sink>(sink: Sink, opts: ParseOpts) -> Parser<Sink> where Sink: TreeSink {
     let tb = TreeBuilder::new(sink, opts.tree_builder);
     let tok = Tokenizer::new(tb, opts.tokenizer);
     Parser { tokenizer: tok }
 }
 
 /// Parse an HTML fragment
+///
+/// The returned value implements `tendril::TendrilSink`
+/// so that Unicode input may be provided incrementally,
+/// or all at once with the `one` method.
+///
+/// If your input is bytes, use `Parser::from_utf8` or `Parser::from_bytes`.
 pub fn parse_fragment<Sink>(mut sink: Sink, opts: ParseOpts,
                             context_name: QualName, context_attrs: Vec<Attribute>)
                             -> Parser<Sink>
@@ -51,8 +65,10 @@ pub fn parse_fragment<Sink>(mut sink: Sink, opts: ParseOpts,
     Parser { tokenizer: tok }
 }
 
+/// An HTML parser,
+/// ready to recieve Unicode input through the `tendril::TendrilSink` trait’s methods.
 pub struct Parser<Sink> where Sink: TreeSink {
-    tokenizer: Tokenizer<TreeBuilder<Sink::Handle, Sink>>
+    tokenizer: Tokenizer<TreeBuilder<Sink::Handle, Sink>>,
 }
 
 impl<Sink: TreeSink> TendrilSink<tendril::fmt::UTF8> for Parser<Sink> {
@@ -74,7 +90,130 @@ impl<Sink: TreeSink> TendrilSink<tendril::fmt::UTF8> for Parser<Sink> {
 }
 
 impl<Sink: TreeSink> Parser<Sink> {
+    /// Wrap this parser into a `TendrilSink` that accepts UTF-8 bytes.
+    ///
+    /// Use this when your input is bytes that are known to be in the UTF-8 encoding.
+    /// Decoding is lossy, like `String::from_utf8_lossy`.
     pub fn from_utf8(self) -> Utf8LossyDecoder<Self> {
         Utf8LossyDecoder::new(self)
     }
+
+    /// Wrap this parser into a `TendrilSink` that accepts bytes
+    /// and tries to detect the correct character encoding.
+    ///
+    /// Currently this looks for a Byte Order Mark,
+    /// then uses `BytesOpts::transport_layer_encoding`,
+    /// then falls back to UTF-8.
+    ///
+    /// FIXME: this should look for `<meta>` elements and other data per
+    /// https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
+    pub fn from_bytes(self, opts: BytesOpts) -> BytesParser<Sink> {
+        BytesParser {
+            state: BytesParserState::Initial { parser: self },
+            opts: opts,
+        }
+    }
+}
+
+/// Options for choosing a character encoding
+#[derive(Clone, Default)]
+pub struct BytesOpts {
+    /// The character encoding specified by the transport layer, if any.
+    /// In HTTP for example, this is the `charset` parameter of the `Content-Type` response header.
+    pub transport_layer_encoding: Option<EncodingRef>,
+}
+
+/// An HTML parser,
+/// ready to recieve bytes input through the `tendril::TendrilSink` trait’s methods.
+///
+/// See `Parser::from_bytes`.
+pub struct BytesParser<Sink> where Sink: TreeSink {
+    state: BytesParserState<Sink>,
+    opts: BytesOpts,
+}
+
+enum BytesParserState<Sink> where Sink: TreeSink {
+    Initial {
+        parser: Parser<Sink>,
+    },
+    Buffering {
+        parser: Parser<Sink>,
+        buffer: ByteTendril
+    },
+    Parsing {
+        decoder: LossyDecoder<Parser<Sink>>,
+    },
+    Transient
+}
+
+impl<Sink: TreeSink> TendrilSink<tendril::fmt::Bytes> for BytesParser<Sink> {
+    fn process(&mut self, t: ByteTendril) {
+        if let &mut BytesParserState::Parsing { ref mut decoder } = &mut self.state {
+            return decoder.process(t)
+        }
+        let (parser, buffer) = match mem::replace(&mut self.state, BytesParserState::Transient) {
+            BytesParserState::Initial{ parser } => (parser, t),
+            BytesParserState::Buffering { parser, mut buffer } => {
+                buffer.push_tendril(&t);
+                (parser, buffer)
+            }
+            BytesParserState::Parsing { .. } | BytesParserState::Transient => unreachable!(),
+        };
+        if buffer.len32() >= PRESCAN_BYTES {
+            let encoding = detect_encoding(&buffer, &self.opts);
+            let decoder = LossyDecoder::new(encoding, parser);
+            self.state = BytesParserState::Parsing { decoder: decoder }
+        } else {
+            self.state = BytesParserState::Buffering {
+                parser: parser,
+                buffer: buffer,
+            }
+        }
+    }
+
+    fn error(&mut self, desc: Cow<'static, str>) {
+        match self.state {
+            BytesParserState::Initial { ref mut parser } => parser.error(desc),
+            BytesParserState::Buffering { ref mut parser, .. } => parser.error(desc),
+            BytesParserState::Parsing { ref mut decoder } => decoder.error(desc),
+            BytesParserState::Transient => unreachable!(),
+        }
+    }
+
+    type Output = Sink::Output;
+
+    fn finish(self) -> Self::Output {
+        match self.state {
+            BytesParserState::Initial { parser } => parser.finish(),
+            BytesParserState::Buffering { parser, buffer } => {
+                let encoding = detect_encoding(&buffer, &self.opts);
+                let decoder = LossyDecoder::new(encoding, parser);
+                decoder.finish()
+            },
+            BytesParserState::Parsing { decoder } => decoder.finish(),
+            BytesParserState::Transient => unreachable!(),
+        }
+    }
+}
+
+/// How many bytes does detect_encoding() need
+// NOTE: 3 would be enough for a BOM, but 1024 is specified for <meta> elements.
+const PRESCAN_BYTES: u32 = 1024;
+
+/// https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
+fn detect_encoding(bytes: &ByteTendril, opts: &BytesOpts) -> EncodingRef {
+    if bytes.starts_with(b"\xEF\xBB\xBF") {
+        return encoding::all::UTF_8
+    }
+    if bytes.starts_with(b"\xFE\xFF") {
+        return encoding::all::UTF_16BE
+    }
+    if bytes.starts_with(b"\xFF\xFE") {
+        return encoding::all::UTF_16LE
+    }
+    if let Some(encoding) = opts.transport_layer_encoding {
+        return encoding
+    }
+    // FIXME: <meta> etc.
+    return encoding::all::UTF_8
 }

src/lib.rs

@@ -24,8 +24,6 @@ extern crate log;
 #[macro_use]
 extern crate string_cache;
 
-extern crate tendril;
-
 #[macro_use]
 extern crate mac;
 
@@ -52,3 +50,12 @@ pub mod tree_builder;
 pub mod serialize;
 pub mod driver;
 pub mod rcdom;
+
+/// Re-export the tendril crate.
+pub mod tendril {
+    extern crate tendril;
+    pub use self::tendril::*;
+}
+
+/// Re-export the encoding crate.
+pub use tendril::encoding;