Implement zero-copy parsing

Based on servo#60 and servo#114.

Fixes servo#20.
Fixes servo#115.

Author: kmcallister

Cargo.toml

@@ -21,6 +21,9 @@ git = "https://github.com/servo/string-cache"
 [dependencies.string_cache_plugin]
 git = "https://github.com/servo/string-cache"
 
+[dependencies.tendril]
+git = "https://github.com/kmcallister/tendril"
+
 [dependencies.mac]
 git = "https://github.com/reem/rust-mac"
 

benches/tokenizer.rs

@@ -10,6 +10,7 @@
 #![feature(box_syntax, std_misc, start, test)]
 
 extern crate test;
+extern crate tendril;
 extern crate html5ever;
 
 use std::{fs, env, cmp, rt};
@@ -21,6 +22,7 @@ use test::{black_box, Bencher, TestDesc, TestDescAndFn};
 use test::{DynTestName, DynBenchFn, TDynBenchFn};
 use test::ShouldPanic::No;
 
+use tendril::{ByteTendril, StrTendril, ReadExt, SliceExt};
 use html5ever::tokenizer::{TokenSink, Token, Tokenizer, TokenizerOpts};
 
 struct Sink;
@@ -36,7 +38,7 @@ impl TokenSink for Sink {
 // This could almost be the TokenSink too, but it's not
 // mut within run().
 struct Bench {
-    input: Vec<String>,
+    input: Vec<StrTendril>,
     clone_only: bool,
     opts: TokenizerOpts,
 }
@@ -50,8 +52,9 @@ impl Bench {
         let mut file = fs::File::open(&path).ok().expect("can't open file");
 
         // Read the file and treat it as an infinitely repeating sequence of characters.
-        let mut file_input = String::new();
-        file.read_to_string(&mut file_input).ok().expect("can't read file");
+        let mut file_input = ByteTendril::new();
+        file.read_to_tendril(&mut file_input).ok().expect("can't read file");
+        let file_input: StrTendril = file_input.try_reinterpret().unwrap();
         let size = size.unwrap_or(file_input.len());
         let mut stream = file_input.chars().cycle();
 
@@ -63,7 +66,7 @@ impl Bench {
             // The by_ref() call is important, otherwise we get wrong results!
             // See rust-lang/rust#18045.
             let sz = cmp::min(1024, size - total);
-            input.push(stream.by_ref().take(sz).collect());
+            input.push(stream.by_ref().take(sz).collect::<String>().to_tendril());
             total += sz;
         }
 

capi/Cargo.toml

@@ -18,3 +18,6 @@ path = "../"
 git = "https://github.com/servo/string-cache"
 [dependencies.string_cache_plugin]
 git = "https://github.com/servo/string-cache"
+
+[dependencies.tendril]
+git = "https://github.com/kmcallister/tendril"

capi/src/lib.rs

@@ -9,6 +9,7 @@
 
 extern crate libc;
 extern crate string_cache;
+extern crate tendril;
 extern crate html5ever;
 
 use std::{ptr, slice, str};
@@ -19,6 +20,8 @@ use libc::{size_t, c_int, c_char, strlen};
 
 use string_cache::Atom;
 
+use tendril::StrTendril;
+
 #[repr(C)]
 pub struct h5e_buf {
     data: *const u8,
@@ -86,6 +89,12 @@ impl AsLifetimeBuf for String {
     }
 }
 
+impl AsLifetimeBuf for StrTendril {
+    fn as_lifetime_buf<'a>(&'a self) -> LifetimeBuf<'a> {
+        LifetimeBuf::from_str(self)
+    }
+}
+
 impl AsLifetimeBuf for Atom {
     fn as_lifetime_buf<'a>(&'a self) -> LifetimeBuf<'a> {
         LifetimeBuf::from_str(self)

capi/src/tokenizer.rs

@@ -14,6 +14,7 @@ use {LifetimeBuf, AsLifetimeBuf, h5e_buf, c_bool};
 use html5ever::tokenizer::{TokenSink, Token, Doctype, Tag, ParseError, DoctypeToken};
 use html5ever::tokenizer::{CommentToken, CharacterTokens, NullCharacterToken};
 use html5ever::tokenizer::{TagToken, StartTag, EndTag, EOFToken, Tokenizer};
+use html5ever::Tendril;
 
 use std::mem;
 use std::default::Default;
@@ -71,7 +72,7 @@ impl TokenSink for *mut h5e_token_sink {
             ($name:ident) => (call!($name,)); // bleh
         }
 
-        fn opt_str_to_buf<'a>(s: &'a Option<String>) -> LifetimeBuf<'a> {
+        fn opt_str_to_buf<'a>(s: &'a Option<Tendril>) -> LifetimeBuf<'a> {
             match *s {
                 None => LifetimeBuf::null(),
                 Some(ref s) => s.as_lifetime_buf(),

dom_sink/src/common.rs

@@ -10,6 +10,7 @@
 use html5ever::tokenizer::Attribute;
 
 use string_cache::QualName;
+use tendril::StrTendril;
 
 pub use self::NodeEnum::{Document, Doctype, Text, Comment, Element};
 
@@ -20,13 +21,13 @@ pub enum NodeEnum {
     Document,
 
     /// A `DOCTYPE` with name, public id, and system id.
-    Doctype(String, String, String),
+    Doctype(StrTendril, StrTendril, StrTendril),
 
     /// A text node.
-    Text(String),
+    Text(StrTendril),
 
     /// A comment.
-    Comment(String),
+    Comment(StrTendril),
 
     /// An element with attributes.
     Element(QualName, Vec<Attribute>),

dom_sink/src/owned_dom.rs

@@ -38,6 +38,7 @@ use std::collections::HashSet;
 use std::ops::{Deref, DerefMut};
 
 use string_cache::QualName;
+use tendril::StrTendril;
 
 /// The internal type we use for nodes during parsing.
 pub struct SquishyNode {
@@ -135,7 +136,7 @@ fn get_parent_and_index(child: Handle) -> Option<(Handle, usize)> {
 fn append_to_existing_text(mut prev: Handle, text: &str) -> bool {
     match prev.deref_mut().node {
         Text(ref mut existing) => {
-            existing.push_str(text);
+            existing.push_slice(text);
             true
         }
         _ => false,
@@ -208,7 +209,7 @@ impl TreeSink for Sink {
         self.new_node(Element(name, attrs))
     }
 
-    fn create_comment(&mut self, text: String) -> Handle {
+    fn create_comment(&mut self, text: StrTendril) -> Handle {
         self.new_node(Comment(text))
     }
 
@@ -262,7 +263,10 @@ impl TreeSink for Sink {
         Ok(())
     }
 
-    fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) {
+    fn append_doctype_to_document(&mut self,
+                                  name: StrTendril,
+                                  public_id: StrTendril,
+                                  system_id: StrTendril) {
         append(self.document, self.new_node(Doctype(name, public_id, system_id)));
     }
 

dom_sink/src/rcdom.rs

@@ -30,6 +30,7 @@ use std::io::{self, Write};
 use std::ops::{Deref, DerefMut};
 
 use string_cache::QualName;
+use tendril::StrTendril;
 
 /// A DOM node.
 pub struct Node {
@@ -99,7 +100,7 @@ fn get_parent_and_index(target: &Handle) -> Option<(Handle, usize)> {
 fn append_to_existing_text(prev: &Handle, text: &str) -> bool {
     match prev.borrow_mut().deref_mut().node {
         Text(ref mut existing) => {
-            existing.push_str(text);
+            existing.push_slice(text);
             true
         }
         _ => false,
@@ -159,7 +160,7 @@ impl TreeSink for RcDom {
         new_node(Element(name, attrs))
     }
 
-    fn create_comment(&mut self, text: String) -> Handle {
+    fn create_comment(&mut self, text: StrTendril) -> Handle {
         new_node(Comment(text))
     }
 
@@ -214,7 +215,10 @@ impl TreeSink for RcDom {
         Ok(())
     }
 
-    fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) {
+    fn append_doctype_to_document(&mut self,
+                                  name: StrTendril,
+                                  public_id: StrTendril,
+                                  system_id: StrTendril) {
         append(&self.document, new_node(Doctype(name, public_id, system_id)));
     }
 

examples/html2html.rs

@@ -15,20 +15,25 @@
 //!
 //! where htmlparser-1.4.jar comes from http://about.validator.nu/htmlparser/
 
+extern crate tendril;
 extern crate html5ever;
 extern crate html5ever_dom_sink;
 
-use std::io::{self, Read, Write};
+use std::io::{self, Write};
 use std::default::Default;
 
+use tendril::{ByteTendril, ReadExt};
+
+use html5ever::sink::rcdom::RcDom;
 use html5ever::driver::ParseOpts;
 use html5ever_dom_sink::rcdom::RcDom;
 use html5ever::tree_builder::TreeBuilderOpts;
 use html5ever::{parse, one_input, serialize};
 
 fn main() {
-    let mut input = String::new();
-    io::stdin().read_to_string(&mut input).unwrap();
+    let mut input = ByteTendril::new();
+    io::stdin().read_to_tendril(&mut input).unwrap();
+    let input = input.try_reinterpret().unwrap();
     let dom: RcDom = parse(one_input(input), ParseOpts {
         tree_builder: TreeBuilderOpts {
             drop_doctype: true,

examples/noop-tokenize.rs

@@ -13,13 +13,15 @@
 
 extern crate test;
 extern crate html5ever;
+extern crate tendril;
 
 use std::io;
-use std::io::prelude::*;
 use std::default::Default;
 
 use test::black_box;
 
+use tendril::{ByteTendril, ReadExt};
+
 use html5ever::tokenizer::{TokenSink, Token};
 use html5ever::driver::{tokenize_to, one_input};
 
@@ -34,8 +36,9 @@ impl TokenSink for Sink {
 }
 
 fn main() {
-    let mut input = String::new();
-    io::stdin().read_to_string(&mut input).unwrap();
+    let mut input = ByteTendril::new();
+    io::stdin().read_to_tendril(&mut input).unwrap();
+    let input = input.try_reinterpret().unwrap();
 
     tokenize_to(Sink, one_input(input), Default::default());
 }

examples/noop-tree-builder.rs

@@ -8,16 +8,17 @@
 // except according to those terms.
 
 extern crate string_cache;
-
+extern crate tendril;
 extern crate html5ever;
 
-use std::io::{self, Read};
+use std::io;
 use std::default::Default;
-use std::string::String;
 use std::collections::HashMap;
 use std::borrow::Cow;
 use string_cache::QualName;
 
+use tendril::{StrTendril, ByteTendril, ReadExt};
+
 use html5ever::{parse_to, one_input};
 use html5ever::tokenizer::Attribute;
 use html5ever::tree_builder::{TreeSink, QuirksMode, NodeOrText};
@@ -56,7 +57,7 @@ impl TreeSink for Sink {
         id
     }
 
-    fn create_comment(&mut self, _text: String) -> usize {
+    fn create_comment(&mut self, _text: StrTendril) -> usize {
         self.get_id()
     }
 
@@ -72,7 +73,7 @@ impl TreeSink for Sink {
     fn set_quirks_mode(&mut self, _mode: QuirksMode) { }
     fn append(&mut self, _parent: usize, _child: NodeOrText<usize>) { }
 
-    fn append_doctype_to_document(&mut self, _name: String, _public_id: String, _system_id: String) { }
+    fn append_doctype_to_document(&mut self, _: StrTendril, _: StrTendril, _: StrTendril) { }
     fn add_attrs_if_missing(&mut self, _target: usize, _attrs: Vec<Attribute>) { }
     fn remove_from_parent(&mut self, _target: usize) { }
     fn reparent_children(&mut self, _node: usize, _new_parent: usize) { }
@@ -85,7 +86,8 @@ fn main() {
         names: HashMap::new(),
     };
 
-    let mut input = String::new();
-    io::stdin().read_to_string(&mut input).unwrap();
+    let mut input = ByteTendril::new();
+    io::stdin().read_to_tendril(&mut input).unwrap();
+    let input = input.try_reinterpret().unwrap();
     parse_to(sink, one_input(input), Default::default());
 }

examples/print-rcdom.rs

@@ -15,12 +15,14 @@ extern crate html5ever_dom_sink;
 
 #[macro_use]
 extern crate string_cache;
+extern crate tendril;
 
 use std::io::{self, Read};
 use std::iter::repeat;
 use std::default::Default;
 use std::string::String;
 
+use tendril::{ByteTendril, ReadExt};
 use html5ever::{parse, one_input};
 use html5ever_dom_sink::common::{Document, Doctype, Text, Comment, Element};
 use html5ever_dom_sink::rcdom::{RcDom, Handle};
@@ -61,8 +63,9 @@ fn walk(indent: usize, handle: Handle) {
 }
 
 fn main() {
-    let mut input = String::new();
-    io::stdin().read_to_string(&mut input).unwrap();
+    let mut input = ByteTendril::new();
+    io::stdin().read_to_tendril(&mut input).unwrap();
+    let input = input.try_reinterpret().unwrap();
     let dom: RcDom = parse(one_input(input), Default::default());
     walk(0, dom.document);
 

examples/print-tree-actions.rs

@@ -10,16 +10,17 @@
 #![feature(collections)]
 
 extern crate string_cache;
-
+extern crate tendril;
 extern crate html5ever;
 
-use std::io::{self, Read};
+use std::io;
 use std::default::Default;
-use std::string::String;
 use std::collections::HashMap;
 use std::borrow::Cow;
 use string_cache::QualName;
 
+use tendril::{ByteTendril, StrTendril, ReadExt};
+
 use html5ever::{parse_to, one_input};
 use html5ever::tokenizer::Attribute;
 use html5ever::tree_builder::{TreeSink, QuirksMode, NodeOrText, AppendNode, AppendText};
@@ -67,7 +68,7 @@ impl TreeSink for Sink {
         id
     }
 
-    fn create_comment(&mut self, text: String) -> usize {
+    fn create_comment(&mut self, text: StrTendril) -> usize {
         let id = self.get_id();
         println!("Created comment \"{}\" as {}", text.escape_default(), id);
         id
@@ -97,7 +98,10 @@ impl TreeSink for Sink {
         Ok(())
     }
 
-    fn append_doctype_to_document(&mut self, name: String, public_id: String, system_id: String) {
+    fn append_doctype_to_document(&mut self,
+                                  name: StrTendril,
+                                  public_id: StrTendril,
+                                  system_id: StrTendril) {
         println!("Append doctype: {} {} {}", name, public_id, system_id);
     }
 
@@ -127,7 +131,8 @@ fn main() {
         names: HashMap::new(),
     };
 
-    let mut input = String::new();
-    io::stdin().read_to_string(&mut input).unwrap();
+    let mut input = ByteTendril::new();
+    io::stdin().read_to_tendril(&mut input).unwrap();
+    let input = input.try_reinterpret().unwrap();
     parse_to(sink, one_input(input), Default::default());
 }