Add spans

Cargo.toml

@@ -26,6 +26,7 @@ markup5ever_rcdom = "0.5.0-unofficial"
 # required for examples/scraper.rs
 scraper = "0.21.0"
 argh = "0.1.12"
+annotate-snippets = "0.11.5"
 
 [features]
 # By default this crate depends on the jetscii library for best performance.
@@ -45,6 +46,10 @@ jetscii = { version = "0.5.1", optional = true }
 name = "patterns"
 harness = false
 
+[[bench]]
+name = "files"
+harness = false
+
 [[test]]
 name = "html5lib-tokenizer"
 path = "tests/html5lib_tokenizer.rs"
@@ -70,5 +75,8 @@ name = "callback_emitter"
 name = "scraper"
 required-features = ["tree-builder"]
 
+[[example]]
+name = "spans"
+
 [lib]
 bench = false

benches/files.rs

@@ -0,0 +1,32 @@
+#![allow(non_snake_case)]
+use iai::{black_box, main};
+
+use html5gum::Tokenizer;
+
+// files taken from https://github.com/AndreasMadsen/htmlparser-benchmark/tree/master/files
+const F0050: &str = include_str!("files/0050.html");
+const F6D30: &str = include_str!("files/6D30.html");
+
+fn file(file: &str, i: usize) {
+    let s: String = black_box((0..i).map(|_| file).collect());
+    for Ok(_) in Tokenizer::new(&s) {}
+}
+
+macro_rules! pattern_tests {
+    ($(($name:ident, $file:expr, $repeat:expr), )*) => {
+        $(
+            fn $name() {
+                file($file, $repeat)
+            }
+        )*
+
+        main!($($name),*);
+    }
+}
+
+pattern_tests![
+    (file_0050_10, F0050, 10),
+    (file_0050_100, F0050, 100),
+    (file_6D30_10, F6D30, 10),
+    (file_6D30_100, F6D30, 100),
+];

examples/callback_emitter.rs

@@ -11,27 +11,29 @@
 //! link: foo
 //! ```
 use html5gum::emitters::callback::{CallbackEmitter, CallbackEvent};
-use html5gum::{Emitter, IoReader, Tokenizer};
+use html5gum::{Emitter, IoReader, Reader, Span, Tokenizer};
 
-fn get_emitter() -> impl Emitter<Token = String> {
+fn get_emitter<R: Reader>() -> impl Emitter<R, Token = String> {
     let mut is_anchor_tag = false;
     let mut is_href_attr = false;
 
-    CallbackEmitter::new(move |event: CallbackEvent<'_>| match event {
-        CallbackEvent::OpenStartTag { name } => {
-            is_anchor_tag = name == b"a";
-            is_href_attr = false;
-            None
-        }
-        CallbackEvent::AttributeName { name } => {
-            is_href_attr = name == b"href";
-            None
-        }
-        CallbackEvent::AttributeValue { value } if is_anchor_tag && is_href_attr => {
-            Some(String::from_utf8_lossy(value).into_owned())
-        }
-        _ => None,
-    })
+    CallbackEmitter::new(
+        move |event: CallbackEvent<'_>, _span: Span<()>, _reader: &R| match event {
+            CallbackEvent::OpenStartTag { name } => {
+                is_anchor_tag = name == b"a";
+                is_href_attr = false;
+                None
+            }
+            CallbackEvent::AttributeName { name } => {
+                is_href_attr = name == b"href";
+                None
+            }
+            CallbackEvent::AttributeValue { value } if is_anchor_tag && is_href_attr => {
+                Some(String::from_utf8_lossy(value).into_owned())
+            }
+            _ => None,
+        },
+    )
 }
 
 fn main() {

examples/custom_emitter.rs

@@ -21,7 +21,7 @@
 //! ```text
 //! link: foo
 //! ```
-use html5gum::{Emitter, Error, IoReader, State, Tokenizer};
+use html5gum::{Emitter, Error, IoReader, Reader, State, Tokenizer};
 
 #[derive(Default)]
 struct LinkExtractor {
@@ -49,32 +49,32 @@ impl LinkExtractor {
     }
 }
 
-impl Emitter for LinkExtractor {
+impl<R: Reader> Emitter<R> for LinkExtractor {
     type Token = String;
 
-    fn set_last_start_tag(&mut self, last_start_tag: Option<&[u8]>) {
+    fn set_last_start_tag(&mut self, last_start_tag: Option<&[u8]>, _reader: &R) {
         self.last_start_tag.clear();
         self.last_start_tag
             .extend(last_start_tag.unwrap_or_default());
     }
 
-    fn pop_token(&mut self) -> Option<String> {
+    fn pop_token(&mut self, _reader: &R) -> Option<String> {
         self.queued_token.take()
     }
 
-    fn emit_string(&mut self, _: &[u8]) {}
+    fn emit_string(&mut self, _: &[u8], _reader: &R) {}
 
-    fn init_start_tag(&mut self) {
+    fn init_start_tag(&mut self, _reader: &R) {
         self.current_tag_name.clear();
         self.current_tag_is_closing = false;
     }
 
-    fn init_end_tag(&mut self) {
+    fn init_end_tag(&mut self, _reader: &R) {
         self.current_tag_name.clear();
         self.current_tag_is_closing = true;
     }
 
-    fn emit_current_tag(&mut self) -> Option<State> {
+    fn emit_current_tag(&mut self, _reader: &R) -> Option<State> {
         self.flush_old_attribute();
         self.last_start_tag.clear();
         if !self.current_tag_is_closing {
@@ -84,20 +84,20 @@ impl Emitter for LinkExtractor {
         html5gum::naive_next_state(&self.last_start_tag)
     }
 
-    fn set_self_closing(&mut self) {}
-    fn push_tag_name(&mut self, s: &[u8]) {
+    fn set_self_closing(&mut self, _reader: &R) {}
+    fn push_tag_name(&mut self, s: &[u8], _reader: &R) {
         self.current_tag_name.extend(s);
     }
 
-    fn init_attribute(&mut self) {
+    fn init_attribute(&mut self, _reader: &R) {
         self.flush_old_attribute();
     }
 
-    fn push_attribute_name(&mut self, s: &[u8]) {
+    fn push_attribute_name(&mut self, s: &[u8], _reader: &R) {
         self.current_attribute_name.extend(s);
     }
 
-    fn push_attribute_value(&mut self, s: &[u8]) {
+    fn push_attribute_value(&mut self, s: &[u8], _reader: &R) {
         self.current_attribute_value.extend(s);
     }
 
@@ -107,19 +107,20 @@ impl Emitter for LinkExtractor {
             && self.current_tag_name == self.last_start_tag
     }
 
-    fn emit_current_comment(&mut self) {}
-    fn emit_current_doctype(&mut self) {}
-    fn emit_eof(&mut self) {}
-    fn emit_error(&mut self, _: Error) {}
-    fn init_comment(&mut self) {}
-    fn init_doctype(&mut self) {}
-    fn push_comment(&mut self, _: &[u8]) {}
-    fn push_doctype_name(&mut self, _: &[u8]) {}
-    fn push_doctype_public_identifier(&mut self, _: &[u8]) {}
-    fn push_doctype_system_identifier(&mut self, _: &[u8]) {}
-    fn set_doctype_public_identifier(&mut self, _: &[u8]) {}
-    fn set_doctype_system_identifier(&mut self, _: &[u8]) {}
-    fn set_force_quirks(&mut self) {}
+    fn emit_current_comment(&mut self, _reader: &R) {}
+    fn emit_current_doctype(&mut self, _reader: &R) {}
+    fn emit_eof(&mut self, _reader: &R) {}
+    fn emit_error(&mut self, _: Error, _reader: &R) {}
+    fn init_comment(&mut self, _reader: &R) {}
+    fn init_doctype(&mut self, _reader: &R) {}
+    fn push_comment(&mut self, _: &[u8], _reader: &R) {}
+    fn push_doctype_name(&mut self, _: &[u8], _reader: &R) {}
+    fn push_doctype_public_identifier(&mut self, _: &[u8], _reader: &R) {}
+    fn push_doctype_system_identifier(&mut self, _: &[u8], _reader: &R) {}
+    fn set_doctype_public_identifier(&mut self, _: &[u8], _reader: &R) {}
+    fn set_doctype_system_identifier(&mut self, _: &[u8], _reader: &R) {}
+    fn start_open_tag(&mut self, _reader: &R) {}
+    fn set_force_quirks(&mut self, _reader: &R) {}
 }
 
 fn main() {

examples/spans.rs

@@ -0,0 +1,81 @@
+//! A modified version of `examples/callback_emitter.rs` which prints the location of all links in
+//! the input.
+//!
+//! ```text
+//! printf '<h1>Hello world!</h1><a href="foo">bar</a>' | cargo run --example=spans
+//! ```
+//!
+//! Output:
+//!
+//! ```text
+//! link: foo
+//! ```
+use annotate_snippets::{Level, Renderer, Snippet};
+use html5gum::emitters::callback::{CallbackEmitter, CallbackEvent};
+use html5gum::{Emitter, IoReader, Reader, Span, SpanBoundFromReader, SpanReader, Tokenizer};
+
+fn get_emitter<R: Reader>() -> impl Emitter<R, Token = (String, Span<usize>)>
+where
+    usize: SpanBoundFromReader<R>,
+{
+    let mut is_anchor_tag = false;
+    let mut is_href_attr = false;
+
+    CallbackEmitter::new(
+        move |event: CallbackEvent<'_>, span: Span<usize>, _reader: &R| match event {
+            CallbackEvent::OpenStartTag { name } => {
+                is_anchor_tag = name == b"a";
+                is_href_attr = false;
+                None
+            }
+            CallbackEvent::AttributeName { name } => {
+                is_href_attr = name == b"href";
+                None
+            }
+            CallbackEvent::AttributeValue { value } if is_anchor_tag && is_href_attr => Some((
+                format!(
+                    "found link with content `{}` here",
+                    String::from_utf8_lossy(value)
+                ),
+                span,
+            )),
+            _ => None,
+        },
+    )
+}
+
+struct CollectingReader<R> {
+    inner: R,
+    read: Vec<u8>,
+}
+
+impl<R: std::io::Read> std::io::Read for CollectingReader<R> {
+    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+        let len = self.inner.read(buf)?;
+        self.read.extend_from_slice(&buf[..len]);
+        Ok(len)
+    }
+}
+
+fn main() {
+    let mut reader = CollectingReader {
+        inner: std::io::stdin().lock(),
+        read: Vec::new(),
+    };
+    let spans =
+        Tokenizer::new_with_emitter(SpanReader::new(IoReader::new(&mut reader)), get_emitter())
+            .flatten()
+            .collect::<Vec<_>>();
+    let source = String::from_utf8_lossy(&reader.read);
+    let mut message = Level::Info.title("found link");
+    for (label, span) in &spans {
+        message = message.snippet(
+            Snippet::source(&source)
+                .origin("<stdin>")
+                .fold(true)
+                .annotation(Level::Info.span(span.start..span.end).label(label)),
+        );
+    }
+    let renderer = Renderer::styled();
+    println!("{}", renderer.render(message));
+}