Merge pull request #19 from SimonSapin/tendrilsink

Upgrade html5ever to a TendrilSink based API.

Author: SimonSapin

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kuchiki"
-version = "0.2.1"
+version = "0.3.0"
 authors = ["Simon Sapin <[email protected]>"]
 license = "MIT"
 description = "(朽木) HTML/XML tree manipulation library"
@@ -13,22 +13,18 @@ doctest = false
 [features]
 unstable = [
     "string_cache/unstable",
-    "tendril/unstable",
     "rc/unstable",
     "html5ever/unstable",
     "selectors/unstable",
 ]
-with-hyper = ["hyper"]
 
 [dependencies]
 matches = "0.1.2"
-html5ever = "0.2.11"
+html5ever = "0.4"
+hyper = {version = "0.7", optional = true}
 string_cache = "0.2"
-tendril = "0.1.1"
 selectors = "0.2.0"
 rc = "0.1.0"
-hyper = {version = "0.7", optional = true}
 
 [dev-dependencies]
 tempdir = "0.3"
-hyper = "0.7"

examples/find_matches.rs

@@ -1,5 +1,7 @@
 extern crate kuchiki;
 
+use kuchiki::traits::*;
+
 fn main() {
     let html = r"
         <DOCTYPE html>
@@ -14,7 +16,7 @@ fn main() {
     ";
     let css_selector = ".foo";
 
-    let document = kuchiki::Html::from_string(html).parse();
+    let document = kuchiki::parse_html().one(html);
 
     for css_match in document.select(css_selector).unwrap() {
         // css_match is a NodeDataRef, but most of the interesting methods are

examples/hyper.rs

@@ -1,23 +1,22 @@
-extern crate hyper;
 extern crate kuchiki;
 
-use hyper::Client;
-
-use kuchiki::Html;
+#[cfg(not(feature = "hyper"))]
+fn main() {
+    // Intentionally trigger an unused_import warning,
+    // with a message on the same line that will be visible in compiler output:
+    use kuchiki::traits::*;  // This file requires the `hyper` feature to be enabled
+}
 
+#[cfg(feature = "hyper")]
 fn main() {
-    // Create a client.
-    let client = Client::new();
+    use kuchiki::traits::*;
+
     let url = "https://www.mozilla.org/en-US/";
     println!("{} - {} ", "Calling site ", url);
 
-    // Get response
-    let mut response = client.get(url).send().unwrap();
-
-    // Parse the html page
-    if let Ok(html) = Html::from_stream(&mut response) {
+    // Fetch and parse the html page
+    if let Ok(doc) = kuchiki::parse_html().from_http(url) {
         println!("{}", "Finding Easter egg");
-        let doc = html.parse();
 
         // Manually navigate to hidden comment
         let x = doc.children().nth(1).unwrap()
@@ -29,6 +28,6 @@ fn main() {
 
         println!("{}", *comment);
     } else {
-        println!("{}", "The page couldn't be parsed");
+        println!("{}", "The page couldn't be fetched");
     }
 }

src/lib.rs

@@ -8,13 +8,12 @@ Kuchiki (朽木), a HTML/XML tree manipulation library for Rust.
 #![deny(missing_docs)]
 
 extern crate html5ever;
-#[cfg(feature = "with-hyper")] extern crate hyper;
+#[cfg(feature = "hyper")] extern crate hyper;
 #[macro_use] extern crate matches;
 extern crate selectors;
 extern crate rc;
 #[macro_use] extern crate string_cache;
 #[cfg(test)] extern crate tempdir;
-extern crate tendril;
 
 mod attributes;
 pub mod iter;
@@ -27,8 +26,20 @@ mod serializer;
 mod tree;
 
 pub use attributes::Attributes;
-pub use iter::{NodeIterator, ElementIterator};
 pub use node_data_ref::NodeDataRef;
-pub use parser::{Html, ParseOpts};
+pub use parser::{parse_html, ParseOpts};
 pub use select::Selectors;
 pub use tree::{NodeRef, Node, NodeData, ElementData, Doctype, DocumentData};
+
+/// This module re-exports a number of traits that are useful when using Kuchiki.
+/// It can be used with:
+///
+/// ```rust
+/// use kuchiki::traits::*;
+/// ```
+pub mod traits {
+    pub use html5ever::tendril::TendrilSink;
+    pub use iter::{NodeIterator, ElementIterator};
+    pub use parser::ParserExt;
+}
+

src/parser.rs

@@ -1,76 +1,12 @@
 use std::borrow::Cow;
-use std::fs::File;
-use std::io::{Error, ErrorKind, Read};
-use std::option;
-use std::path::Path;
 use html5ever::{self, Attribute};
+use html5ever::tendril::StrTendril;
 use html5ever::tree_builder::{TreeSink, NodeOrText, QuirksMode};
-#[cfg(feature = "with-hyper")] use hyper::client::IntoUrl;
+#[cfg(feature = "hyper")] use hyper::client::IntoUrl;
 use string_cache::QualName;
-use tendril::{StrTendril, ReadExt, Tendril};
 
 use tree::NodeRef;
 
-/// The HTML parser.
-pub struct Html {
-    opts: ParseOpts,
-    data: option::IntoIter<StrTendril>,
-}
-
-impl Html  {
-    /// Parse from a single string in memory.
-    #[inline]
-    pub fn from_string<S: Into<StrTendril>>(string: S) -> Html {
-        Html {
-            opts: ParseOpts::default(),
-            data: Some(string.into()).into_iter(),
-        }
-    }
-
-    /// Parse from reading a file.
-    #[inline]
-    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Html, Error> {
-        Html::from_stream(&mut try!(File::open(&path)))
-    }
-
-    /// Fetch an HTTP or HTTPS URL with Hyper and parse.
-    #[cfg(feature = "with-hyper")]
-    pub fn from_http<U: IntoUrl>(url: U) -> Result<Html, ::hyper::Error> {
-        let mut response = try!(::hyper::Client::new().get(url).send());
-        Ok(try!(Html::from_stream(&mut response)))
-    }
-
-    /// Parse from reading a stream of bytes.
-    #[inline]
-    pub fn from_stream<S: Read>(stream: &mut S) -> Result<Html, Error> {
-        let mut buf = Tendril::new();
-        try!(stream.read_to_tendril(&mut buf));
-        Ok(Html {
-            opts: ParseOpts::default(),
-            // FIXME: Make UTF-8 decoding lossy, but try to minimize copying.
-            data: Some(try!(buf.try_reinterpret().map_err(|_| {
-                Error::new(ErrorKind::Other, "Invalid UTF-8.")
-            }))).into_iter(),
-        })
-    }
-
-    /// Run the parser and return a reference to the document node, the root of the tree.
-    #[inline]
-    pub fn parse(self) -> NodeRef {
-        let parser = Parser {
-            document_node: NodeRef::new_document(),
-            on_parse_error: self.opts.on_parse_error,
-        };
-        let html5opts = html5ever::ParseOpts {
-            tokenizer: self.opts.tokenizer,
-            tree_builder: self.opts.tree_builder,
-        };
-        let parser = html5ever::parse_to(parser, self.data, html5opts);
-        parser.document_node
-    }
-
-}
-
 /// Options for the HTML parser.
 #[derive(Default)]
 pub struct ParseOpts {
@@ -84,14 +20,64 @@ pub struct ParseOpts {
     pub on_parse_error: Option<Box<FnMut(Cow<'static, str>)>>,
 }
 
+/// Parse an HTML document with html5ever and the default configuration.
+pub fn parse_html() -> html5ever::Parser<Sink> {
+    parse_html_with_options(ParseOpts::default())
+}
+
+/// Parse an HTML document with html5ever.
+pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {
+    let sink = Sink {
+        document_node: NodeRef::new_document(),
+        on_parse_error: opts.on_parse_error,
+    };
+    let html5opts = html5ever::ParseOpts {
+        tokenizer: opts.tokenizer,
+        tree_builder: opts.tree_builder,
+    };
+    html5ever::parse_document(sink, html5opts)
+}
 
-struct Parser {
+/// Additional methods for html5ever::Parser
+pub trait ParserExt {
+    /// Fetch an HTTP or HTTPS URL with Hyper and parse,
+    /// giving the `charset` parameter of a `Content-Type` response header, if any,
+    /// as a character encoding hint to html5ever.
+    #[cfg(feature = "hyper")]
+    fn from_http<U: IntoUrl>(self, url: U) -> Result<NodeRef, ::hyper::Error>;
+}
+
+impl ParserExt for html5ever::Parser<Sink> {
+    #[cfg(feature = "hyper")]
+    fn from_http<U: IntoUrl>(self, url: U) -> Result<NodeRef, ::hyper::Error> {
+        use html5ever::encoding::label::encoding_from_whatwg_label;
+        use html5ever::tendril::TendrilSink;
+        use hyper::Client;
+        use hyper::header::ContentType;
+        use hyper::mime::Attr::Charset;
+        use html5ever::driver::BytesOpts;
+
+        let mut response = try!(Client::new().get(url).send());
+        let opts = BytesOpts {
+            transport_layer_encoding: response.headers.get::<ContentType>()
+                .and_then(|content_type| content_type.get_param(Charset))
+                .and_then(|charset| encoding_from_whatwg_label(charset))
+        };
+        Ok(try!(self.from_bytes(opts).read_from(&mut response)))
+    }
+}
+
+
+pub struct Sink {
     document_node: NodeRef,
     on_parse_error: Option<Box<FnMut(Cow<'static, str>)>>,
 }
 
+impl TreeSink for Sink {
+    type Output = NodeRef;
+
+    fn finish(self) -> NodeRef { self.document_node }
 
-impl TreeSink for Parser {
     type Handle = NodeRef;
 
     #[inline]

src/tests.rs

@@ -3,16 +3,16 @@ use std::path::Path;
 
 use tempdir::TempDir;
 
-use Html;
-use iter::NodeIterator;
+use parser::parse_html;
+use traits::*;
 
 #[test]
 fn text_nodes() {
     let html = r"
 <!doctype html>
 <title>Test case</title>
 <p>Content contains <b>Important</b> data</p>";
-    let document = Html::from_string(html).parse();
+    let document = parse_html().one(html);
     let paragraph = document.select("p").unwrap().collect::<Vec<_>>();
     assert_eq!(paragraph.len(), 1);
     assert_eq!(paragraph[0].text_contents(), "Content contains Important data");
@@ -35,7 +35,7 @@ fn parse_and_serialize() {
 <!doctype html>
 <title>Test case</title>
 <p>Content";
-    let document = Html::from_string(html).parse();
+    let document = parse_html().one(html);
     assert_eq!(document.as_document().unwrap().quirks_mode(), QuirksMode::NoQuirks);
     assert_eq!(document.to_string(), r"<!DOCTYPE html>
 <html><head><title>Test case</title>
@@ -57,7 +57,7 @@ fn parse_file() {
     
 
 </body></html>";
-    let document = Html::from_file(&path).unwrap().parse();
+    let document = parse_html().from_utf8().from_file(&path).unwrap();
     assert_eq!(document.to_string(), html);
 }
 
@@ -68,10 +68,10 @@ fn serialize_and_read_file() {
     path.push("temp.html");
 
     let html = r"<!DOCTYPE html><html><head><title>Title</title></head><body>Body</body></html>";
-    let document = Html::from_string(html).parse();
+    let document = parse_html().one(html);
     let _ = document.serialize_to_file(path.clone());
 
-    let document2 = Html::from_file(&path).unwrap().parse();
+    let document2 = parse_html().from_utf8().from_file(&path).unwrap();
     assert_eq!(document.to_string(), document2.to_string());
 }
 
@@ -84,7 +84,7 @@ fn select() {
 <p class=foo>Foo
 ";
 
-    let document = Html::from_string(html).parse();
+    let document = parse_html().one(html);
     let matching = document.select("p.foo").unwrap().collect::<Vec<_>>();
     assert_eq!(matching.len(), 2);
     let child = matching[0].as_node().first_child().unwrap();
@@ -105,6 +105,6 @@ fn to_string() {
     </body>
 </html>";
 
-    let document = Html::from_string(html).parse();
+    let document = parse_html().one(html);
     assert_eq!(document.inclusive_descendants().nth(11).unwrap().to_string(), "<p class=\"foo\">Foo\n    \n</p>");
 }