auto merge of #11120 : alexcrichton/rust/rustdoc-test, r=brson

This commit adds a `--test` flag to rustdoc to expose the ability to test code examples in doc strings. This work by using sundown's `lang` attribute to figure out how a code block should be tested. The format for this is:

```
1. ```rust
2. ```rust,ignore
3. ```rust,notest
4. ```rust,should_fail
```

Where `rust` means that rustdoc will attempt to test is, `ignore` means that it will not execute the test but it will compile it, `notest` means that rustdoc completely ignores it, and `should_fail` means that the test should fail. This commit also leverages `extra::test` for the testing harness in order to allow parallelization and whatnot.

I have fixed all existing code examples in libstd and libextra, but I have not made a pass through the crates in order to change code blocks to testable code blocks.

It may also be a questionable decision to require opting-in to a testable code block.

Finally, I kept our sugar in the doc suite to omit lines starting with `#` in documentation but still process them during tests.

Closes #2925

Author: bors

doc/rustdoc.md

@@ -85,3 +85,92 @@ javascript and a statically-generated search index. No special web server is
 required for the search.
 
 [sundown]: https://github.com/vmg/sundown/
+
+# Testing the Documentation
+
+`rustdoc` has support for testing code examples which appear in the
+documentation. This is helpful for keeping code examples up to date with the
+source code.
+
+To test documentation, the `--test` argument is passed to rustdoc:
+
+~~~
+rustdoc --test crate.rs
+~~~
+
+## Defining tests
+
+Rust documentation currently uses the markdown format, and code blocks can refer
+to any piece of code-related documentation, which isn't always rust. Because of
+this, only code blocks with the language of "rust" will be considered for
+testing.
+
+~~~
+```rust
+// This is a testable code block
+```
+
+```
+// This is not a testable code block
+```
+
+    // This is not a testable code block (4-space indent)
+~~~
+
+In addition to only testing "rust"-language code blocks, there are additional
+specifiers that can be used to dictate how a code block is tested:
+
+~~~
+```rust,ignore
+// This code block is ignored by rustdoc, but is passed through to the test
+// harness
+```
+
+```rust,should_fail
+// This code block is expected to generate a failure
+```
+~~~
+
+Rustdoc also supplies some extra sugar for helping with some tedious
+documentation examples. If a line is prefixed with a `#` character, then the
+line will not show up in the HTML documentation, but it will be used when
+testing the code block.
+
+~~~
+```rust
+# // showing 'fib' in this documentation would just be tedious and detracts from
+# // what's actualy being documented.
+# fn fib(n: int) { n + 2 }
+
+do spawn { fib(200); }
+```
+~~~
+
+The documentation online would look like `do spawn { fib(200); }`, but when
+testing this code, the `fib` function will be included (so it can compile).
+
+## Running tests (advanced)
+
+Running tests often requires some special configuration to filter tests, find
+libraries, or try running ignored examples. The testing framework that rustdoc
+uses is build on `extra::test`, which is also used when you compile crates with
+rustc's `--test` flag. Extra arguments can be passed to rustdoc's test harness
+with the `--test-args` flag.
+
+~~~
+// Only run tests containing 'foo' in their name
+rustdoc --test lib.rs --test-args 'foo'
+
+// See what's possible when running tests
+rustdoc --test lib.rs --test-args '--help'
+
+// Run all ignored tests
+rustdoc --test lib.rs --test-args '--ignored'
+~~~
+
+When testing a library, code examples will often show how functions are used,
+and this code often requires `use`-ing paths from the crate. To accomodate this,
+rustdoc will implicitly add `extern mod <crate>;` where `<crate>` is the name of
+the crate being tested to the top of each code example. This means that rustdoc
+must be able to find a compiled version of the library crate being tested. Extra
+search paths may be added via the `-L` flag to `rustdoc`.

mk/tests.mk

@@ -15,6 +15,7 @@
 
 # The names of crates that must be tested
 TEST_TARGET_CRATES = std extra rustuv
+TEST_DOC_CRATES = std extra
 TEST_HOST_CRATES = rustpkg rustc rustdoc syntax
 TEST_CRATES = $(TEST_TARGET_CRATES) $(TEST_HOST_CRATES)
 
@@ -281,6 +282,7 @@ check-stage$(1)-T-$(2)-H-$(3)-exec:     				\
 	check-stage$(1)-T-$(2)-H-$(3)-rpass-full-exec			\
 	check-stage$(1)-T-$(2)-H-$(3)-rmake-exec			\
         check-stage$(1)-T-$(2)-H-$(3)-crates-exec                       \
+        check-stage$(1)-T-$(2)-H-$(3)-doc-crates-exec                   \
 	check-stage$(1)-T-$(2)-H-$(3)-bench-exec			\
 	check-stage$(1)-T-$(2)-H-$(3)-debuginfo-exec \
 	check-stage$(1)-T-$(2)-H-$(3)-codegen-exec \
@@ -303,6 +305,10 @@ check-stage$(1)-T-$(2)-H-$(3)-crates-exec: \
 
 endif
 
+check-stage$(1)-T-$(2)-H-$(3)-doc-crates-exec: \
+        $$(foreach crate,$$(TEST_DOC_CRATES), \
+           check-stage$(1)-T-$(2)-H-$(3)-doc-$$(crate)-exec)
+
 check-stage$(1)-T-$(2)-H-$(3)-doc-exec: \
         $$(foreach docname,$$(DOC_TEST_NAMES), \
            check-stage$(1)-T-$(2)-H-$(3)-doc-$$(docname)-exec)
@@ -734,6 +740,26 @@ $(foreach host,$(CFG_HOST), \
    $(foreach docname,$(DOC_TEST_NAMES), \
     $(eval $(call DEF_RUN_DOC_TEST,$(stage),$(target),$(host),$(docname)))))))
 
+CRATE_DOC_LIB-std = $(STDLIB_CRATE)
+CRATE_DOC_LIB-extra = $(EXTRALIB_CRATE)
+
+define DEF_CRATE_DOC_TEST
+
+check-stage$(1)-T-$(2)-H-$(2)-doc-$(3)-exec: $$(call TEST_OK_FILE,$(1),$(2),$(2),doc-$(3))
+
+$$(call TEST_OK_FILE,$(1),$(2),$(2),doc-$(3)):		\
+	        $$(TEST_SREQ$(1)_T_$(2)_H_$(2))		\
+		$$(HBIN$(1)_H_$(2))/rustdoc$$(X_$(2))
+	@$$(call E, run doc-$(3) [$(2)])
+	$$(Q)$$(HBIN$(1)_H_$(2))/rustdoc$$(X_$(2)) --test \
+	    $$(CRATE_DOC_LIB-$(3)) && touch $$@
+
+endef
+
+$(foreach host,$(CFG_HOST), \
+ $(foreach stage,$(STAGES), \
+  $(foreach crate,$(TEST_DOC_CRATES), \
+   $(eval $(call DEF_CRATE_DOC_TEST,$(stage),$(host),$(crate))))))
 
 ######################################################################
 # Extracting tests for docs
@@ -762,6 +788,7 @@ $(foreach host,$(CFG_HOST), \
 TEST_GROUPS = \
 	crates \
 	$(foreach crate,$(TEST_CRATES),$(crate)) \
+	$(foreach crate,$(TEST_DOC_CRATES),doc-$(crate)) \
 	rpass \
 	rpass-full \
 	rfail \

src/libextra/arc.rs

@@ -18,18 +18,19 @@
  * With simple pipes, without Arc, a copy would have to be made for each task.
  *
  * ```rust
- * extern mod std;
- * use extra::arc;
- * let numbers=vec::from_fn(100, |ind| (ind as float)*rand::random());
- * let shared_numbers=arc::Arc::new(numbers);
+ * use extra::arc::Arc;
+ * use std::{rand, vec};
  *
- *   do 10.times {
- *       let (port, chan)  = stream();
+ * let numbers = vec::from_fn(100, |i| (i as f32) * rand::random());
+ * let shared_numbers = Arc::new(numbers);
+ *
+ *   for _ in range(0, 10) {
+ *       let (port, chan) = Chan::new();
  *       chan.send(shared_numbers.clone());
  *
  *       do spawn {
- *           let shared_numbers=port.recv();
- *           let local_numbers=shared_numbers.get();
+ *           let shared_numbers = port.recv();
+ *           let local_numbers = shared_numbers.get();
  *
  *           // Work with the local numbers
  *       }
@@ -448,15 +449,18 @@ impl<T:Freeze + Send> RWArc<T> {
      * # Example
      *
      * ```rust
-     * do arc.write_downgrade |mut write_token| {
-     *     do write_token.write_cond |state, condvar| {
-     *         ... exclusive access with mutable state ...
-     *     }
+     * use extra::arc::RWArc;
+     *
+     * let arc = RWArc::new(1);
+     * arc.write_downgrade(|mut write_token| {
+     *     write_token.write_cond(|state, condvar| {
+     *         // ... exclusive access with mutable state ...
+     *     });
      *     let read_token = arc.downgrade(write_token);
-     *     do read_token.read |state| {
-     *         ... shared access with immutable state ...
-     *     }
-     * }
+     *     read_token.read(|state| {
+     *         // ... shared access with immutable state ...
+     *     });
+     * })
      * ```
      */
     pub fn write_downgrade<U>(&self, blk: |v: RWWriteMode<T>| -> U) -> U {

src/libextra/future.rs

@@ -15,9 +15,10 @@
  * # Example
  *
  * ```rust
+ * use extra::future::Future;
  * # fn fib(n: uint) -> uint {42};
  * # fn make_a_sandwich() {};
- * let mut delayed_fib = extra::future::spawn (|| fib(5000) );
+ * let mut delayed_fib = do Future::spawn { fib(5000) };
  * make_a_sandwich();
  * println!("fib(5000) = {}", delayed_fib.get())
  * ```

src/libextra/glob.rs

@@ -53,8 +53,10 @@ pub struct GlobIterator {
 /// `puppies.jpg` and `hamsters.gif`:
 ///
 /// ```rust
+/// use extra::glob::glob;
+///
 /// for path in glob("/media/pictures/*.jpg") {
-///     println(path.to_str());
+///     println!("{}", path.display());
 /// }
 /// ```
 ///
@@ -188,21 +190,23 @@ enum MatchResult {
 impl Pattern {
 
     /**
-     * This function compiles Unix shell style patterns: `?` matches any single character,
-     * `*` matches any (possibly empty) sequence of characters and `[...]` matches any character
-     * inside the brackets, unless the first character is `!` in which case it matches any
-     * character except those between the `!` and the `]`. Character sequences can also specify
-     * ranges of characters, as ordered by Unicode, so e.g. `[0-9]` specifies any character
-     * between 0 and 9 inclusive.
+     * This function compiles Unix shell style patterns: `?` matches any single
+     * character, `*` matches any (possibly empty) sequence of characters and
+     * `[...]` matches any character inside the brackets, unless the first
+     * character is `!` in which case it matches any character except those
+     * between the `!` and the `]`. Character sequences can also specify ranges
+     * of characters, as ordered by Unicode, so e.g. `[0-9]` specifies any
+     * character between 0 and 9 inclusive.
      *
-     * The metacharacters `?`, `*`, `[`, `]` can be matched by using brackets (e.g. `[?]`).
-     * When a `]` occurs immediately following `[` or `[!` then it is interpreted as
-     * being part of, rather then ending, the character set, so `]` and NOT `]` can be
-     * matched by `[]]` and `[!]]` respectively. The `-` character can be specified inside a
-     * character sequence pattern by placing it at the start or the end, e.g. `[abc-]`.
+     * The metacharacters `?`, `*`, `[`, `]` can be matched by using brackets
+     * (e.g. `[?]`).  When a `]` occurs immediately following `[` or `[!` then
+     * it is interpreted as being part of, rather then ending, the character
+     * set, so `]` and NOT `]` can be matched by `[]]` and `[!]]` respectively.
+     * The `-` character can be specified inside a character sequence pattern by
+     * placing it at the start or the end, e.g. `[abc-]`.
      *
-     * When a `[` does not have a closing `]` before the end of the string then the `[` will
-     * be treated literally.
+     * When a `[` does not have a closing `]` before the end of the string then
+     * the `[` will be treated literally.
      */
     pub fn new(pattern: &str) -> Pattern {
 
@@ -229,7 +233,8 @@ impl Pattern {
                         match chars.slice_from(i + 3).position_elem(&']') {
                             None => (),
                             Some(j) => {
-                                let cs = parse_char_specifiers(chars.slice(i + 2, i + 3 + j));
+                                let chars = chars.slice(i + 2, i + 3 + j);
+                                let cs = parse_char_specifiers(chars);
                                 tokens.push(AnyExcept(cs));
                                 i += j + 4;
                                 continue;
@@ -292,6 +297,8 @@ impl Pattern {
      * # Example
      *
      * ```rust
+     * use extra::glob::Pattern;
+     *
      * assert!(Pattern::new("c?t").matches("cat"));
      * assert!(Pattern::new("k[!e]tteh").matches("kitteh"));
      * assert!(Pattern::new("d*g").matches("doog"));
@@ -509,7 +516,7 @@ impl MatchOptions {
      *
      * This function always returns this value:
      *
-     * ```rust
+     * ```rust,ignore
      * MatchOptions {
      *     case_sensitive: true,
      *     require_literal_separator: false.

src/libextra/hex.rs

@@ -28,7 +28,6 @@ impl<'a> ToHex for &'a [u8] {
      * # Example
      *
      * ```rust
-     * extern mod extra;
      * use extra::hex::ToHex;
      *
      * fn main () {
@@ -71,12 +70,11 @@ impl<'a> FromHex for &'a str {
      * This converts a string literal to hexadecimal and back.
      *
      * ```rust
-     * extern mod extra;
      * use extra::hex::{FromHex, ToHex};
      * use std::str;
      *
      * fn main () {
-     *     let hello_str = "Hello, World".to_hex();
+     *     let hello_str = "Hello, World".as_bytes().to_hex();
      *     println!("{}", hello_str);
      *     let bytes = hello_str.from_hex().unwrap();
      *     println!("{:?}", bytes);

src/libextra/lru_cache.rs

@@ -17,6 +17,8 @@
 //! # Example
 //!
 //! ```rust
+//! use extra::lru_cache::LruCache;
+//!
 //! let mut cache: LruCache<int, int> = LruCache::new(2);
 //! cache.put(1, 10);
 //! cache.put(2, 20);

src/libextra/sync.rs

@@ -568,13 +568,16 @@ impl RWLock {
      * # Example
      *
      * ```rust
+     * use extra::sync::RWLock;
+     *
+     * let lock = RWLock::new();
      * lock.write_downgrade(|mut write_token| {
      *     write_token.write_cond(|condvar| {
-     *         ... exclusive access ...
+     *         // ... exclusive access ...
      *     });
      *     let read_token = lock.downgrade(write_token);
      *     read_token.read(|| {
-     *         ... shared access ...
+     *         // ... shared access ...
      *     })
      * })
      * ```

src/libextra/url.rs

@@ -26,6 +26,8 @@ use std::uint;
 /// # Example
 ///
 /// ```rust
+/// use extra::url::{Url, UserInfo};
+///
 /// let url = Url { scheme: ~"https",
 ///                 user: Some(UserInfo { user: ~"username", pass: None }),
 ///                 host: ~"example.com",
@@ -388,8 +390,10 @@ fn query_from_str(rawquery: &str) -> Query {
  * # Example
  *
  * ```rust
+ * use extra::url;
+ *
  * let query = ~[(~"title", ~"The Village"), (~"north", ~"52.91"), (~"west", ~"4.10")];
- * println(query_to_str(&query));  // title=The%20Village&north=52.91&west=4.10
+ * println(url::query_to_str(&query));  // title=The%20Village&north=52.91&west=4.10
  * ```
  */
 pub fn query_to_str(query: &Query) -> ~str {