diff --git a/Cargo.lock b/Cargo.lock
index 8fc7154299..4827e7fd8a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3,6 +3,7 @@ name = "miri"
 version = "0.1.0"
 dependencies = [
  "byteorder 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "compiletest_rs 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -10,3 +11,16 @@ name = "byteorder"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "compiletest_rs"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "log"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
diff --git a/Cargo.toml b/Cargo.toml
index a369f6070c..9fa9f145c7 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -12,3 +12,6 @@ name = "miri"
 
 [dependencies]
 byteorder = "0.4.2"
+
+[dev-dependencies]
+compiletest_rs = "0.1.1"
diff --git a/src/bin/miri.rs b/src/bin/miri.rs
index 8efa1a6864..ebfb379d31 100644
--- a/src/bin/miri.rs
+++ b/src/bin/miri.rs
@@ -6,14 +6,13 @@ extern crate rustc_driver;
 
 use miri::interpreter;
 use rustc::session::Session;
-use rustc_driver::{driver, CompilerCalls, Compilation};
+use rustc_driver::{driver, CompilerCalls};
 
 struct MiriCompilerCalls;
 
 impl<'a> CompilerCalls<'a> for MiriCompilerCalls {
     fn build_controller(&mut self, _: &Session) -> driver::CompileController<'a> {
         let mut control = driver::CompileController::basic();
-        control.after_analysis.stop = Compilation::Stop;
 
         control.after_analysis.callback = Box::new(|state| {
             state.session.abort_if_errors();
diff --git a/tests/run-pass/errors.rs b/tests/compile-fail/errors.rs
similarity index 65%
rename from tests/run-pass/errors.rs
rename to tests/compile-fail/errors.rs
index e123538f6c..d971724f68 100644
--- a/tests/run-pass/errors.rs
+++ b/tests/compile-fail/errors.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -9,33 +8,33 @@ fn overwriting_part_of_relocation_makes_the_rest_undefined() -> i32 {
         let ptr: *mut _ = &mut p;
         *(ptr as *mut u32) = 123;
     }
-    *p
+    *p //~ ERROR: attempted to read undefined bytes
 }
 
 #[miri_run]
 fn pointers_to_different_allocations_are_unorderable() -> bool {
     let x: *const u8 = &1;
     let y: *const u8 = &2;
-    x < y
+    x < y //~ ERROR: attempted to do math or a comparison on pointers into different allocations
 }
 
 #[miri_run]
 fn invalid_bool() -> u8 {
     let b = unsafe { std::mem::transmute::<u8, bool>(2) };
-    if b { 1 } else { 2 }
+    if b { 1 } else { 2 } //~ ERROR: invalid boolean value read
 }
 
 #[miri_run]
 fn undefined_byte_read() -> u8 {
     let v: Vec<u8> = Vec::with_capacity(10);
     let undef = unsafe { *v.get_unchecked(5) };
-    undef + 1
+    undef + 1 //~ ERROR: attempted to read undefined bytes
 }
 
 #[miri_run]
 fn out_of_bounds_read() -> u8 {
     let v: Vec<u8> = vec![1, 2];
-    unsafe { *v.get_unchecked(5) }
+    unsafe { *v.get_unchecked(5) } //~ ERROR: pointer offset outside bounds of allocation
 }
 
 #[miri_run]
@@ -44,5 +43,7 @@ fn dangling_pointer_deref() -> i32 {
         let b = Box::new(42);
         &*b as *const i32
     };
-    unsafe { *p }
+    unsafe { *p } //~ ERROR: dangling pointer was dereferenced
 }
+
+fn main() {}
diff --git a/tests/compile-test.rs b/tests/compile-test.rs
deleted file mode 100644
index c7cdb974ca..0000000000
--- a/tests/compile-test.rs
+++ /dev/null
@@ -1,37 +0,0 @@
-use std::{env, fs};
-use std::process::{Command, Output};
-
-fn run_miri(file: &str, sysroot: &str) -> Output {
-    Command::new("cargo")
-        .args(&["run", "--", "--sysroot", sysroot, file])
-        .output()
-        .unwrap_or_else(|e| panic!("failed to execute process: {}", e))
-}
-
-#[test]
-fn run_pass() {
-    let sysroot = env::var("RUST_SYSROOT").expect("env variable `RUST_SYSROOT` not set");
-
-    let test_files = fs::read_dir("./tests/run-pass/")
-                         .expect("Can't read `run-pass` directory")
-                         .filter_map(|entry| entry.ok())
-                         .filter(|entry| {
-                             entry.clone()
-                                  .file_type()
-                                  .map(|x| x.is_file())
-                                  .unwrap_or(false)
-                         })
-                         .filter_map(|entry| entry.path().to_str().map(|x| x.to_string()));
-
-    for file in test_files {
-        println!("{}: compile test running", file);  
-
-        let test_run = run_miri(&file, &sysroot);
-
-        if test_run.status.code().unwrap_or(-1) != 0 {
-            println!("{}: error {:?}", file, test_run);
-        } else {
-            println!("{}: ok", file);
-        }
-    }
-}
diff --git a/tests/compiletest.rs b/tests/compiletest.rs
new file mode 100644
index 0000000000..224d7f0611
--- /dev/null
+++ b/tests/compiletest.rs
@@ -0,0 +1,23 @@
+extern crate compiletest_rs as compiletest;
+
+use std::path::PathBuf;
+
+fn run_mode(mode: &'static str) {
+    let mut config = compiletest::default_config();
+    config.rustc_path = "target/debug/miri".into();
+    let path = std::env::var("RUST_SYSROOT").expect("env variable `RUST_SYSROOT` not set");
+    config.target_rustcflags = Some(format!("--sysroot {}", path));
+    config.host_rustcflags = Some(format!("--sysroot {}", path));
+    let cfg_mode = mode.parse().ok().expect("Invalid mode");
+
+    config.mode = cfg_mode;
+    config.src_base = PathBuf::from(format!("tests/{}", mode));
+
+    compiletest::run_tests(&config);
+}
+
+#[test]
+fn compile_test() {
+    run_mode("compile-fail");
+    run_mode("run-pass");
+}
diff --git a/tests/run-pass/arrays.rs b/tests/run-pass/arrays.rs
index 05925fa98c..26a4196b1b 100644
--- a/tests/run-pass/arrays.rs
+++ b/tests/run-pass/arrays.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -33,3 +32,5 @@ fn index() -> i32 {
 fn array_repeat() -> [u8; 8] {
     [42; 8]
 }
+
+fn main() {}
diff --git a/tests/run-pass/bools.rs b/tests/run-pass/bools.rs
index 699409e17a..948c09c0fd 100644
--- a/tests/run-pass/bools.rs
+++ b/tests/run-pass/bools.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -27,3 +26,5 @@ fn match_bool() -> i16 {
         _ => 0,
     }
 }
+
+fn main() {}
diff --git a/tests/run-pass/c_enums.rs b/tests/run-pass/c_enums.rs
index 1da35d64b8..442e58a22a 100644
--- a/tests/run-pass/c_enums.rs
+++ b/tests/run-pass/c_enums.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -20,3 +19,5 @@ fn unsafe_match() -> bool {
         _ => false,
     }
 }
+
+fn main() {}
diff --git a/tests/run-pass/calls.rs b/tests/run-pass/calls.rs
index 0e9199f969..62ea521d95 100644
--- a/tests/run-pass/calls.rs
+++ b/tests/run-pass/calls.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -39,3 +38,5 @@ fn cross_crate_fn_call() -> i64 {
 fn test_size_of() -> usize {
     ::std::mem::size_of::<Option<i32>>()
 }
+
+fn main() {}
diff --git a/tests/run-pass/closures.rs b/tests/run-pass/closures.rs
index 9b6e296664..3612cfcb4c 100644
--- a/tests/run-pass/closures.rs
+++ b/tests/run-pass/closures.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -37,3 +36,5 @@ fn crazy_closure() -> (i32, i32, i32) {
 //     }
 //     y
 // }
+
+fn main() {}
diff --git a/tests/run-pass/heap.rs b/tests/run-pass/heap.rs
index 268b4121dc..3cc01e5829 100644
--- a/tests/run-pass/heap.rs
+++ b/tests/run-pass/heap.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute, box_syntax)]
 #![allow(dead_code, unused_attributes)]
 
@@ -11,3 +10,5 @@ fn make_box() -> Box<(i16, i16)> {
 fn make_box_syntax() -> Box<(i16, i16)> {
     box (1, 2)
 }
+
+fn main() {}
diff --git a/tests/run-pass/ints.rs b/tests/run-pass/ints.rs
index b790afe3b5..cc113eaed5 100644
--- a/tests/run-pass/ints.rs
+++ b/tests/run-pass/ints.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -53,3 +52,5 @@ fn match_int_range() -> i64 {
         _ => 5,
     }
 }
+
+fn main() {}
diff --git a/tests/run-pass/loops.rs b/tests/run-pass/loops.rs
index 008c6e6ad8..081e7bb228 100644
--- a/tests/run-pass/loops.rs
+++ b/tests/run-pass/loops.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -34,3 +33,5 @@ fn for_loop() -> usize {
     }
     sum
 }
+
+fn main() {}
diff --git a/tests/run-pass/pointers.rs b/tests/run-pass/pointers.rs
index ab1226c759..b66aabb250 100644
--- a/tests/run-pass/pointers.rs
+++ b/tests/run-pass/pointers.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -58,3 +57,5 @@ fn dangling_pointer() -> *const i32 {
     let b = Box::new(42);
     &*b as *const i32
 }
+
+fn main() {}
diff --git a/tests/run-pass/products.rs b/tests/run-pass/products.rs
index 2fad1daae0..1d65006ae6 100644
--- a/tests/run-pass/products.rs
+++ b/tests/run-pass/products.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -30,3 +29,5 @@ fn field_access() -> (i8, i8) {
     p.x += 5;
     (p.x, p.y)
 }
+
+fn main() {}
diff --git a/tests/run-pass/specialization.rs b/tests/run-pass/specialization.rs
index a7323431f4..b8b101fe84 100644
--- a/tests/run-pass/specialization.rs
+++ b/tests/run-pass/specialization.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute, specialization)]
 #![allow(dead_code, unused_attributes)]
 
@@ -18,3 +17,5 @@ impl IsUnit for () {
 fn specialization() -> (bool, bool) {
     (i32::is_unit(), <()>::is_unit())
 }
+
+fn main() {}
diff --git a/tests/run-pass/std.rs b/tests/run-pass/std.rs
index 55e801b93d..d9c0e3ca1f 100644
--- a/tests/run-pass/std.rs
+++ b/tests/run-pass/std.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute, box_syntax)]
 #![allow(dead_code, unused_attributes)]
 
@@ -44,3 +43,5 @@ fn rc_reference_cycle() -> Loop {
 fn true_assert() {
     assert_eq!(1, 1);
 }
+
+fn main() {}
diff --git a/tests/run-pass/strings.rs b/tests/run-pass/strings.rs
index 6d27153aaa..3d71fa4e09 100644
--- a/tests/run-pass/strings.rs
+++ b/tests/run-pass/strings.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -21,3 +20,5 @@ fn hello_bytes() -> &'static [u8; 13] {
 fn hello_bytes_fat() -> &'static [u8] {
     b"Hello, world!"
 }
+
+fn main() {}
diff --git a/tests/run-pass/sums.rs b/tests/run-pass/sums.rs
index 209629eabb..7f92f0d5ff 100644
--- a/tests/run-pass/sums.rs
+++ b/tests/run-pass/sums.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -53,3 +52,5 @@ fn match_opt_some() -> i8 {
 fn two_nones() -> (Option<i16>, Option<i16>) {
     (None, None)
 }
+
+fn main() {}
diff --git a/tests/run-pass/trivial.rs b/tests/run-pass/trivial.rs
index 9761e94792..e9d0f69464 100644
--- a/tests/run-pass/trivial.rs
+++ b/tests/run-pass/trivial.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -10,3 +9,5 @@ fn unit_var() {
     let x = ();
     x
 }
+
+fn main() {}
diff --git a/tests/run-pass/vecs.rs b/tests/run-pass/vecs.rs
index 63757947b0..325762289c 100644
--- a/tests/run-pass/vecs.rs
+++ b/tests/run-pass/vecs.rs
@@ -1,4 +1,3 @@
-#![crate_type = "lib"]
 #![feature(custom_attribute)]
 #![allow(dead_code, unused_attributes)]
 
@@ -36,3 +35,5 @@ fn vec_reallocate() -> Vec<u8> {
     v.push(5);
     v
 }
+
+fn main() {}