diff --git a/ci/script.sh b/ci/script.sh index b9e7589266..91ea7c1368 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -41,7 +41,7 @@ case "$BINDGEN_JOB" in "quickchecking") cd ./tests/quickchecking # TODO: Actually run quickchecks once `bindgen` is reliable enough. - cargo check + cargo test ;; *) echo "Error! Unknown \$BINDGEN_JOB: '$BINDGEN_JOB'" diff --git a/tests/quickchecking/Cargo.toml b/tests/quickchecking/Cargo.toml index 2f5b3b71bc..eb5cdfcf58 100644 --- a/tests/quickchecking/Cargo.toml +++ b/tests/quickchecking/Cargo.toml @@ -2,9 +2,19 @@ name = "quickchecking" description = "Bindgen property tests with quickcheck. Generate random valid C code and pass it to the csmith/predicate.py script" version = "0.1.0" -authors = ["Shea Newton "] +authors = ["Shea Newton "] + +[lib] +name = "quickchecking" +path = "src/lib.rs" + +[[bin]] +name = "quickchecking" +path = "src/bin.rs" [dependencies] +clap = "2.28" +lazy_static = "1.0" quickcheck = "0.4" -tempdir = "0.3" rand = "0.3" +tempdir = "0.3" diff --git a/tests/quickchecking/src/bin.rs b/tests/quickchecking/src/bin.rs new file mode 100644 index 0000000000..9cf313cdc4 --- /dev/null +++ b/tests/quickchecking/src/bin.rs @@ -0,0 +1,110 @@ +//! An application to run property tests for `bindgen` with _fuzzed_ C headers +//! using `quickcheck` +//! +//! ## Usage +//! +//! Print help +//! ```bash +//! $ cargo run --bin=quickchecking -- -h +//! ``` +//! +//! Run with default values +//! ```bash +//! $ cargo run --bin=quickchecking +//! ``` +//! +#![deny(missing_docs)] +extern crate clap; +extern crate quickchecking; + +use clap::{App, Arg}; +use std::path::Path; + +// Validate CLI argument input for generation range. +fn validate_generate_range(v: String) -> Result<(), String> { + match v.parse::() { + Ok(_) => Ok(()), + Err(_) => Err(String::from( + "Generate range could not be converted to a usize.", + )), + } +} + +// Validate CLI argument input for tests count. +fn validate_tests_count(v: String) -> Result<(), String> { + match v.parse::() { + Ok(_) => Ok(()), + Err(_) => Err(String::from( + "Tests count could not be converted to a usize.", + )), + } +} + +// Validate CLI argument input for fuzzed headers output path. +fn validate_path(v: String) -> Result<(), String> { + match Path::new(&v).is_dir() { + true => Ok(()), + false => Err(String::from("Provided directory path does not exist.")), + } +} + +fn main() { + let matches = App::new("quickchecking") + .version("0.2.0") + .about( + "Bindgen property tests with quickcheck. \ + Generate random valid C code and pass it to the \ + csmith/predicate.py script", + ) + .arg( + Arg::with_name("path") + .short("p") + .long("path") + .value_name("PATH") + .help( + "Optional. Preserve generated headers for inspection, \ + provide directory path for header output. [default: None] ", + ) + .takes_value(true) + .validator(validate_path), + ) + .arg( + Arg::with_name("range") + .short("r") + .long("range") + .value_name("RANGE") + .help( + "Sets the range quickcheck uses during generation. \ + Corresponds to things like arbitrary usize and \ + arbitrary vector length. This number doesn't have \ + to grow much for that execution time to increase \ + significantly.", + ) + .takes_value(true) + .default_value("32") + .validator(validate_generate_range), + ) + .arg( + Arg::with_name("count") + .short("c") + .long("count") + .value_name("COUNT") + .help( + "Count / number of tests to run. Running a fuzzed \ + header through the predicate.py script can take a \ + long time, especially if the generation range is \ + large. Increase this number if you're willing to \ + wait a while.", + ) + .takes_value(true) + .default_value("2") + .validator(validate_tests_count), + ) + .get_matches(); + + let output_path: Option<&str> = matches.value_of("path"); + let generate_range: usize = matches.value_of("range").unwrap().parse::().unwrap(); + let tests: usize = matches.value_of("count").unwrap().parse::().unwrap(); + + quickchecking::test_bindgen(generate_range, tests, output_path) +} diff --git a/tests/quickchecking/src/lib.rs b/tests/quickchecking/src/lib.rs index 3bea8a8ebb..d8633dfb92 100644 --- a/tests/quickchecking/src/lib.rs +++ b/tests/quickchecking/src/lib.rs @@ -20,9 +20,107 @@ //! ``` //! #![deny(missing_docs)] +#[macro_use] +extern crate lazy_static; extern crate quickcheck; extern crate rand; extern crate tempdir; +use std::sync::Mutex; +use quickcheck::{QuickCheck, StdGen, TestResult}; +use std::fs::File; +use std::io::Write; +use tempdir::TempDir; +use std::process::{Command, Output}; +use std::path::PathBuf; +use std::error::Error; +use rand::thread_rng; + /// Contains definitions of and impls for types used to fuzz C declarations. pub mod fuzzers; + +// Global singleton, manages context across tests. For now that context is +// only the output_path for inspecting fuzzed headers (if specified). +struct Context { + output_path: Option, +} + +// Initialize global context. +lazy_static! { + static ref CONTEXT: Mutex = Mutex::new(Context { output_path: None }); +} + +// Passes fuzzed header to the `csmith-fuzzing/predicate.py` script, returns +// output of the associated command. +fn run_predicate_script(header: fuzzers::HeaderC) -> Result> { + let dir = TempDir::new("bindgen_prop")?; + let header_path = dir.path().join("prop_test.h"); + + let mut header_file = File::create(&header_path)?; + header_file.write_all(header.to_string().as_bytes())?; + header_file.sync_all()?; + + let header_path_string; + match header_path.into_os_string().into_string() { + Ok(s) => header_path_string = s, + Err(_) => return Err(From::from("error converting path into String")), + } + + let mut predicate_script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + predicate_script_path.push("../../csmith-fuzzing/predicate.py"); + + let predicate_script_path_string; + match predicate_script_path.into_os_string().into_string() { + Ok(s) => predicate_script_path_string = s, + Err(_) => return Err(From::from("error converting path into String")), + } + + // Copy generated temp files to output_path directory for inspection. + // If `None`, output path not specified, don't copy. + match CONTEXT.lock().unwrap().output_path { + Some(ref path) => { + Command::new("cp") + .arg("-a") + .arg(&dir.path().to_str().unwrap()) + .arg(&path) + .output()?; + } + None => {} + } + + Ok(Command::new(&predicate_script_path_string) + .arg(&header_path_string) + .output()?) +} + +// Generatable property. Pass generated headers off to run through the +// `csmith-fuzzing/predicate.py` script. Success is measured by the success +// status of that command. +fn bindgen_prop(header: fuzzers::HeaderC) -> TestResult { + match run_predicate_script(header) { + Ok(o) => return TestResult::from_bool(o.status.success()), + Err(e) => { + println!("{:?}", e); + return TestResult::from_bool(false); + } + } +} + +/// Instantiate a Quickcheck object and use it to run property tests using +/// fuzzed C headers generated with types defined in the `fuzzers` module. +/// Success/Failure is dictated by the result of passing the fuzzed headers +/// to the `csmith-fuzzing/predicate.py` script. +pub fn test_bindgen(generate_range: usize, tests: usize, output_path: Option<&str>) { + match output_path { + Some(path) => { + CONTEXT.lock().unwrap().output_path = + Some(String::from(PathBuf::from(path).to_str().unwrap())); + } + None => {} // Path not specified, don't provide output. + } + + QuickCheck::new() + .tests(tests) + .gen(StdGen::new(thread_rng(), generate_range)) + .quickcheck(bindgen_prop as fn(fuzzers::HeaderC) -> TestResult) +} diff --git a/tests/quickchecking/tests/fuzzed-c-headers.rs b/tests/quickchecking/tests/fuzzed-c-headers.rs index f550cf0c27..6b58d24b23 100644 --- a/tests/quickchecking/tests/fuzzed-c-headers.rs +++ b/tests/quickchecking/tests/fuzzed-c-headers.rs @@ -1,78 +1,95 @@ + extern crate quickcheck; extern crate quickchecking; extern crate rand; -extern crate tempdir; - -use quickchecking::fuzzers; -use quickcheck::{QuickCheck, StdGen, TestResult}; -use std::fs::File; -use std::io::Write; -use tempdir::TempDir; -use std::process::{Command, Output}; -use std::path::PathBuf; -use std::error::Error; + +use quickchecking::fuzzers::{ArrayDimensionC, BaseTypeC, BasicTypeDeclarationC, DeclarationC, + DeclarationListC, FunctionPointerDeclarationC, FunctionPrototypeC, + HeaderC, ParameterC, ParameterListC, PointerLevelC, + StructDeclarationC, TypeQualifierC, UnionDeclarationC}; +use quickcheck::{Arbitrary, StdGen}; use rand::thread_rng; -fn run_predicate_script(header: fuzzers::HeaderC, header_name: &str) -> Result> { - let dir = TempDir::new("bindgen_prop")?; - let header_path = dir.path().join(header_name); - - let mut header_file = File::create(&header_path)?; - header_file.write_all(header.to_string().as_bytes())?; - header_file.sync_all()?; - - let header_path_string; - match header_path.into_os_string().into_string() { - Ok(s) => header_path_string = s, - Err(_) => return Err(From::from("error converting path into String")), - } - - let mut predicate_script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - predicate_script_path.push("../../csmith-fuzzing/predicate.py"); - - let predicate_script_path_string; - match predicate_script_path.into_os_string().into_string() { - Ok(s) => predicate_script_path_string = s, - Err(_) => return Err(From::from("error converting path into String")), - } - - // Copy generated temp files to test directory for inspection. - // Preserved for anyone interested in validating the behavior. - - let mut debug_output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - debug_output_path.push("tests"); - Command::new("cp") - .arg("-a") - .arg(&dir.path().to_str().unwrap()) - .arg(&debug_output_path.to_str().unwrap()) - .output()?; - - Ok(Command::new(&predicate_script_path_string) - .arg(&header_path_string) - .output()?) +#[test] +fn test_declaraion_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: DeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_declaraion_list_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: DeclarationListC = Arbitrary::arbitrary(gen); } -fn bindgen_prop(header: fuzzers::HeaderC) -> TestResult { - match run_predicate_script(header, "prop_test.h") { - Ok(o) => return TestResult::from_bool(o.status.success()), - Err(e) => { - println!("{:?}", e); - return TestResult::from_bool(false); - } - } +#[test] +fn test_base_type_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: BaseTypeC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_type_qualifier_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: TypeQualifierC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_pointer_level_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: PointerLevelC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_array_dimension_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: ArrayDimensionC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_basic_type_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: BasicTypeDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_struct_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: StructDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_union_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: UnionDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_function_pointer_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: FunctionPointerDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_function_prototype_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: FunctionPrototypeC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_parameter_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: ParameterC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_parameter_list_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: ParameterListC = Arbitrary::arbitrary(gen); } #[test] -fn test_bindgen() { - // Enough to generate any value in the PrimitiveTypeC `base_type` list. - let generate_range: usize = 32; - QuickCheck::new() - // Generating is relatively quick (generate_range 150 takes ~5 seconds) - // but running predicate.py takes ~30 seconds per source file / test - // when the generation range is just 32. It can take a lot longer with a - // higher generate_range. Up the number of tests or generate_range if - // you're willing to wait awhile. - .tests(2) - .gen(StdGen::new(thread_rng(), generate_range)) - .quickcheck(bindgen_prop as fn(fuzzers::HeaderC) -> TestResult) +fn test_header_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: HeaderC = Arbitrary::arbitrary(gen); }