From b4ba0923fb00c916584fd2f79455bf179efa1558 Mon Sep 17 00:00:00 2001 From: Shea Newton Date: Sun, 3 Dec 2017 18:29:06 -0800 Subject: [PATCH 1/2] Quickchecking crate CLI Prior to this commit the quickchecking crate used for generating proprty tests for bindgen was a [lib] target and had configurations that required commenting/uncommenting code to enable/disable. This meant it was inconvienent/prohibitive to configure the property tests on a per-run basis. This commit reorganizes the `quickchecking` crate to provide both [lib] and [[bin]] targets in order to expose those configurations through a CLI. The configurations that are exposed through the [[bin]] target's CLI are: * Count/number of tests to run. * Directory to provide fuzzed headers * Generation range corresponding to the range quickcheck uses to * generate arbitrary. __Usage from the__ `tests/quickchecking` __directory__ ```bash quickchecking 0.2.0 Bindgen property tests with quickcheck. Generate random valid C code and pass it to the csmith/predicate.py script USAGE: quickchecking [OPTIONS] FLAGS: -h, --help Prints help information -V, --version Prints version information OPTIONS: -c, --count Count / number of tests to run. Running a fuzzed header through the predicate.py script can take a long time, especially if the generation range is large. Increase this number if you're willing to wait a while. [default: 2] -p, --path Optional. Preserve generated headers for inspection, provide directory path for header output. [default: None] -r, --range Sets the range quickcheck uses during generation. Corresponds to things like arbitrary usize and arbitrary vector length. This number doesn't have to grow much for that execution time to increase significantly. [default: 32] ``` Because the actual work of running the property tests moved to the [[bin]] target, rather than duplicate that code in the `quickchecking` crate's tests directory, some actual (very basic) tests for the `quickchecking` crate were added. *Note: I'm not attached to any of the option flags, if there are better characters/words for any of the options I've exposed I'll be happy to revise! Thanks for taking a look, looking forward to feedback! Closes #1168 r? @fitzgen --- tests/quickchecking/Cargo.toml | 14 +- tests/quickchecking/src/bin.rs | 110 +++++++++++++ tests/quickchecking/src/lib.rs | 98 ++++++++++++ tests/quickchecking/tests/fuzzed-c-headers.rs | 151 ++++++++++-------- 4 files changed, 304 insertions(+), 69 deletions(-) create mode 100644 tests/quickchecking/src/bin.rs diff --git a/tests/quickchecking/Cargo.toml b/tests/quickchecking/Cargo.toml index 2f5b3b71bc..eb5cdfcf58 100644 --- a/tests/quickchecking/Cargo.toml +++ b/tests/quickchecking/Cargo.toml @@ -2,9 +2,19 @@ name = "quickchecking" description = "Bindgen property tests with quickcheck. Generate random valid C code and pass it to the csmith/predicate.py script" version = "0.1.0" -authors = ["Shea Newton "] +authors = ["Shea Newton "] + +[lib] +name = "quickchecking" +path = "src/lib.rs" + +[[bin]] +name = "quickchecking" +path = "src/bin.rs" [dependencies] +clap = "2.28" +lazy_static = "1.0" quickcheck = "0.4" -tempdir = "0.3" rand = "0.3" +tempdir = "0.3" diff --git a/tests/quickchecking/src/bin.rs b/tests/quickchecking/src/bin.rs new file mode 100644 index 0000000000..9cf313cdc4 --- /dev/null +++ b/tests/quickchecking/src/bin.rs @@ -0,0 +1,110 @@ +//! An application to run property tests for `bindgen` with _fuzzed_ C headers +//! using `quickcheck` +//! +//! ## Usage +//! +//! Print help +//! ```bash +//! $ cargo run --bin=quickchecking -- -h +//! ``` +//! +//! Run with default values +//! ```bash +//! $ cargo run --bin=quickchecking +//! ``` +//! +#![deny(missing_docs)] +extern crate clap; +extern crate quickchecking; + +use clap::{App, Arg}; +use std::path::Path; + +// Validate CLI argument input for generation range. +fn validate_generate_range(v: String) -> Result<(), String> { + match v.parse::() { + Ok(_) => Ok(()), + Err(_) => Err(String::from( + "Generate range could not be converted to a usize.", + )), + } +} + +// Validate CLI argument input for tests count. +fn validate_tests_count(v: String) -> Result<(), String> { + match v.parse::() { + Ok(_) => Ok(()), + Err(_) => Err(String::from( + "Tests count could not be converted to a usize.", + )), + } +} + +// Validate CLI argument input for fuzzed headers output path. +fn validate_path(v: String) -> Result<(), String> { + match Path::new(&v).is_dir() { + true => Ok(()), + false => Err(String::from("Provided directory path does not exist.")), + } +} + +fn main() { + let matches = App::new("quickchecking") + .version("0.2.0") + .about( + "Bindgen property tests with quickcheck. \ + Generate random valid C code and pass it to the \ + csmith/predicate.py script", + ) + .arg( + Arg::with_name("path") + .short("p") + .long("path") + .value_name("PATH") + .help( + "Optional. Preserve generated headers for inspection, \ + provide directory path for header output. [default: None] ", + ) + .takes_value(true) + .validator(validate_path), + ) + .arg( + Arg::with_name("range") + .short("r") + .long("range") + .value_name("RANGE") + .help( + "Sets the range quickcheck uses during generation. \ + Corresponds to things like arbitrary usize and \ + arbitrary vector length. This number doesn't have \ + to grow much for that execution time to increase \ + significantly.", + ) + .takes_value(true) + .default_value("32") + .validator(validate_generate_range), + ) + .arg( + Arg::with_name("count") + .short("c") + .long("count") + .value_name("COUNT") + .help( + "Count / number of tests to run. Running a fuzzed \ + header through the predicate.py script can take a \ + long time, especially if the generation range is \ + large. Increase this number if you're willing to \ + wait a while.", + ) + .takes_value(true) + .default_value("2") + .validator(validate_tests_count), + ) + .get_matches(); + + let output_path: Option<&str> = matches.value_of("path"); + let generate_range: usize = matches.value_of("range").unwrap().parse::().unwrap(); + let tests: usize = matches.value_of("count").unwrap().parse::().unwrap(); + + quickchecking::test_bindgen(generate_range, tests, output_path) +} diff --git a/tests/quickchecking/src/lib.rs b/tests/quickchecking/src/lib.rs index 3bea8a8ebb..d8633dfb92 100644 --- a/tests/quickchecking/src/lib.rs +++ b/tests/quickchecking/src/lib.rs @@ -20,9 +20,107 @@ //! ``` //! #![deny(missing_docs)] +#[macro_use] +extern crate lazy_static; extern crate quickcheck; extern crate rand; extern crate tempdir; +use std::sync::Mutex; +use quickcheck::{QuickCheck, StdGen, TestResult}; +use std::fs::File; +use std::io::Write; +use tempdir::TempDir; +use std::process::{Command, Output}; +use std::path::PathBuf; +use std::error::Error; +use rand::thread_rng; + /// Contains definitions of and impls for types used to fuzz C declarations. pub mod fuzzers; + +// Global singleton, manages context across tests. For now that context is +// only the output_path for inspecting fuzzed headers (if specified). +struct Context { + output_path: Option, +} + +// Initialize global context. +lazy_static! { + static ref CONTEXT: Mutex = Mutex::new(Context { output_path: None }); +} + +// Passes fuzzed header to the `csmith-fuzzing/predicate.py` script, returns +// output of the associated command. +fn run_predicate_script(header: fuzzers::HeaderC) -> Result> { + let dir = TempDir::new("bindgen_prop")?; + let header_path = dir.path().join("prop_test.h"); + + let mut header_file = File::create(&header_path)?; + header_file.write_all(header.to_string().as_bytes())?; + header_file.sync_all()?; + + let header_path_string; + match header_path.into_os_string().into_string() { + Ok(s) => header_path_string = s, + Err(_) => return Err(From::from("error converting path into String")), + } + + let mut predicate_script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + predicate_script_path.push("../../csmith-fuzzing/predicate.py"); + + let predicate_script_path_string; + match predicate_script_path.into_os_string().into_string() { + Ok(s) => predicate_script_path_string = s, + Err(_) => return Err(From::from("error converting path into String")), + } + + // Copy generated temp files to output_path directory for inspection. + // If `None`, output path not specified, don't copy. + match CONTEXT.lock().unwrap().output_path { + Some(ref path) => { + Command::new("cp") + .arg("-a") + .arg(&dir.path().to_str().unwrap()) + .arg(&path) + .output()?; + } + None => {} + } + + Ok(Command::new(&predicate_script_path_string) + .arg(&header_path_string) + .output()?) +} + +// Generatable property. Pass generated headers off to run through the +// `csmith-fuzzing/predicate.py` script. Success is measured by the success +// status of that command. +fn bindgen_prop(header: fuzzers::HeaderC) -> TestResult { + match run_predicate_script(header) { + Ok(o) => return TestResult::from_bool(o.status.success()), + Err(e) => { + println!("{:?}", e); + return TestResult::from_bool(false); + } + } +} + +/// Instantiate a Quickcheck object and use it to run property tests using +/// fuzzed C headers generated with types defined in the `fuzzers` module. +/// Success/Failure is dictated by the result of passing the fuzzed headers +/// to the `csmith-fuzzing/predicate.py` script. +pub fn test_bindgen(generate_range: usize, tests: usize, output_path: Option<&str>) { + match output_path { + Some(path) => { + CONTEXT.lock().unwrap().output_path = + Some(String::from(PathBuf::from(path).to_str().unwrap())); + } + None => {} // Path not specified, don't provide output. + } + + QuickCheck::new() + .tests(tests) + .gen(StdGen::new(thread_rng(), generate_range)) + .quickcheck(bindgen_prop as fn(fuzzers::HeaderC) -> TestResult) +} diff --git a/tests/quickchecking/tests/fuzzed-c-headers.rs b/tests/quickchecking/tests/fuzzed-c-headers.rs index f550cf0c27..6b58d24b23 100644 --- a/tests/quickchecking/tests/fuzzed-c-headers.rs +++ b/tests/quickchecking/tests/fuzzed-c-headers.rs @@ -1,78 +1,95 @@ + extern crate quickcheck; extern crate quickchecking; extern crate rand; -extern crate tempdir; - -use quickchecking::fuzzers; -use quickcheck::{QuickCheck, StdGen, TestResult}; -use std::fs::File; -use std::io::Write; -use tempdir::TempDir; -use std::process::{Command, Output}; -use std::path::PathBuf; -use std::error::Error; + +use quickchecking::fuzzers::{ArrayDimensionC, BaseTypeC, BasicTypeDeclarationC, DeclarationC, + DeclarationListC, FunctionPointerDeclarationC, FunctionPrototypeC, + HeaderC, ParameterC, ParameterListC, PointerLevelC, + StructDeclarationC, TypeQualifierC, UnionDeclarationC}; +use quickcheck::{Arbitrary, StdGen}; use rand::thread_rng; -fn run_predicate_script(header: fuzzers::HeaderC, header_name: &str) -> Result> { - let dir = TempDir::new("bindgen_prop")?; - let header_path = dir.path().join(header_name); - - let mut header_file = File::create(&header_path)?; - header_file.write_all(header.to_string().as_bytes())?; - header_file.sync_all()?; - - let header_path_string; - match header_path.into_os_string().into_string() { - Ok(s) => header_path_string = s, - Err(_) => return Err(From::from("error converting path into String")), - } - - let mut predicate_script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - predicate_script_path.push("../../csmith-fuzzing/predicate.py"); - - let predicate_script_path_string; - match predicate_script_path.into_os_string().into_string() { - Ok(s) => predicate_script_path_string = s, - Err(_) => return Err(From::from("error converting path into String")), - } - - // Copy generated temp files to test directory for inspection. - // Preserved for anyone interested in validating the behavior. - - let mut debug_output_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - debug_output_path.push("tests"); - Command::new("cp") - .arg("-a") - .arg(&dir.path().to_str().unwrap()) - .arg(&debug_output_path.to_str().unwrap()) - .output()?; - - Ok(Command::new(&predicate_script_path_string) - .arg(&header_path_string) - .output()?) +#[test] +fn test_declaraion_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: DeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_declaraion_list_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: DeclarationListC = Arbitrary::arbitrary(gen); } -fn bindgen_prop(header: fuzzers::HeaderC) -> TestResult { - match run_predicate_script(header, "prop_test.h") { - Ok(o) => return TestResult::from_bool(o.status.success()), - Err(e) => { - println!("{:?}", e); - return TestResult::from_bool(false); - } - } +#[test] +fn test_base_type_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: BaseTypeC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_type_qualifier_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: TypeQualifierC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_pointer_level_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: PointerLevelC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_array_dimension_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: ArrayDimensionC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_basic_type_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: BasicTypeDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_struct_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: StructDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_union_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: UnionDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_function_pointer_declaration_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: FunctionPointerDeclarationC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_function_prototype_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: FunctionPrototypeC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_parameter_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: ParameterC = Arbitrary::arbitrary(gen); +} + +#[test] +fn test_parameter_list_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: ParameterListC = Arbitrary::arbitrary(gen); } #[test] -fn test_bindgen() { - // Enough to generate any value in the PrimitiveTypeC `base_type` list. - let generate_range: usize = 32; - QuickCheck::new() - // Generating is relatively quick (generate_range 150 takes ~5 seconds) - // but running predicate.py takes ~30 seconds per source file / test - // when the generation range is just 32. It can take a lot longer with a - // higher generate_range. Up the number of tests or generate_range if - // you're willing to wait awhile. - .tests(2) - .gen(StdGen::new(thread_rng(), generate_range)) - .quickcheck(bindgen_prop as fn(fuzzers::HeaderC) -> TestResult) +fn test_header_c_does_not_panic() { + let ref mut gen = StdGen::new(thread_rng(), 50); + let _: HeaderC = Arbitrary::arbitrary(gen); } From 2c776522aea807a37a78e47154c128c198b5d985 Mon Sep 17 00:00:00 2001 From: Shea Newton Date: Tue, 5 Dec 2017 18:31:56 -0800 Subject: [PATCH 2/2] quickchecking CI tests The changes reflected in this PR include the logic to test the `quickcheking` crate itself. Rather that just validate that the `quickchecking` crate builds in CI with `cargo check`, we can run now `cargo test`. --- ci/script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/script.sh b/ci/script.sh index b9e7589266..91ea7c1368 100755 --- a/ci/script.sh +++ b/ci/script.sh @@ -41,7 +41,7 @@ case "$BINDGEN_JOB" in "quickchecking") cd ./tests/quickchecking # TODO: Actually run quickchecks once `bindgen` is reliable enough. - cargo check + cargo test ;; *) echo "Error! Unknown \$BINDGEN_JOB: '$BINDGEN_JOB'"