diff --git a/Cargo.lock b/Cargo.lock index e644eed..8823fdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,7 @@ dependencies = [ "clap", "console", "include_dir", + "indexmap", "kdl", "kdl-script", "libloading", @@ -24,6 +25,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "toml 0.8.14", "tracing", "tracing-subscriber", ] @@ -169,7 +171,7 @@ checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996" dependencies = [ "semver", "serde", - "toml", + "toml 0.5.11", "url", ] @@ -354,6 +356,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] @@ -841,6 +844,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1048,6 +1060,41 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1331,3 +1378,12 @@ name = "windows_x86_64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] diff --git a/Cargo.toml b/Cargo.toml index ad8d6c9..dea5f5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ cc.workspace = true clap.workspace = true console.workspace = true kdl.workspace = true +include_dir.workspace = true +indexmap.workspace = true libloading.workspace = true linked-hash-map.workspace = true miette.workspace = true @@ -32,7 +34,8 @@ thiserror.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -include_dir = "0.7.4" +toml.workspace = true + [build-dependencies] built.workspace = true @@ -60,6 +63,8 @@ camino = { version = "1.1.7", features = ["serde1"] } cc = { version = "1.1.0" } clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] } console = "0.15.8" +include_dir = "0.7.4" +indexmap = { version = "2.2.6", features = ["serde"] } kdl = "4.6.0" libloading = "0.7.3" linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] } @@ -73,6 +78,7 @@ serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.83" thiserror = "1.0.30" tokio = { version = "1.37.0", features = ["full", "tracing"] } +toml = { version = "0.8.14", features = ["preserve_order"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } # build diff --git a/include/harness/abi-cafe-rules.toml b/include/harness/abi-cafe-rules.toml new file mode 100644 index 0000000..29b29ce --- /dev/null +++ b/include/harness/abi-cafe-rules.toml @@ -0,0 +1,60 @@ +# i128 types are fake on windows so this is all random garbage that might +# not even compile, but that datapoint is a little interesting/useful +# so let's keep running them and just ignore the result for now. +# +# Anyone who cares about this situation more can make the expectations more precise. +[targets.x86_64-pc-windows-msvc."i128::cc_toolchain"] +random = true +[targets.x86_64-pc-windows-msvc."u128::cc_toolchain"] +random = true + +# FIXME: investigate why this is failing to build +[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] +busted = "build" +[targets.x86_64-pc-windows-msvc."EmptyStructInside::cc_toolchain"] +busted = "build" + +# CI GCC is too old to support _Float16 +[targets.x86_64-unknown-linux-gnu."f16::conv_c"] +random = true + + + +# +# +# Here are some example annotations for test expecations +# +# + +# this test fails on this platform, with this toolchain pairing +# +# [targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"] +# fail = "check" + +# this test has random results on this platform, whenever rustc is the caller (callee also supported) +# +# [targets.x86_64-pc-windows-msvc."simple::rustc_caller"] +# random = true + +# whenever this test involves cc, only link it, and expect linking to fail +# +# [targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] +# run = "link" +# fail = "link" + +# any repr(c) version of this test fails to run +# +# [targets.x86_64-unknown-linux-gnu."simple::repr_c"] +# busted = "run" + +# for this pairing, with the rust calling convention, only generate the test, and expect it to work +# +# [targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"] +# run = "generate" +# pass = "generate" + +# can match all tests with leading :: +# +# [targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"] +# run = "generate" +# pass = "generate" diff --git a/kdl-script/src/parse.rs b/kdl-script/src/parse.rs index 5b336b2..f26d6a4 100644 --- a/kdl-script/src/parse.rs +++ b/kdl-script/src/parse.rs @@ -218,7 +218,7 @@ pub enum Repr { Transparent, } -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, serde::Serialize)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)] pub enum LangRepr { Rust, C, diff --git a/src/cli.rs b/src/cli.rs index 0e08daf..445c2cf 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -130,12 +130,27 @@ struct Cli { #[clap(long)] add_tests: Option, + /// Add the test expectations at the given path + /// + /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir) + /// + /// Note that there are already builtin rules (disabled with `--disable-builtin-rules`), + /// and it would be nice for rules to be upstreamed so everyone can benefit! + #[clap(long)] + rules: Option, + /// disable the builtin tests /// /// See also `--add-tests` #[clap(long)] disable_builtin_tests: bool, + /// disable the builtin rules + /// + /// See also `--add-rules` + #[clap(long)] + disable_builtin_rules: bool, + /// deprecated, does nothing (we always procgen now) #[clap(long, hide = true)] procgen_tests: bool, @@ -154,7 +169,9 @@ pub fn make_app() -> Config { output_format, add_rustc_codegen_backend, add_tests, + rules, disable_builtin_tests, + disable_builtin_rules, // unimplemented select_vals: _, key: _, @@ -228,12 +245,14 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`. let out_dir = target_dir.join("temp"); let generated_src_dir = target_dir.join("generated_impls"); let runtime_test_input_dir = add_tests; + let runtime_rules_file = rules.unwrap_or_else(|| "abi-cafe-rules.toml".into()); let paths = Paths { target_dir, out_dir, generated_src_dir, runtime_test_input_dir, + runtime_rules_file, }; Config { output_format, @@ -248,6 +267,7 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`. run_selections, minimizing_write_impl, disable_builtin_tests, + disable_builtin_rules, paths, } } diff --git a/src/error.rs b/src/error.rs index 723da86..ec2cf14 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,8 @@ pub enum GenerateError { #[error(transparent)] #[diagnostic(transparent)] KdlScriptError(#[from] kdl_script::KdlScriptError), + #[error(transparent)] + TomlError(#[from] toml::de::Error), /// Used to signal we just skipped it #[error("")] Skipped, @@ -72,13 +74,13 @@ pub enum BuildError { #[derive(Debug, thiserror::Error, Diagnostic)] pub enum CheckFailure { #[error( - " func {func_name}'s values differed - values (native-endian hex bytes): - expect: {} - caller: {} - callee: {} - the value was {val_path}: {val_ty_name} - whose arg was {arg_name}: {arg_ty_name}", + " func {func_name}'s values differed + values (native-endian hex bytes): + expect: {} + caller: {} + callee: {} + the value was {val_path}: {val_ty_name} + whose arg was {arg_name}: {arg_ty_name}", fmt_bytes(expected), fmt_bytes(caller), fmt_bytes(callee) @@ -97,13 +99,13 @@ pub enum CheckFailure { callee: Vec, }, #[error( - " func {func_name}'s value had unexpected variant - values: - expect: {expected} - caller: {caller} - callee: {callee} - the value was {val_path}: {val_ty_name} - whose arg was {arg_name}: {arg_ty_name}" + " func {func_name}'s value had unexpected variant + values: + expect: {expected} + caller: {caller} + callee: {callee} + the value was {val_path}: {val_ty_name} + whose arg was {arg_name}: {arg_ty_name}" )] TagMismatch { func_idx: usize, diff --git a/src/files.rs b/src/files.rs index f5108b1..982c6c1 100644 --- a/src/files.rs +++ b/src/files.rs @@ -13,6 +13,7 @@ pub struct Paths { pub out_dir: Utf8PathBuf, pub generated_src_dir: Utf8PathBuf, pub runtime_test_input_dir: Option, + pub runtime_rules_file: Utf8PathBuf, } impl Paths { pub fn harness_dylib_main_file(&self) -> Utf8PathBuf { @@ -89,12 +90,16 @@ pub fn load_file(file: &File) -> String { clean_newlines(string) } -pub fn tests() -> &'static Dir<'static> { +pub fn static_tests() -> &'static Dir<'static> { INCLUDES .get_dir("tests") .expect("includes didn't contain ./test") } +pub fn static_rules() -> String { + get_file("harness/abi-cafe-rules.toml") +} + fn clean_newlines(input: &str) -> String { input.replace('\r', "") } diff --git a/src/harness/build.rs b/src/harness/build.rs index ed70068..4d80b13 100644 --- a/src/harness/build.rs +++ b/src/harness/build.rs @@ -5,8 +5,8 @@ use camino::Utf8Path; use tracing::info; use crate::error::*; +use crate::harness::report::*; use crate::harness::test::*; -use crate::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/check.rs b/src/harness/check.rs index 38cf7f5..88d2de4 100644 --- a/src/harness/check.rs +++ b/src/harness/check.rs @@ -5,7 +5,6 @@ use kdl_script::types::Ty; use tracing::{error, info}; use crate::error::*; -use crate::report::*; use crate::*; impl TestHarness { @@ -29,7 +28,7 @@ impl TestHarness { // Start peeling back the layers of the buffers. // funcs (subtests) -> vals (args/returns) -> fields -> bytes - let mut results: Vec> = Vec::new(); + let mut results: Vec = Vec::new(); // `Run` already checks that this length is congruent with all the inputs/outputs Vecs let expected_funcs = key.options.functions.active_funcs(&test.types); @@ -54,7 +53,10 @@ impl TestHarness { let caller_val = caller_func.vals.get(val_idx).unwrap_or(&empty_val); let callee_val = callee_func.vals.get(val_idx).unwrap_or(&empty_val); if let Err(e) = self.check_val(&test, expected_val, caller_val, callee_val) { - results.push(Err(e)); + results.push(SubtestDetails { + result: Err(e), + minimized: None, + }); // FIXME: now that each value is absolutely indexed, // we should be able to check all the values independently // and return all errors. However the first one is the most @@ -64,7 +66,10 @@ impl TestHarness { } // If we got this far then the test passes - results.push(Ok(())); + results.push(SubtestDetails { + result: Ok(()), + minimized: None, + }); } // Report the results of each subtest @@ -78,12 +83,12 @@ impl TestHarness { .map(|func_id| self.full_subtest_name(key, &test.types.realize_func(func_id).name)) .collect::>(); let max_name_len = names.iter().fold(0, |max, name| max.max(name.len())); - let num_passed = results.iter().filter(|r| r.is_ok()).count(); + let num_passed = results.iter().filter(|t| t.result.is_ok()).count(); let all_passed = num_passed == results.len(); if !all_passed { - for (subtest_name, result) in names.iter().zip(&results) { - match result { + for (subtest_name, subtest) in names.iter().zip(&results) { + match &subtest.result { Ok(()) => { info!("Test {subtest_name:width$} passed", width = max_name_len); } diff --git a/src/harness/mod.rs b/src/harness/mod.rs index ae8379a..c06f0a0 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -17,11 +17,12 @@ mod build; mod check; mod generate; mod read; +pub mod report; mod run; pub mod test; pub mod vals; -pub use read::{find_tests, spawn_read_test}; +pub use read::{find_test_rules, find_tests, spawn_read_test}; pub use run::TestBuffer; pub type Memoized = Mutex>>>; @@ -30,6 +31,7 @@ pub struct TestHarness { paths: Paths, toolchains: Toolchains, tests: SortedMap>, + test_rules: Vec, tests_with_vals: Memoized<(TestId, ValueGeneratorKind), Arc>, tests_with_toolchain: Memoized<(TestId, ValueGeneratorKind, ToolchainId), Arc>, @@ -39,11 +41,16 @@ pub struct TestHarness { } impl TestHarness { - pub fn new(tests: SortedMap>, cfg: &Config) -> Self { + pub fn new( + test_rules: Vec, + tests: SortedMap>, + cfg: &Config, + ) -> Self { let toolchains = toolchains::create_toolchains(cfg); Self { paths: cfg.paths.clone(), tests, + test_rules, toolchains, tests_with_vals: Default::default(), tests_with_toolchain: Default::default(), @@ -109,13 +116,6 @@ impl TestHarness { .clone(); Ok(output) } - pub fn get_test_rules(&self, test_key: &TestKey) -> TestRules { - let caller = self.toolchains[&test_key.caller].clone(); - let callee = self.toolchains[&test_key.callee].clone(); - - get_test_rules(test_key, &*caller, &*callee) - } - pub fn spawn_test( self: Arc, rt: &tokio::runtime::Runtime, @@ -262,26 +262,18 @@ impl TestHarness { WriteImpl::HarnessCallback => { // Do nothing, implicit default } - WriteImpl::Print => { - output.push_str(separator); - output.push_str("print"); - } - WriteImpl::Assert => { - output.push_str(separator); - output.push_str("assert"); - } - WriteImpl::Noop => { + other => { output.push_str(separator); - output.push_str("noop"); + output.push_str(&other.to_string()) } } match val_generator { ValueGeneratorKind::Graffiti => { // Do nothing, implicit default } - ValueGeneratorKind::Random { seed } => { + other => { output.push_str(separator); - output.push_str(&format!("random{seed}")); + output.push_str(&other.to_string()) } } output diff --git a/src/harness/read.rs b/src/harness/read.rs index a05fc95..9b56d21 100644 --- a/src/harness/read.rs +++ b/src/harness/read.rs @@ -32,6 +32,32 @@ impl Pathish { } } +pub fn find_test_rules(cfg: &Config) -> Result, GenerateError> { + let static_rules = find_test_rules_static(cfg.disable_builtin_rules)?; + let rules = find_test_rules_runtime(&cfg.paths.runtime_rules_file)?; + Ok(vec![static_rules, rules]) +} + +pub fn find_test_rules_static(disable_builtin_rules: bool) -> Result { + let rules = if disable_builtin_rules { + ExpectFile::default() + } else { + let data = files::static_rules(); + toml::from_str(&data)? + }; + Ok(rules) +} + +pub fn find_test_rules_runtime(rule_file: &Utf8Path) -> Result { + if rule_file.exists() { + let data = read_runtime_file_to_string(rule_file)?; + let rules = toml::from_str(&data)?; + Ok(rules) + } else { + Ok(ExpectFile::default()) + } +} + pub fn find_tests(cfg: &Config) -> Result, GenerateError> { let mut tests = find_tests_runtime(cfg.paths.runtime_test_input_dir.as_deref())?; let mut more_tests = find_tests_static(cfg.disable_builtin_tests)?; @@ -77,7 +103,7 @@ pub fn find_tests_static( return Ok(tests); } - let mut dirs = vec![crate::files::tests()]; + let mut dirs = vec![crate::files::static_tests()]; while let Some(dir) = dirs.pop() { for entry in dir.entries() { // If it's a dir, add it to the working set diff --git a/src/report.rs b/src/harness/report.rs similarity index 54% rename from src/report.rs rename to src/harness/report.rs index 958ab43..847f31e 100644 --- a/src/report.rs +++ b/src/harness/report.rs @@ -1,58 +1,63 @@ use camino::Utf8PathBuf; use console::Style; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::json; use crate::error::*; use crate::harness::test::*; -use crate::toolchains::*; use crate::*; /// These are the builtin test-expectations, edit these if there are new rules! -#[allow(unused_variables)] -pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolchain) -> TestRules { - use TestCheckMode::*; - use TestRunMode::*; - - // By default, require tests to run completely and pass - let mut result = TestRules { - run: Check, - check: Pass(Check), - }; +impl TestHarness { + #[allow(unused_variables)] + pub fn get_test_rules(&self, key: &TestKey) -> TestRules { + use TestCheckMode::*; + use TestRunMode::*; - // Now apply specific custom expectations for platforms/suites - let is_c = caller.lang() == "c" || callee.lang() == "c"; - let is_rust = caller.lang() == "rust" || callee.lang() == "rust"; - let is_rust_and_c = is_c && is_rust; - - // i128 types are fake on windows so this is all random garbage that might - // not even compile, but that datapoint is a little interesting/useful - // so let's keep running them and just ignore the result for now. - // - // Anyone who cares about this situation more can make the expectations more precise. - if cfg!(windows) && (test.test == "i128" || test.test == "u128") { - result.check = Random; - } + // By default, require tests to run completely and pass + let mut result = TestRules { + run: Check, + check: Pass(Check), + }; - // CI GCC is too old to support `_Float16`. - if cfg!(all(target_arch = "x86_64", target_os = "linux")) && is_c && test.test == "f16" { - result.check = Random; - } + for expect_file in &self.test_rules { + let rulesets = [ + expect_file.targets.get("*"), + expect_file.targets.get(built_info::TARGET), + ]; + for rules in rulesets { + let Some(rules) = rules else { + continue; + }; + for (pattern, rules) in rules { + if pattern.matches(key) { + if let Some(run) = rules.run { + result.run = run; + } + if let Some(check) = rules.check { + result.check = check; + } + } + } + } + } - // FIXME: investigate why this is failing to build - if cfg!(windows) && is_c && (test.test == "EmptyStruct" || test.test == "EmptyStructInside") { - result.check = Busted(Build); - } + // + // + // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES - // - // - // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + // END OF VENDOR RESERVED AREA + // + // - // END OF VENDOR RESERVED AREA - // - // + result + } +} - result +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ExpectFile { + #[serde(default)] + pub targets: IndexMap>, } impl Serialize for BuildError { @@ -123,33 +128,12 @@ pub fn report_test(results: TestRunResults) -> TestReport { Skipped } else { let passed = match &results.rules.check { - TestCheckMode::Pass(must_pass) => match must_pass { - Skip => true, - Generate => results.source.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Build => results.build.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Link => results.link.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Run => results.run.as_ref().map(|r| r.is_ok()).unwrap_or(false), - Check => results - .check - .as_ref() - .map(|r| r.all_passed) - .unwrap_or(false), - }, - TestCheckMode::Fail(must_fail) | TestCheckMode::Busted(must_fail) => match must_fail { - Skip => true, - Generate => results.source.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Build => results.build.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Link => results.link.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Run => results.run.as_ref().map(|r| !r.is_ok()).unwrap_or(false), - Check => results - .check - .as_ref() - .map(|r| !r.all_passed) - .unwrap_or(false), - }, - TestCheckMode::Random => true, + TestCheckMode::Pass(must_pass) => success_at_step(&results, must_pass, true), + TestCheckMode::Fail(must_fail) => success_at_step(&results, must_fail, false), + TestCheckMode::Busted(must_fail) => success_at_step(&results, must_fail, false), + TestCheckMode::Random(_) => Some(true), }; - if passed { + if passed.unwrap_or(false) { if matches!(results.rules.check, TestCheckMode::Busted(_)) { TestConclusion::Busted } else { @@ -159,18 +143,47 @@ pub fn report_test(results: TestRunResults) -> TestReport { TestConclusion::Failed } }; + + // Compute what the annotation *could* be to make CI green + let did_pass = success_at_step(&results, &results.ran_to, true).unwrap_or(false); + let could_be = TestRulesPattern { + run: if results.rules.run != TestRunMode::Check { + Some(results.rules.run) + } else { + None + }, + check: if did_pass { + Some(TestCheckMode::Pass(results.rules.run)) + } else { + Some(TestCheckMode::Busted(results.rules.run)) + }, + }; TestReport { key: results.key.clone(), - rules: results.rules.clone(), + rules: results.rules, conclusion, + could_be, results, } } +fn success_at_step(results: &TestRunResults, step: &TestRunMode, wants_pass: bool) -> Option { + use TestRunMode::*; + let res = match step { + Skip => return Some(true), + Generate => results.source.as_ref().map(|r| r.is_ok()), + Build => results.build.as_ref().map(|r| r.is_ok()), + Link => results.link.as_ref().map(|r| r.is_ok()), + Run => results.run.as_ref().map(|r| r.is_ok()), + Check => results.check.as_ref().map(|r| r.all_passed), + }; + res.map(|res| res == wants_pass) +} + #[derive(Debug, Serialize)] pub struct FullReport { pub summary: TestSummary, - pub config: TestConfig, + pub possible_rules: Option, pub tests: Vec, } @@ -180,10 +193,9 @@ pub struct TestReport { pub rules: TestRules, pub results: TestRunResults, pub conclusion: TestConclusion, + pub could_be: TestRulesPattern, } -#[derive(Debug, Serialize)] -pub struct TestConfig {} #[derive(Debug, Serialize)] pub struct TestSummary { pub num_tests: u64, @@ -200,6 +212,21 @@ pub struct TestKey { pub callee: ToolchainId, pub options: TestOptions, } + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TestKeyPattern { + pub test: Option, + pub caller: Option, + pub callee: Option, + pub toolchain: Option, + pub options: TestOptionsPattern, +} +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TestOptionsPattern { + pub convention: Option, + pub val_generator: Option, + pub repr: Option, +} impl TestKey { pub(crate) fn toolchain_id(&self, call_side: CallSide) -> &str { match call_side { @@ -209,18 +236,225 @@ impl TestKey { } } -#[derive(Debug, Clone, Serialize)] +impl TestKeyPattern { + fn matches(&self, key: &TestKey) -> bool { + let TestKeyPattern { + test, + caller, + callee, + toolchain, + options: + TestOptionsPattern { + convention, + val_generator, + repr, + }, + } = self; + + if let Some(test) = test { + if test != &key.test { + return false; + } + } + + if let Some(caller) = caller { + if caller != &key.caller { + return false; + } + } + if let Some(callee) = callee { + if callee != &key.callee { + return false; + } + } + if let Some(toolchain) = toolchain { + if toolchain != &key.caller && toolchain != &key.callee { + return false; + } + } + + if let Some(convention) = convention { + if convention != &key.options.convention { + return false; + } + } + if let Some(val_generator) = val_generator { + if val_generator != &key.options.val_generator { + return false; + } + } + if let Some(repr) = repr { + if repr != &key.options.repr { + return false; + } + } + + true + } +} + +impl std::str::FromStr for TestKeyPattern { + type Err = String; + + fn from_str(input: &str) -> Result { + let separator = "::"; + let parts = input.split(separator).collect::>(); + + let mut key = TestKeyPattern { + test: None, + caller: None, + callee: None, + toolchain: None, + options: TestOptionsPattern { + convention: None, + repr: None, + val_generator: None, + }, + }; + + let [test, rest @ ..] = &parts[..] else { + return Ok(key); + }; + key.test = (!test.is_empty()).then(|| test.to_string()); + + for part in rest { + // pairs + if let Some((caller, callee)) = part.split_once("_calls_") { + key.caller = Some(caller.to_owned()); + key.callee = Some(callee.to_owned()); + continue; + } + if let Some(caller) = part.strip_suffix("_caller") { + key.caller = Some(caller.to_owned()); + continue; + } + if let Some(callee) = part.strip_suffix("_callee") { + key.callee = Some(callee.to_owned()); + continue; + } + if let Some(toolchain) = part.strip_suffix("_toolchain") { + key.toolchain = Some(toolchain.to_owned()); + continue; + } + + // repr + if let Some(repr) = part.strip_prefix("repr_") { + key.options.repr = Some(repr.parse()?); + continue; + } + + // conv + if let Some(conv) = part.strip_prefix("conv_") { + key.options.convention = Some(conv.parse()?); + continue; + } + // generator + if let Ok(val_generator) = part.parse() { + key.options.val_generator = Some(val_generator); + continue; + } + + return Err(format!("unknown testkey part: {part}")); + } + Ok(key) + } +} +impl std::fmt::Display for TestKeyPattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let TestKeyPattern { + test, + caller, + callee, + toolchain, + options: + TestOptionsPattern { + convention, + val_generator, + repr, + }, + } = self; + let separator = "::"; + let mut output = String::new(); + if let Some(test) = test { + output.push_str(test); + } + if let Some(convention) = convention { + output.push_str(separator); + output.push_str(&format!("conv_{convention}")); + } + if let Some(repr) = repr { + output.push_str(separator); + output.push_str(&format!("repr_{repr}")); + } + if let Some(toolchain) = toolchain { + output.push_str(separator); + output.push_str(&format!("{toolchain}_toolchain")); + } + match (caller, callee) { + (Some(caller), Some(callee)) => { + output.push_str(separator); + output.push_str(caller); + output.push_str("_calls_"); + output.push_str(callee); + } + (Some(caller), None) => { + output.push_str(separator); + output.push_str(caller); + output.push_str("_caller"); + } + (None, Some(callee)) => { + output.push_str(separator); + output.push_str(callee); + output.push_str("_callee"); + } + (None, None) => { + // Noting + } + } + if let Some(val_generator) = val_generator { + output.push_str(separator); + output.push_str(&val_generator.to_string()); + } + output.fmt(f) + } +} +impl<'de> Deserialize<'de> for TestKeyPattern { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error; + let input = String::deserialize(deserializer)?; + input.parse().map_err(D::Error::custom) + } +} +impl Serialize for TestKeyPattern { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.to_string().serialize(serializer) + } +} + +#[derive(Debug, Clone, Copy, Serialize)] pub struct TestRules { pub run: TestRunMode, + #[serde(flatten)] pub check: TestCheckMode, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestRulesPattern { + pub run: Option, + #[serde(flatten)] + pub check: Option, +} /// How far the test should be executed /// /// Each case implies all the previous cases. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize)] -#[allow(dead_code)] - +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum TestRunMode { /// Don't run the test at all (marked as skipped) Skip, @@ -239,8 +473,8 @@ pub enum TestRunMode { /// To what level of correctness should the test be graded? /// /// Tests that are Skipped ignore this. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize)] -#[allow(dead_code)] +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] pub enum TestCheckMode { /// The test must successfully complete this phase, /// whatever happens after that is gravy. @@ -252,7 +486,7 @@ pub enum TestCheckMode { Busted(TestRunMode), /// The test is flakey and random but we want to run it anyway, /// so accept whatever result we get as ok. - Random, + Random(bool), } #[derive(Debug, Serialize)] @@ -303,10 +537,17 @@ pub struct LinkOutput { pub struct CheckOutput { pub all_passed: bool, pub subtest_names: Vec, - pub subtest_checks: Vec>, + pub subtest_checks: Vec, +} + +#[derive(Debug, Serialize)] +pub struct SubtestDetails { + pub result: Result<(), CheckFailure>, + pub minimized: Option, } #[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] pub enum TestConclusion { Skipped, Passed, @@ -342,7 +583,7 @@ impl FullReport { } (Passed, Pass(_)) => write!(f, "passed")?, - (Passed, Random) => write!(f, "passed (random, result ignored)")?, + (Passed, Random(_)) => write!(f, "passed (random, result ignored)")?, (Passed, Fail(_)) => write!(f, "passed (failed as expected)")?, (Failed, Pass(_)) => { @@ -366,7 +607,7 @@ impl FullReport { writeln!(f, " {}", red.apply_to(err))?; } } - (Failed, Random) => { + (Failed, Random(_)) => { write!(f, "{}", red.apply_to("failed!? (failed but random!?)"))? } (Failed, Fail(_)) => { @@ -383,7 +624,8 @@ impl FullReport { } } - let be_detailed = test.results.ran_to >= TestRunMode::Check; + let be_detailed = test.results.ran_to >= TestRunMode::Check + && test.conclusion != TestConclusion::Busted; if !be_detailed { writeln!(f)?; continue; @@ -392,7 +634,7 @@ impl FullReport { continue; }; let sub_results = &check_result.subtest_checks; - let num_passed = sub_results.iter().filter(|r| r.is_ok()).count(); + let num_passed = sub_results.iter().filter(|t| t.result.is_ok()).count(); writeln!(f, " ({num_passed:>3}/{:<3} passed)", sub_results.len())?; // If all the subtests pass, don't bother with a breakdown. @@ -404,11 +646,16 @@ impl FullReport { .subtest_names .iter() .fold(0, |max, name| max.max(name.len())); - for (subtest_name, result) in check_result.subtest_names.iter().zip(sub_results.iter()) + for (subtest_name, subtest) in check_result.subtest_names.iter().zip(sub_results.iter()) { write!(f, " {:width$} ", subtest_name, width = max_name_len)?; - if let Err(e) = result { + if let Err(e) = &subtest.result { writeln!(f, "{}", red.apply_to("failed!"))?; + if let Some(minimized) = &subtest.minimized { + writeln!(f, " {}", blue.apply_to("minimized to:"))?; + writeln!(f, " caller: {}", blue.apply_to(&minimized.caller_src))?; + writeln!(f, " callee: {}", blue.apply_to(&minimized.callee_src))?; + } writeln!(f, "{}", red.apply_to(e))?; } else { writeln!(f)?; @@ -420,7 +667,7 @@ impl FullReport { let summary_style = if self.summary.num_failed > 0 { red } else if self.summary.num_busted > 0 { - blue + blue.clone() } else { green }; @@ -433,6 +680,16 @@ impl FullReport { self.summary.num_skipped ); writeln!(f, "{}", summary_style.apply_to(summary),)?; + if let Some(rules) = &self.possible_rules { + writeln!(f)?; + writeln!( + f, + "{}", + blue.apply_to("(experimental) adding this to your abi-cafe-rules.toml might help:") + )?; + let toml = toml::to_string_pretty(rules).expect("failed to serialize possible rules!?"); + writeln!(f, "{}", toml)?; + } Ok(()) } diff --git a/src/harness/run.rs b/src/harness/run.rs index cf7df25..e2e31e2 100644 --- a/src/harness/run.rs +++ b/src/harness/run.rs @@ -6,7 +6,7 @@ use serde::Serialize; use tracing::info; use crate::error::*; -use crate::report::*; +use crate::harness::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/test.rs b/src/harness/test.rs index f13642a..90884b1 100644 --- a/src/harness/test.rs +++ b/src/harness/test.rs @@ -114,19 +114,19 @@ impl FunctionSelector { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum FunctionSelector { All, One { idx: FuncIdx, args: ArgSelector }, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum ArgSelector { All, One { idx: usize, vals: ValSelector }, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum ValSelector { All, One { idx: usize }, @@ -202,7 +202,7 @@ impl std::ops::Deref for TestImpl { } } -#[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] pub enum WriteImpl { HarnessCallback, Assert, @@ -275,7 +275,9 @@ impl TestWithToolchain { } } -#[derive(Copy, Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[derive( + Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize, serde::Serialize, +)] #[serde(rename = "lowercase")] pub enum CallingConvention { /// The platform's default C convention (cdecl?) diff --git a/src/main.rs b/src/main.rs index e0e7849..46634eb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,17 +6,16 @@ mod harness; mod log; mod toolchains; -mod report; - use error::*; use files::Paths; +use harness::report::*; use harness::test::*; use harness::vals::*; use harness::*; +use indexmap::IndexMap; use toolchains::*; use kdl_script::parse::LangRepr; -use report::*; use std::error::Error; use std::process::Command; use std::sync::Arc; @@ -76,6 +75,7 @@ pub struct Config { pub minimizing_write_impl: WriteImpl, pub rustc_codegen_backends: Vec<(String, String)>, pub disable_builtin_tests: bool, + pub disable_builtin_rules: bool, pub paths: Paths, } @@ -92,6 +92,7 @@ fn main() -> Result<(), Box> { let _handle = rt.enter(); // Grab all the tests + let test_rules = harness::find_test_rules(&cfg)?; let test_sources = harness::find_tests(&cfg)?; let read_tasks = test_sources .into_iter() @@ -119,12 +120,10 @@ fn main() -> Result<(), Box> { } debug!("loaded tests!"); - let harness = Arc::new(TestHarness::new(tests, &cfg)); + let harness = Arc::new(TestHarness::new(test_rules, tests, &cfg)); debug!("initialized test harness!"); // Run the tests - use TestConclusion::*; - let mut tasks = vec![]; // The cruel bastard that is combinatorics... THE GOD LOOPS @@ -161,11 +160,7 @@ fn main() -> Result<(), Box> { }, }; let rules = harness.get_test_rules(&test_key); - let task = harness.clone().spawn_test( - &rt, - rules.clone(), - test_key.clone(), - ); + let task = harness.clone().spawn_test(&rt, rules, test_key.clone()); tasks.push(task); } @@ -185,6 +180,34 @@ fn main() -> Result<(), Box> { .collect::>(); // Compute the final report + let mut full_report = compute_final_report(&cfg, &harness, reports); + + if full_report.failed() { + generate_minimized_failures(&cfg, &harness, &rt, &mut full_report); + } + + let mut output = std::io::stdout(); + match cfg.output_format { + OutputFormat::Human => full_report.print_human(&harness, &mut output)?, + OutputFormat::Json => full_report.print_json(&harness, &mut output)?, + OutputFormat::RustcJson => full_report.print_rustc_json(&harness, &mut output)?, + } + + if full_report.failed() { + Err(TestsFailed {})?; + } + Ok(()) +} + +fn compute_final_report( + _cfg: &Config, + harness: &Arc, + reports: Vec, +) -> FullReport { + use TestConclusion::*; + + let mut expects = IndexMap::::new(); + let mut num_tests = 0; let mut num_passed = 0; let mut num_busted = 0; @@ -196,11 +219,25 @@ fn main() -> Result<(), Box> { Busted => num_busted += 1, Skipped => num_skipped += 1, Passed => num_passed += 1, - Failed => num_failed += 1, + Failed => { + num_failed += 1; + let pattern = harness.base_id(&report.key, None, "::"); + if let Ok(pattern) = pattern.parse() { + expects.insert(pattern, report.could_be.clone()); + } + } } } - let full_report = FullReport { + let possible_rules = if expects.is_empty() { + None + } else { + Some(ExpectFile { + targets: IndexMap::from_iter([(built_info::TARGET.to_owned(), expects)]), + }) + }; + + FullReport { summary: TestSummary { num_tests, num_passed, @@ -208,78 +245,70 @@ fn main() -> Result<(), Box> { num_failed, num_skipped, }, - // FIXME: put in a bunch of metadata here? - config: TestConfig {}, + possible_rules, tests: reports, - }; - - let mut output = std::io::stdout(); - match cfg.output_format { - OutputFormat::Human => full_report.print_human(&harness, &mut output)?, - OutputFormat::Json => full_report.print_json(&harness, &mut output)?, - OutputFormat::RustcJson => full_report.print_rustc_json(&harness, &mut output)?, - } - - if full_report.failed() { - generate_minimized_failures(&cfg, &harness, &rt, &full_report); - Err(TestsFailed {})?; } - Ok(()) } fn generate_minimized_failures( cfg: &Config, harness: &Arc, rt: &tokio::runtime::Runtime, - reports: &FullReport, + reports: &mut FullReport, ) { - info!("rerunning failures"); - let tasks = reports.tests.iter().flat_map(|report| { + info!("minimizing failures..."); + let mut tasks = vec![]; + for (test_idx, report) in reports.tests.iter().enumerate() { let Some(check) = report.results.check.as_ref() else { - return vec![]; + continue; }; - check - .subtest_checks - .iter() - .filter_map(|func_result| { - let Err(failure) = func_result else { - return None; - }; - let functions = match *failure { - CheckFailure::ValMismatch { - func_idx, - arg_idx, - val_idx, - .. - } - | CheckFailure::TagMismatch { - func_idx, - arg_idx, - val_idx, - .. - } => FunctionSelector::One { - idx: func_idx, - args: ArgSelector::One { - idx: arg_idx, - vals: ValSelector::One { idx: val_idx }, - }, + // FIXME: certainly classes of run failure could also be minimized, + // because we have information indicating there was an error in a specific func! + for (subtest_idx, subtest) in check.subtest_checks.iter().enumerate() { + let Err(failure) = &subtest.result else { + continue; + }; + + let functions = match *failure { + CheckFailure::ValMismatch { + func_idx, + arg_idx, + val_idx, + .. + } + | CheckFailure::TagMismatch { + func_idx, + arg_idx, + val_idx, + .. + } => FunctionSelector::One { + idx: func_idx, + args: ArgSelector::One { + idx: arg_idx, + vals: ValSelector::One { idx: val_idx }, }, - }; + }, + }; - let mut test_key = report.key.clone(); - test_key.options.functions = functions; - test_key.options.val_writer = cfg.minimizing_write_impl; - let mut rules = report.rules.clone(); - rules.run = TestRunMode::Generate; + let mut test_key = report.key.clone(); + test_key.options.functions = functions; + test_key.options.val_writer = cfg.minimizing_write_impl; + let mut rules = report.rules; + rules.run = TestRunMode::Generate; - let task = harness.clone().spawn_test(rt, rules, test_key); - Some(task) - }) - .collect() - }); + let task = harness.clone().spawn_test(rt, rules, test_key); + tasks.push((test_idx, subtest_idx, task)); + } + } - let _results = tasks - .into_iter() - .map(|task| rt.block_on(task).expect("failed to join task")) - .collect::>(); + for (test_idx, subtest_idx, task) in tasks { + let results = rt.block_on(task).expect("failed to join task"); + reports.tests[test_idx] + .results + .check + .as_mut() + .unwrap() + .subtest_checks[subtest_idx] + .minimized = results.source.and_then(|r| r.ok()); + } } diff --git a/src/toolchains/mod.rs b/src/toolchains/mod.rs index 69cd852..01c7115 100644 --- a/src/toolchains/mod.rs +++ b/src/toolchains/mod.rs @@ -23,6 +23,7 @@ const C_TOOLCHAINS: &[&str] = &[TOOLCHAIN_CC, TOOLCHAIN_GCC, TOOLCHAIN_CLANG, TO /// A compiler/language toolchain! pub trait Toolchain { + #[allow(dead_code)] fn lang(&self) -> &'static str; fn src_ext(&self) -> &'static str; fn pun_env(&self) -> Arc;