feat: runtime test expectation file

Cargo.toml

@@ -20,6 +20,7 @@ cc.workspace = true
 clap.workspace = true
 console.workspace = true
 kdl.workspace = true
+include_dir.workspace = true
 libloading.workspace = true
 linked-hash-map.workspace = true
 miette.workspace = true
@@ -32,7 +33,8 @@ thiserror.workspace = true
 tokio.workspace = true
 tracing.workspace = true
 tracing-subscriber.workspace = true
-include_dir = "0.7.4"
+toml.workspace = true
+
 
 [build-dependencies]
 built.workspace = true
@@ -60,6 +62,7 @@ camino = { version = "1.1.7", features = ["serde1"] }
 cc = { version = "1.1.0" }
 clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] }
 console = "0.15.8"
+include_dir = "0.7.4"
 kdl = "4.6.0"
 libloading = "0.7.3"
 linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] }
@@ -73,6 +76,7 @@ serde = { version = "1.0.136", features = ["derive"] }
 serde_json = "1.0.83"
 thiserror = "1.0.30"
 tokio = { version = "1.37.0", features = ["full", "tracing"] }
+toml = "0.8.14"
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 # build

abi-cafe-rules-example.toml

@@ -0,0 +1,28 @@
+# Here are some example annotations for test expecations
+
+# this test fails on this platform, with this toolchain pairing
+[targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"]
+fail = "check"
+
+# this test has random results on this platform, whenever rustc is the caller (callee also supported)
+[targets.x86_64-pc-windows-msvc."simple::rustc_caller"]
+random = true
+
+# whenever this test involves cc, only link it, and expect linking to fail
+[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"]
+run = "link"
+fail = "link"
+
+# any repr(c) version of this test fails to run
+[targets.x86_64-unknown-linux-gnu."simple::repr_c"]
+busted = "run"
+
+# for this pairing, with the rust calling convention, only generate the test, and expect it to work
+[targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"]
+run = "generate"
+pass = "generate"
+
+# can match all tests with leading ::
+[targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"]
+run = "generate"
+pass = "generate"

include/harness/abi-cafe-rules.toml

@@ -0,0 +1,19 @@
+# i128 types are fake on windows so this is all random garbage that might
+# not even compile, but that datapoint is a little interesting/useful
+# so let's keep running them and just ignore the result for now.
+#
+# Anyone who cares about this situation more can make the expectations more precise.
+[targets.x86_64-pc-windows-msvc."i128::cc_toolchain"]
+random = true
+[targets.x86_64-pc-windows-msvc."u128::cc_toolchain"]
+random = true
+
+# FIXME: investigate why this is failing to build
+[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"]
+busted = "build"
+[targets.x86_64-pc-windows-msvc."EmptyStructInside::cc_toolchain"]
+busted = "build"
+
+# CI GCC is too old to support _Float16
+[targets.x86_64-unknown-linux-gnu."f16::conv_c"]
+random = true

kdl-script/src/parse.rs

@@ -218,7 +218,7 @@ pub enum Repr {
     Transparent,
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, serde::Serialize)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize)]
 pub enum LangRepr {
     Rust,
     C,

src/cli.rs

@@ -130,12 +130,27 @@ struct Cli {
     #[clap(long)]
     add_tests: Option<Utf8PathBuf>,
 
+    /// Add the test expectations at the given path
+    ///
+    /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir)
+    ///
+    /// Note that there are already builtin rules (disabled with `--disable-builtin-riles`),
+    /// and it would be nice for rules to be upstreamed so everyone can benefit!
+    #[clap(long)]
+    rules: Option<Utf8PathBuf>,
+
     /// disable the builtin tests
     ///
     /// See also `--add-tests`
     #[clap(long)]
     disable_builtin_tests: bool,
 
+    /// disable the builtin rules
+    ///
+    /// See also `--add-rules`
+    #[clap(long)]
+    disable_builtin_rules: bool,
+
     /// deprecated, does nothing (we always procgen now)
     #[clap(long, hide = true)]
     procgen_tests: bool,
@@ -154,7 +169,9 @@ pub fn make_app() -> Config {
         output_format,
         add_rustc_codegen_backend,
         add_tests,
+        rules,
         disable_builtin_tests,
+        disable_builtin_rules,
         // unimplemented
         select_vals: _,
         key: _,
@@ -228,12 +245,14 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`.
     let out_dir = target_dir.join("temp");
     let generated_src_dir = target_dir.join("generated_impls");
     let runtime_test_input_dir = add_tests;
+    let runtime_rules_file = rules.unwrap_or_else(|| "abi-cafe-rules.toml".into());
 
     let paths = Paths {
         target_dir,
         out_dir,
         generated_src_dir,
         runtime_test_input_dir,
+        runtime_rules_file,
     };
     Config {
         output_format,
@@ -248,6 +267,7 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`.
         run_selections,
         minimizing_write_impl,
         disable_builtin_tests,
+        disable_builtin_rules,
         paths,
     }
 }

src/error.rs

@@ -26,6 +26,8 @@ pub enum GenerateError {
     #[error(transparent)]
     #[diagnostic(transparent)]
     KdlScriptError(#[from] kdl_script::KdlScriptError),
+    #[error(transparent)]
+    TomlError(#[from] toml::de::Error),
     /// Used to signal we just skipped it
     #[error("<skipped>")]
     Skipped,

src/files.rs

@@ -13,6 +13,7 @@ pub struct Paths {
     pub out_dir: Utf8PathBuf,
     pub generated_src_dir: Utf8PathBuf,
     pub runtime_test_input_dir: Option<Utf8PathBuf>,
+    pub runtime_rules_file: Utf8PathBuf,
 }
 impl Paths {
     pub fn harness_dylib_main_file(&self) -> Utf8PathBuf {
@@ -89,12 +90,16 @@ pub fn load_file(file: &File) -> String {
     clean_newlines(string)
 }
 
-pub fn tests() -> &'static Dir<'static> {
+pub fn static_tests() -> &'static Dir<'static> {
     INCLUDES
         .get_dir("tests")
         .expect("includes didn't contain ./test")
 }
 
+pub fn static_rules() -> String {
+    get_file("harness/abi-cafe-rules.toml")
+}
+
 fn clean_newlines(input: &str) -> String {
     input.replace('\r', "")
 }

src/harness/build.rs

@@ -5,8 +5,8 @@ use camino::Utf8Path;
 use tracing::info;
 
 use crate::error::*;
+use crate::harness::report::*;
 use crate::harness::test::*;
-use crate::report::*;
 use crate::*;
 
 impl TestHarness {

src/harness/check.rs

@@ -5,7 +5,6 @@ use kdl_script::types::Ty;
 use tracing::{error, info};
 
 use crate::error::*;
-use crate::report::*;
 use crate::*;
 
 impl TestHarness {

src/harness/mod.rs

@@ -17,11 +17,12 @@ mod build;
 mod check;
 mod generate;
 mod read;
+pub mod report;
 mod run;
 pub mod test;
 pub mod vals;
 
-pub use read::{find_tests, spawn_read_test};
+pub use read::{find_test_rules, find_tests, spawn_read_test};
 pub use run::TestBuffer;
 
 pub type Memoized<K, V> = Mutex<SortedMap<K, Arc<OnceCell<V>>>>;
@@ -30,6 +31,7 @@ pub struct TestHarness {
     paths: Paths,
     toolchains: Toolchains,
     tests: SortedMap<TestId, Arc<Test>>,
+    test_rules: Vec<ExpectFile>,
     tests_with_vals: Memoized<(TestId, ValueGeneratorKind), Arc<TestWithVals>>,
     tests_with_toolchain:
         Memoized<(TestId, ValueGeneratorKind, ToolchainId), Arc<TestWithToolchain>>,
@@ -39,11 +41,16 @@ pub struct TestHarness {
 }
 
 impl TestHarness {
-    pub fn new(tests: SortedMap<TestId, Arc<Test>>, cfg: &Config) -> Self {
+    pub fn new(
+        test_rules: Vec<ExpectFile>,
+        tests: SortedMap<TestId, Arc<Test>>,
+        cfg: &Config,
+    ) -> Self {
         let toolchains = toolchains::create_toolchains(cfg);
         Self {
             paths: cfg.paths.clone(),
             tests,
+            test_rules,
             toolchains,
             tests_with_vals: Default::default(),
             tests_with_toolchain: Default::default(),
@@ -109,13 +116,6 @@ impl TestHarness {
             .clone();
         Ok(output)
     }
-    pub fn get_test_rules(&self, test_key: &TestKey) -> TestRules {
-        let caller = self.toolchains[&test_key.caller].clone();
-        let callee = self.toolchains[&test_key.callee].clone();
-
-        get_test_rules(test_key, &*caller, &*callee)
-    }
-
     pub fn spawn_test(
         self: Arc<Self>,
         rt: &tokio::runtime::Runtime,
@@ -262,26 +262,18 @@ impl TestHarness {
             WriteImpl::HarnessCallback => {
                 // Do nothing, implicit default
             }
-            WriteImpl::Print => {
-                output.push_str(separator);
-                output.push_str("print");
-            }
-            WriteImpl::Assert => {
-                output.push_str(separator);
-                output.push_str("assert");
-            }
-            WriteImpl::Noop => {
+            other => {
                 output.push_str(separator);
-                output.push_str("noop");
+                output.push_str(&other.to_string())
             }
         }
         match val_generator {
             ValueGeneratorKind::Graffiti => {
                 // Do nothing, implicit default
             }
-            ValueGeneratorKind::Random { seed } => {
+            other => {
                 output.push_str(separator);
-                output.push_str(&format!("random{seed}"));
+                output.push_str(&other.to_string())
             }
         }
         output