Cleanup/add some testing in wasmtime-fuzzing (#12038)

Share the fuzz-generator-helpers across more tests and add smoke tests
for the dynamic/static component API tests too.

Author: alexcrichton

crates/fuzzing/src/generators/component_types.rs

@@ -187,3 +187,25 @@ where
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test::test_n_times;
+    use wasmtime_test_util::component_fuzz::{TestCase, Type};
+
+    #[test]
+    fn static_api_smoke_test() {
+        test_n_times(10, |(), u| {
+            let case = TestCase {
+                params: vec![&Type::S32, &Type::Bool, &Type::String],
+                result: Some(&Type::String),
+                encoding1: u.arbitrary()?,
+                encoding2: u.arbitrary()?,
+            };
+
+            let declarations = case.declarations();
+            static_api_test::<(i32, bool, String), (String,)>(u, &declarations)
+        });
+    }
+}

crates/fuzzing/src/generators/stacks.rs

@@ -404,21 +404,15 @@ impl Stacks {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use rand::prelude::*;
     use wasmparser::Validator;
 
     #[test]
     fn stacks_generates_valid_wasm_modules() {
-        let mut rng = SmallRng::seed_from_u64(0);
-        let mut buf = vec![0; 2048];
-        for _ in 0..1024 {
-            rng.fill_bytes(&mut buf);
-            let u = Unstructured::new(&buf);
-            if let Ok(stacks) = Stacks::arbitrary_take_rest(u) {
-                let wasm = stacks.wasm();
-                validate(&wasm);
-            }
-        }
+        crate::test::test_n_times(10, |stacks: Stacks, _u| {
+            let wasm = stacks.wasm();
+            validate(&wasm);
+            Ok(())
+        })
     }
 
     fn validate(wasm: &[u8]) {

crates/fuzzing/src/lib.rs

@@ -29,3 +29,44 @@ pub fn init_fuzzing() {
         let _ = env_logger::try_init();
     });
 }
+
+#[cfg(test)]
+mod test {
+    use arbitrary::{Arbitrary, Unstructured};
+    use rand::prelude::*;
+
+    pub fn gen_until_pass<T: for<'a> Arbitrary<'a>>(
+        mut f: impl FnMut(T, &mut Unstructured<'_>) -> anyhow::Result<bool>,
+    ) -> bool {
+        let mut rng = SmallRng::seed_from_u64(0);
+        let mut buf = vec![0; 2048];
+        let n = 3000;
+        for _ in 0..n {
+            rng.fill_bytes(&mut buf);
+            let mut u = Unstructured::new(&buf);
+
+            if let Ok(config) = u.arbitrary() {
+                if f(config, &mut u).unwrap() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Runs `f` with random data until it returns `Ok(())` `iters` times.
+    pub fn test_n_times<T: for<'a> Arbitrary<'a>>(
+        iters: u32,
+        mut f: impl FnMut(T, &mut Unstructured<'_>) -> arbitrary::Result<()>,
+    ) {
+        let mut to_test = 0..iters;
+        let ok = gen_until_pass(|a, b| {
+            if f(a, b).is_ok() {
+                Ok(to_test.next().is_none())
+            } else {
+                Ok(false)
+            }
+        });
+        assert!(ok);
+    }
+}

crates/fuzzing/src/oracles.rs

@@ -1369,45 +1369,9 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arbitrary::Unstructured;
-    use rand::prelude::*;
+    use crate::test::{gen_until_pass, test_n_times};
     use wasmparser::{Validator, WasmFeatures};
 
-    fn gen_until_pass<T: for<'a> Arbitrary<'a>>(
-        mut f: impl FnMut(T, &mut Unstructured<'_>) -> Result<bool>,
-    ) -> bool {
-        let mut rng = SmallRng::seed_from_u64(0);
-        let mut buf = vec![0; 2048];
-        let n = 3000;
-        for _ in 0..n {
-            rng.fill_bytes(&mut buf);
-            let mut u = Unstructured::new(&buf);
-
-            if let Ok(config) = u.arbitrary() {
-                if f(config, &mut u).unwrap() {
-                    return true;
-                }
-            }
-        }
-        false
-    }
-
-    /// Runs `f` with random data until it returns `Ok(())` `iters` times.
-    fn test_n_times<T: for<'a> Arbitrary<'a>>(
-        iters: u32,
-        mut f: impl FnMut(T, &mut Unstructured<'_>) -> arbitrary::Result<()>,
-    ) {
-        let mut to_test = 0..iters;
-        let ok = gen_until_pass(|a, b| {
-            if f(a, b).is_ok() {
-                Ok(to_test.next().is_none())
-            } else {
-                Ok(false)
-            }
-        });
-        assert!(ok);
-    }
-
     // Test that the `gc_ops` fuzzer eventually runs the gc function in the host.
     // We've historically had issues where this fuzzer accidentally wasn't fuzzing
     // anything for a long time so this is an attempt to prevent that from happening
@@ -1501,4 +1465,9 @@ mod tests {
     fn wast_smoke_test() {
         test_n_times(50, |(), u| super::wast_test(u));
     }
+
+    #[test]
+    fn dynamic_component_api_smoke_test() {
+        test_n_times(50, |(), u| super::dynamic_component_api_target(u));
+    }
 }

crates/fuzzing/src/oracles/engine.rs

@@ -181,27 +181,16 @@ pub fn smoke_test_engine<T>(
 ) where
     T: DiffEngine,
 {
-    use rand::prelude::*;
-
-    let mut rng = SmallRng::seed_from_u64(0);
-    let mut buf = vec![0; 2048];
-    let n = 100;
-    for _ in 0..n {
-        rng.fill_bytes(&mut buf);
-        let mut u = Unstructured::new(&buf);
-        let mut config = match u.arbitrary::<Config>() {
-            Ok(config) => config,
-            Err(_) => continue,
-        };
+    crate::test::test_n_times(5, |mut config: Config, u| {
         // This will ensure that wasmtime, which uses this configuration
         // settings, can guaranteed instantiate a module.
         config.set_differential_config();
 
-        let mut engine = match mk_engine(&mut u, &mut config) {
+        let mut engine = match mk_engine(u, &mut config) {
             Ok(engine) => engine,
             Err(e) => {
                 println!("skip {e:?}");
-                continue;
+                return Ok(());
             }
         };
 
@@ -240,8 +229,6 @@ pub fn smoke_test_engine<T>(
             }
         }
 
-        return;
-    }
-
-    panic!("after {n} runs nothing ever ran, something is probably wrong");
+        Ok(())
+    })
 }

crates/fuzzing/src/oracles/memory.rs

@@ -392,20 +392,13 @@ fn build_wasm(image: &HeapImage, offset: u32) -> Vec<u8> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arbitrary::{Arbitrary, Unstructured};
-    use rand::prelude::*;
+    use crate::test::test_n_times;
 
     #[test]
     fn smoke_test_memory_access() {
-        let mut rng = SmallRng::seed_from_u64(0);
-        let mut buf = vec![0; 1024];
-
-        for _ in 0..1024 {
-            rng.fill_bytes(&mut buf);
-            let u = Unstructured::new(&buf);
-            if let Ok(input) = MemoryAccesses::arbitrary_take_rest(u) {
-                check_memory_accesses(input);
-            }
-        }
+        test_n_times(50, |input: MemoryAccesses, _u| {
+            check_memory_accesses(input);
+            Ok(())
+        })
     }
 }

crates/fuzzing/src/oracles/stacks.rs

@@ -138,30 +138,15 @@ fn assert_stack_matches(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use arbitrary::{Arbitrary, Unstructured};
-    use rand::prelude::*;
+    use crate::test::gen_until_pass;
 
     const TARGET_STACK_DEPTH: usize = 10;
 
     #[test]
     fn smoke_test() {
-        let mut rng = SmallRng::seed_from_u64(0);
-        let mut buf = vec![0; 2048];
-
-        for _ in 0..1024 {
-            rng.fill_bytes(&mut buf);
-            let u = Unstructured::new(&buf);
-            if let Ok(stacks) = Stacks::arbitrary_take_rest(u) {
-                let max_stack_depth = check_stacks(stacks);
-                if max_stack_depth >= TARGET_STACK_DEPTH {
-                    return;
-                }
-            }
-        }
-
-        panic!(
-            "never generated a `Stacks` test case that reached {TARGET_STACK_DEPTH} \
-             deep stack frames",
-        );
+        gen_until_pass(|stacks: Stacks, _u| {
+            let max_stack_depth = check_stacks(stacks);
+            Ok(max_stack_depth >= TARGET_STACK_DEPTH)
+        });
     }
 }