diff --git a/Cargo.lock b/Cargo.lock index c156f405b4fb..d427b85b6515 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4989,16 +4989,22 @@ version = "40.0.0" dependencies = [ "anyhow", "arbitrary", + "arbtest", "env_logger 0.11.5", + "indexmap 2.11.4", "proc-macro2", "quote", + "rand 0.9.2", "serde", "serde_derive", "target-lexicon", "toml", + "wasmparser 0.241.2", + "wasmprinter", "wasmtime", "wasmtime-environ", "wasmtime-internal-component-util", + "wat", ] [[package]] diff --git a/crates/environ/fuzz/fuzz_targets/fact-valid-module.rs b/crates/environ/fuzz/fuzz_targets/fact-valid-module.rs index fdfee86021e1..c08eabbe912f 100644 --- a/crates/environ/fuzz/fuzz_targets/fact-valid-module.rs +++ b/crates/environ/fuzz/fuzz_targets/fact-valid-module.rs @@ -15,7 +15,6 @@ use wasmtime_environ::{ScopeVec, Tunables, component::*}; use wasmtime_test_util::component_fuzz::{MAX_TYPE_DEPTH, TestCase, Type}; const TYPE_COUNT: usize = 50; -const MAX_ARITY: u32 = 5; #[derive(Debug)] struct GenAdapter<'a> { @@ -48,21 +47,7 @@ fn target(data: &[u8]) -> arbitrary::Result<()> { } // Next generate a static API test case driven by the above types. - let mut params = Vec::new(); - let mut result = None; - for _ in 0..u.int_in_range(0..=MAX_ARITY)? { - params.push(u.choose(&types)?); - } - if u.arbitrary()? { - result = Some(u.choose(&types)?); - } - - let test = TestCase { - params, - result, - encoding1: u.arbitrary()?, - encoding2: u.arbitrary()?, - }; + let test = TestCase::generate(&types, &mut u)?; let adapter = GenAdapter { test }; let wat_decls = adapter.test.declarations(); diff --git a/crates/fuzzing/src/generators/component_types.rs b/crates/fuzzing/src/generators/component_types.rs index 5b1def366ebd..3c9de24e7521 100644 --- a/crates/fuzzing/src/generators/component_types.rs +++ b/crates/fuzzing/src/generators/component_types.rs @@ -6,12 +6,13 @@ //! parameters to the imported one and forwards the result back to the caller. This serves to exercise Wasmtime's //! lifting and lowering code and verify the values remain intact during both processes. +use crate::block_on; use arbitrary::{Arbitrary, Unstructured}; use std::any::Any; use std::fmt::Debug; use std::ops::ControlFlow; use wasmtime::component::{self, Component, ComponentNamedList, Lift, Linker, Lower, Val}; -use wasmtime::{Config, Engine, Store, StoreContextMut}; +use wasmtime::{AsContextMut, Config, Engine, Store, StoreContextMut}; use wasmtime_test_util::component_fuzz::{Declarations, EXPORT_FUNCTION, IMPORT_FUNCTION}; /// Minimum length of an arbitrary list value generated for a test case @@ -147,6 +148,9 @@ where let mut config = Config::new(); config.wasm_component_model(true); + config.wasm_component_model_async(true); + config.wasm_component_model_async_stackful(true); + config.async_support(true); config.debug_adapter_modules(input.arbitrary()?); let engine = Engine::new(&config).unwrap(); let wat = declarations.make_component(); @@ -154,38 +158,72 @@ where crate::oracles::log_wasm(wat); let component = Component::new(&engine, wat).unwrap(); let mut linker = Linker::new(&engine); - linker - .root() - .func_wrap( - IMPORT_FUNCTION, - |cx: StoreContextMut<'_, Box>, params: P| { - log::trace!("received parameters {params:?}"); - let data: &(P, R) = cx.data().downcast_ref().unwrap(); - let (expected_params, result) = data; - assert_eq!(params, *expected_params); - log::trace!("returning result {result:?}"); - Ok(result.clone()) - }, - ) - .unwrap(); - let mut store: Store> = Store::new(&engine, Box::new(())); - let instance = linker.instantiate(&mut store, &component).unwrap(); - let func = instance - .get_typed_func::(&mut store, EXPORT_FUNCTION) - .unwrap(); - - while input.arbitrary()? { - let params = input.arbitrary::

()?; - let result = input.arbitrary::()?; - *store.data_mut() = Box::new((params.clone(), result.clone())); - log::trace!("passing in parameters {params:?}"); - let actual = func.call(&mut store, params).unwrap(); - log::trace!("got result {actual:?}"); - assert_eq!(actual, result); - func.post_return(&mut store).unwrap(); + + fn host_function( + cx: StoreContextMut<'_, Box>, + params: P, + ) -> anyhow::Result + where + P: Debug + PartialEq + 'static, + R: Debug + Clone + 'static, + { + log::trace!("received parameters {params:?}"); + let data: &(P, R) = cx.data().downcast_ref().unwrap(); + let (expected_params, result) = data; + assert_eq!(params, *expected_params); + log::trace!("returning result {result:?}"); + Ok(result.clone()) + } + + if declarations.options.host_async { + linker + .root() + .func_wrap_concurrent(IMPORT_FUNCTION, |a, params| { + Box::pin(async move { + a.with(|mut cx| host_function::(cx.as_context_mut(), params)) + }) + }) + .unwrap(); + } else { + linker + .root() + .func_wrap(IMPORT_FUNCTION, |cx, params| { + host_function::(cx, params) + }) + .unwrap(); } + let mut store: Store> = Store::new(&engine, Box::new(())); + + block_on(async { + let instance = linker + .instantiate_async(&mut store, &component) + .await + .unwrap(); + let func = instance + .get_typed_func::(&mut store, EXPORT_FUNCTION) + .unwrap(); - Ok(()) + while input.arbitrary()? { + let params = input.arbitrary::

()?; + let result = input.arbitrary::()?; + *store.data_mut() = Box::new((params.clone(), result.clone())); + log::trace!("passing in parameters {params:?}"); + let actual = if declarations.options.guest_caller_async { + store + .run_concurrent(async |a| func.call_concurrent(a, params).await.unwrap().0) + .await + .unwrap() + } else { + let result = func.call_async(&mut store, params).await.unwrap(); + func.post_return_async(&mut store).await.unwrap(); + result + }; + log::trace!("got result {actual:?}"); + assert_eq!(actual, result); + } + + Ok(()) + }) } #[cfg(test)] @@ -197,12 +235,9 @@ mod tests { #[test] fn static_api_smoke_test() { test_n_times(10, |(), u| { - let case = TestCase { - params: vec![&Type::S32, &Type::Bool, &Type::String], - result: Some(&Type::String), - encoding1: u.arbitrary()?, - encoding2: u.arbitrary()?, - }; + let mut case = TestCase::generate(&[], u)?; + case.params = vec![&Type::S32, &Type::Bool, &Type::String]; + case.result = Some(&Type::String); let declarations = case.declarations(); static_api_test::<(i32, bool, String), (String,)>(u, &declarations) diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs index 0a2134e6f4a2..cdcc1bdf37bd 100644 --- a/crates/fuzzing/src/lib.rs +++ b/crates/fuzzing/src/lib.rs @@ -2,6 +2,8 @@ #![deny(missing_docs)] +use std::task::{Context, Poll, Waker}; + pub use wasm_mutate; pub use wasm_smith; pub mod generators; @@ -30,6 +32,17 @@ pub fn init_fuzzing() { }); } +fn block_on(future: F) -> F::Output { + let mut f = Box::pin(future); + let mut cx = Context::from_waker(Waker::noop()); + loop { + match f.as_mut().poll(&mut cx) { + Poll::Ready(val) => break val, + Poll::Pending => {} + } + } +} + #[cfg(test)] mod test { use arbitrary::{Arbitrary, Unstructured}; diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index 090db067b30b..de70fcf9afcd 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -21,6 +21,7 @@ mod stacks; use self::diff_wasmtime::WasmtimeInstance; use self::engine::{DiffEngine, DiffInstance}; +use crate::block_on; use crate::generators::GcOps; use crate::generators::{self, CompilerStrategy, DiffValue, DiffValueType}; use crate::single_module_fuzzer::KnownValid; @@ -30,8 +31,9 @@ use std::future::Future; use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}; use std::sync::{Arc, Condvar, Mutex}; -use std::task::{Context, Poll, Waker}; +use std::task::{Context, Poll}; use std::time::{Duration, Instant}; +use wasmtime::component::Accessor; use wasmtime::*; use wasmtime_wast::WastContext; @@ -59,7 +61,10 @@ pub fn log_wasm(wasm: &[u8]) { // If wasmprinter failed remove a `*.wat` file, if any, to avoid // confusing a preexisting one with this wasm which failed to get // printed. - Err(_) => drop(std::fs::remove_file(&wat)), + Err(e) => { + log::debug!("failed to print to wat: {e}"); + drop(std::fs::remove_file(&wat)) + } } } @@ -1077,7 +1082,6 @@ impl Drop for HelperThread { pub fn dynamic_component_api_target(input: &mut arbitrary::Unstructured) -> arbitrary::Result<()> { use crate::generators::component_types; use wasmtime::component::{Component, Linker, Val}; - use wasmtime_test_util::component::FuncExt; use wasmtime_test_util::component_fuzz::{ EXPORT_FUNCTION, IMPORT_FUNCTION, MAX_TYPE_DEPTH, TestCase, Type, }; @@ -1090,23 +1094,13 @@ pub fn dynamic_component_api_target(input: &mut arbitrary::Unstructured) -> arbi for _ in 0..5 { types.push(Type::generate(input, MAX_TYPE_DEPTH, &mut type_fuel)?); } - let params = (0..input.int_in_range(0..=5)?) - .map(|_| input.choose(&types)) - .collect::>>()?; - let result = if input.arbitrary()? { - Some(input.choose(&types)?) - } else { - None - }; - let case = TestCase { - params, - result, - encoding1: input.arbitrary()?, - encoding2: input.arbitrary()?, - }; + let case = TestCase::generate(&types, input)?; let mut config = wasmtime_test_util::component::config(); + config.async_support(true); + config.wasm_component_model_async(true); + config.wasm_component_model_async_stackful(true); config.debug_adapter_modules(input.arbitrary()?); let engine = Engine::new(&config).unwrap(); let mut store = Store::new(&engine, (Vec::new(), None)); @@ -1116,52 +1110,82 @@ pub fn dynamic_component_api_target(input: &mut arbitrary::Unstructured) -> arbi let component = Component::new(&engine, wat).unwrap(); let mut linker = Linker::new(&engine); - linker - .root() - .func_new(IMPORT_FUNCTION, { - move |mut cx: StoreContextMut<'_, (Vec, Option>)>, - _, - params: &[Val], - results: &mut [Val]| - -> Result<()> { - log::trace!("received params {params:?}"); - let (expected_args, expected_results) = cx.data_mut(); - assert_eq!(params.len(), expected_args.len()); - for (expected, actual) in expected_args.iter().zip(params) { - assert_eq!(expected, actual); - } - results.clone_from_slice(&expected_results.take().unwrap()); - log::trace!("returning results {results:?}"); - Ok(()) - } - }) - .unwrap(); - - let instance = linker.instantiate(&mut store, &component).unwrap(); - let func = instance.get_func(&mut store, EXPORT_FUNCTION).unwrap(); - let ty = func.ty(&store); - - while input.arbitrary()? { - let params = ty - .params() - .map(|(_, ty)| component_types::arbitrary_val(&ty, input)) - .collect::>>()?; - let results = ty - .results() - .map(|ty| component_types::arbitrary_val(&ty, input)) - .collect::>>()?; - - *store.data_mut() = (params.clone(), Some(results.clone())); + fn host_function( + mut cx: StoreContextMut<'_, (Vec, Option>)>, + params: &[Val], + results: &mut [Val], + ) -> Result<()> { + log::trace!("received params {params:?}"); + let (expected_args, expected_results) = cx.data_mut(); + assert_eq!(params.len(), expected_args.len()); + for (expected, actual) in expected_args.iter().zip(params) { + assert_eq!(expected, actual); + } + results.clone_from_slice(&expected_results.take().unwrap()); + log::trace!("returning results {results:?}"); + Ok(()) + } - log::trace!("passing params {params:?}"); - let mut actual = vec![Val::Bool(false); results.len()]; - func.call_and_post_return(&mut store, ¶ms, &mut actual) + if case.options.host_async { + linker + .root() + .func_new_concurrent(IMPORT_FUNCTION, { + move |cx: &Accessor<_, _>, _, params: &[Val], results: &mut [Val]| { + Box::pin(async move { + cx.with(|mut store| host_function(store.as_context_mut(), params, results)) + }) + } + }) + .unwrap(); + } else { + linker + .root() + .func_new(IMPORT_FUNCTION, { + move |cx, _, params, results| host_function(cx, params, results) + }) .unwrap(); - log::trace!("received results {actual:?}"); - assert_eq!(actual, results); } - Ok(()) + block_on(async { + let instance = linker + .instantiate_async(&mut store, &component) + .await + .unwrap(); + let func = instance.get_func(&mut store, EXPORT_FUNCTION).unwrap(); + let ty = func.ty(&store); + + while input.arbitrary()? { + let params = ty + .params() + .map(|(_, ty)| component_types::arbitrary_val(&ty, input)) + .collect::>>()?; + let results = ty + .results() + .map(|ty| component_types::arbitrary_val(&ty, input)) + .collect::>>()?; + + *store.data_mut() = (params.clone(), Some(results.clone())); + + log::trace!("passing params {params:?}"); + let mut actual = vec![Val::Bool(false); results.len()]; + if case.options.guest_caller_async { + store + .run_concurrent(async |a| { + func.call_concurrent(a, ¶ms, &mut actual).await.unwrap(); + }) + .await + .unwrap(); + } else { + func.call_async(&mut store, ¶ms, &mut actual) + .await + .unwrap(); + func.post_return_async(&mut store).await.unwrap(); + } + log::trace!("received results {actual:?}"); + assert_eq!(actual, results); + } + Ok(()) + }) } /// Instantiates a wasm module and runs its exports with dummy values, all in @@ -1221,7 +1245,7 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32 // Run the instantiation process, asynchronously, and if everything // succeeds then pull out the instance. // log::info!("starting instantiation"); - let instance = run(Timeout { + let instance = block_on(Timeout { future: Instance::new_async(&mut store, &module, &imports), polls: take_poll_amt(&mut poll_amts), end: Instant::now() + Duration::from_millis(2_000), @@ -1267,7 +1291,7 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32 log::info!("invoking export {name:?}"); let future = func.call_async(&mut store, ¶ms, &mut results); - match run(Timeout { + match block_on(Timeout { future, polls: take_poll_amt(&mut poll_amts), end: Instant::now() + Duration::from_millis(2_000), @@ -1353,17 +1377,6 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32 } } } - - fn run(future: F) -> F::Output { - let mut f = Box::pin(future); - let mut cx = Context::from_waker(Waker::noop()); - loop { - match f.as_mut().poll(&mut cx) { - Poll::Ready(val) => break val, - Poll::Pending => {} - } - } - } } #[cfg(test)] diff --git a/crates/test-util/Cargo.toml b/crates/test-util/Cargo.toml index 6069d7b25781..9350d75591ba 100644 --- a/crates/test-util/Cargo.toml +++ b/crates/test-util/Cargo.toml @@ -25,11 +25,19 @@ wasmtime-environ = { workspace = true, optional = true } target-lexicon = { workspace = true, optional = true } env_logger = { workspace = true, optional = true } wasmtime = { workspace = true, optional = true } +indexmap = { workspace = true, optional = true, features = ['std'] } # NB: this crate is compiled both in as a dependency of a proc-macro and as a # dependency of tests themselves. That means dependencies of this crate are # compiled twice. Try to ensure "big" dependencies are optional and feature # gated. +[dev-dependencies] +rand = { workspace = true } +wat = { workspace = true } +wasmparser = { workspace = true, features = ['validate', 'component-model', 'features'] } +wasmprinter = { workspace = true, features = ['component-model'] } +arbtest = { workspace = true } + [features] wast = [ 'dep:serde', @@ -61,6 +69,8 @@ component = [ ] component-fuzz = [ 'dep:arbitrary', + 'arbitrary/derive', + 'dep:indexmap', 'dep:quote', 'dep:proc-macro2', 'dep:wasmtime-component-util', diff --git a/crates/test-util/src/component_fuzz.rs b/crates/test-util/src/component_fuzz.rs index af8eac8df471..2910bdfd4e12 100644 --- a/crates/test-util/src/component_fuzz.rs +++ b/crates/test-util/src/component_fuzz.rs @@ -7,15 +7,18 @@ //! lifting and lowering code and verify the values remain intact during both processes. use arbitrary::{Arbitrary, Unstructured}; +use indexmap::IndexSet; use proc_macro2::{Ident, TokenStream}; use quote::{ToTokens, format_ident, quote}; use std::borrow::Cow; use std::fmt::{self, Debug, Write}; +use std::hash::{Hash, Hasher}; use std::iter; use std::ops::Deref; use wasmtime_component_util::{DiscriminantSize, FlagsSize, REALLOC_AND_FREE}; const MAX_FLAT_PARAMS: usize = 16; +const MAX_FLAT_ASYNC_PARAMS: usize = 4; const MAX_FLAT_RESULTS: usize = 1; /// The name of the imported host function which the generated component will call @@ -27,7 +30,19 @@ pub const EXPORT_FUNCTION: &str = "echo-export"; /// Wasmtime allows up to 100 type depth so limit this to just under that. pub const MAX_TYPE_DEPTH: u32 = 99; -#[derive(Copy, Clone, PartialEq, Eq)] +macro_rules! uwriteln { + ($($arg:tt)*) => { + writeln!($($arg)*).unwrap() + }; +} + +macro_rules! uwrite { + ($($arg:tt)*) => { + write!($($arg)*).unwrap() + }; +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] enum CoreType { I32, I64, @@ -204,6 +219,393 @@ impl Type { ) -> arbitrary::Result> { VecInRange::new(u, fuel, |u, fuel| Type::generate(u, depth, fuel)) } + + /// Generates text format wasm into `s` to store a value of this type, in + /// its flat representation stored in the `locals` provided, to the local + /// named `ptr` at the `offset` provided. + /// + /// This will register helper functions necessary in `helpers`. The + /// `locals` iterator will be advanced for all locals consumed by this + /// store operation. + fn store_flat<'a>( + &'a self, + s: &mut String, + ptr: &str, + offset: u32, + locals: &mut dyn Iterator, + helpers: &mut IndexSet>, + ) { + enum Kind { + Primitive(&'static str), + PointerPair, + Helper, + } + let kind = match self { + Type::Bool | Type::S8 | Type::U8 => Kind::Primitive("i32.store8"), + Type::S16 | Type::U16 => Kind::Primitive("i32.store16"), + Type::S32 | Type::U32 | Type::Char => Kind::Primitive("i32.store"), + Type::S64 | Type::U64 => Kind::Primitive("i64.store"), + Type::Float32 => Kind::Primitive("f32.store"), + Type::Float64 => Kind::Primitive("f64.store"), + Type::String | Type::List(_) => Kind::PointerPair, + Type::Enum(n) if *n <= (1 << 8) => Kind::Primitive("i32.store8"), + Type::Enum(n) if *n <= (1 << 16) => Kind::Primitive("i32.store16"), + Type::Enum(_) => Kind::Primitive("i32.store"), + Type::Flags(n) if *n <= 8 => Kind::Primitive("i32.store8"), + Type::Flags(n) if *n <= 16 => Kind::Primitive("i32.store16"), + Type::Flags(n) if *n <= 32 => Kind::Primitive("i32.store"), + Type::Flags(_) => unreachable!(), + Type::Record(_) + | Type::Tuple(_) + | Type::Variant(_) + | Type::Option(_) + | Type::Result { .. } => Kind::Helper, + }; + + match kind { + Kind::Primitive(op) => uwriteln!( + s, + "({op} offset={offset} (local.get {ptr}) {})", + locals.next().unwrap() + ), + Kind::PointerPair => { + let abi_ptr = locals.next().unwrap(); + let abi_len = locals.next().unwrap(); + uwriteln!(s, "(i32.store offset={offset} (local.get {ptr}) {abi_ptr})",); + let offset = offset + 4; + uwriteln!(s, "(i32.store offset={offset} (local.get {ptr}) {abi_len})",); + } + Kind::Helper => { + let (index, _) = helpers.insert_full(Helper(self)); + uwriteln!(s, "(i32.add (local.get {ptr}) (i32.const {offset}))"); + for _ in 0..self.lowered().len() { + let i = locals.next().unwrap(); + uwriteln!(s, "{i}"); + } + uwriteln!(s, "call $store_helper_{index}"); + } + } + } + + /// Generates a text-format wasm function which takes a pointer and this + /// type's flat representation as arguments and then stores this value in + /// the first argument. + /// + /// This is used to store records/variants to cut down on the size of final + /// functions and make codegen here a bit easier. + fn store_flat_helper<'a>( + &'a self, + s: &mut String, + i: usize, + helpers: &mut IndexSet>, + ) { + uwrite!(s, "(func $store_helper_{i} (param i32)"); + let lowered = self.lowered(); + for ty in &lowered { + uwrite!(s, " (param {ty})"); + } + s.push_str("\n"); + let locals = (0..lowered.len() as u32).map(|i| i + 1).collect::>(); + let record = |s: &mut String, helpers: &mut IndexSet>, types: &'a [Type]| { + let mut locals = locals.iter().cloned().map(FlatSource::Local); + for (offset, ty) in record_field_offsets(types) { + ty.store_flat(s, "0", offset, &mut locals, helpers); + } + assert!(locals.next().is_none()); + }; + let variant = |s: &mut String, + helpers: &mut IndexSet>, + types: &[Option<&'a Type>]| { + let (size, offset) = variant_memory_info(types.iter().cloned()); + // One extra block for out-of-bounds discriminants. + for _ in 0..types.len() + 1 { + s.push_str("block\n"); + } + + // Store the discriminant in memory, then branch on it to figure + // out which case we're in. + let store = match size { + DiscriminantSize::Size1 => "i32.store8", + DiscriminantSize::Size2 => "i32.store16", + DiscriminantSize::Size4 => "i32.store", + }; + uwriteln!(s, "({store} (local.get 0) (local.get 1))"); + s.push_str("local.get 1\n"); + s.push_str("br_table"); + for i in 0..types.len() + 1 { + uwrite!(s, " {i}"); + } + s.push_str("\nend\n"); + + // Store each payload individually while converting locals from + // their source types to the precise type necessary for this + // variant. + for ty in types { + if let Some(ty) = ty { + let ty_lowered = ty.lowered(); + let mut locals = locals[1..].iter().zip(&lowered[1..]).zip(&ty_lowered).map( + |((i, from), to)| FlatSource::LocalConvert { + local: *i, + from: *from, + to: *to, + }, + ); + ty.store_flat(s, "0", offset, &mut locals, helpers); + } + s.push_str("return\n"); + s.push_str("end\n"); + } + + // Catch-all result which is for out-of-bounds discriminants. + s.push_str("unreachable\n"); + }; + match self { + Type::Bool + | Type::S8 + | Type::U8 + | Type::S16 + | Type::U16 + | Type::S32 + | Type::U32 + | Type::Char + | Type::S64 + | Type::U64 + | Type::Float32 + | Type::Float64 + | Type::String + | Type::List(_) + | Type::Flags(_) + | Type::Enum(_) => unreachable!(), + + Type::Record(r) => record(s, helpers, r), + Type::Tuple(t) => record(s, helpers, t), + Type::Variant(v) => variant( + s, + helpers, + &v.iter().map(|t| t.as_ref()).collect::>(), + ), + Type::Option(o) => variant(s, helpers, &[None, Some(&**o)]), + Type::Result { ok, err } => variant(s, helpers, &[ok.as_deref(), err.as_deref()]), + }; + s.push_str(")\n"); + } + + /// Same as `store_flat`, except loads the flat values from `ptr+offset`. + /// + /// Results are placed directly on the wasm stack. + fn load_flat<'a>( + &'a self, + s: &mut String, + ptr: &str, + offset: u32, + helpers: &mut IndexSet>, + ) { + enum Kind { + Primitive(&'static str), + PointerPair, + Helper, + } + let kind = match self { + Type::Bool | Type::U8 => Kind::Primitive("i32.load8_u"), + Type::S8 => Kind::Primitive("i32.load8_s"), + Type::U16 => Kind::Primitive("i32.load16_u"), + Type::S16 => Kind::Primitive("i32.load16_s"), + Type::U32 | Type::S32 | Type::Char => Kind::Primitive("i32.load"), + Type::U64 | Type::S64 => Kind::Primitive("i64.load"), + Type::Float32 => Kind::Primitive("f32.load"), + Type::Float64 => Kind::Primitive("f64.load"), + Type::String | Type::List(_) => Kind::PointerPair, + Type::Enum(n) if *n <= (1 << 8) => Kind::Primitive("i32.load8_u"), + Type::Enum(n) if *n <= (1 << 16) => Kind::Primitive("i32.load16_u"), + Type::Enum(_) => Kind::Primitive("i32.load"), + Type::Flags(n) if *n <= 8 => Kind::Primitive("i32.load8_u"), + Type::Flags(n) if *n <= 16 => Kind::Primitive("i32.load16_u"), + Type::Flags(n) if *n <= 32 => Kind::Primitive("i32.load"), + Type::Flags(_) => unreachable!(), + + Type::Record(_) + | Type::Tuple(_) + | Type::Variant(_) + | Type::Option(_) + | Type::Result { .. } => Kind::Helper, + }; + match kind { + Kind::Primitive(op) => uwriteln!(s, "({op} offset={offset} (local.get {ptr}))"), + Kind::PointerPair => { + uwriteln!(s, "(i32.load offset={offset} (local.get {ptr}))",); + let offset = offset + 4; + uwriteln!(s, "(i32.load offset={offset} (local.get {ptr}))",); + } + Kind::Helper => { + let (index, _) = helpers.insert_full(Helper(self)); + uwriteln!(s, "(i32.add (local.get {ptr}) (i32.const {offset}))"); + uwriteln!(s, "call $load_helper_{index}"); + } + } + } + + /// Same as `store_flat_helper` but for loading the flat representation. + fn load_flat_helper<'a>( + &'a self, + s: &mut String, + i: usize, + helpers: &mut IndexSet>, + ) { + uwrite!(s, "(func $load_helper_{i} (param i32)"); + let lowered = self.lowered(); + for ty in &lowered { + uwrite!(s, " (result {ty})"); + } + s.push_str("\n"); + let record = |s: &mut String, helpers: &mut IndexSet>, types: &'a [Type]| { + for (offset, ty) in record_field_offsets(types) { + ty.load_flat(s, "0", offset, helpers); + } + }; + let variant = |s: &mut String, + helpers: &mut IndexSet>, + types: &[Option<&'a Type>]| { + let (size, offset) = variant_memory_info(types.iter().cloned()); + + // Destination locals where the flat representation will be stored. + // These are automatically zero which handles unused fields too. + for (i, ty) in lowered.iter().enumerate() { + uwriteln!(s, " (local $r{i} {ty})"); + } + + // Return block each case jumps to after setting all locals. + s.push_str("block $r\n"); + + // One extra block for "out of bounds discriminant". + for _ in 0..types.len() + 1 { + s.push_str("block\n"); + } + + // Load the discriminant and branch on it, storing it in + // `$r0` as well which is the first flat local representation. + let load = match size { + DiscriminantSize::Size1 => "i32.load8_u", + DiscriminantSize::Size2 => "i32.load16", + DiscriminantSize::Size4 => "i32.load", + }; + uwriteln!(s, "({load} (local.get 0))"); + s.push_str("local.tee $r0\n"); + s.push_str("br_table"); + for i in 0..types.len() + 1 { + uwrite!(s, " {i}"); + } + s.push_str("\nend\n"); + + // For each payload, which is in its own block, load payloads from + // memory as necessary and convert them into the final locals. + for ty in types { + if let Some(ty) = ty { + let ty_lowered = ty.lowered(); + ty.load_flat(s, "0", offset, helpers); + for (i, (from, to)) in ty_lowered.iter().zip(&lowered[1..]).enumerate().rev() { + let i = i + 1; + match (from, to) { + (CoreType::F32, CoreType::I32) => { + s.push_str("i32.reinterpret_f32\n"); + } + (CoreType::I32, CoreType::I64) => { + s.push_str("i64.extend_i32_u\n"); + } + (CoreType::F32, CoreType::I64) => { + s.push_str("i32.reinterpret_f32\n"); + s.push_str("i64.extend_i32_u\n"); + } + (CoreType::F64, CoreType::I64) => { + s.push_str("i64.reinterpret_f64\n"); + } + (a, b) if a == b => {} + _ => unimplemented!("convert {from:?} to {to:?}"), + } + uwriteln!(s, "local.set $r{i}"); + } + } + s.push_str("br $r\n"); + s.push_str("end\n"); + } + + // The catch-all block for out-of-bounds discriminants. + s.push_str("unreachable\n"); + s.push_str("end\n"); + for i in 0..lowered.len() { + uwriteln!(s, " local.get $r{i}"); + } + }; + + match self { + Type::Bool + | Type::S8 + | Type::U8 + | Type::S16 + | Type::U16 + | Type::S32 + | Type::U32 + | Type::Char + | Type::S64 + | Type::U64 + | Type::Float32 + | Type::Float64 + | Type::String + | Type::List(_) + | Type::Flags(_) + | Type::Enum(_) => unreachable!(), + + Type::Record(r) => record(s, helpers, r), + Type::Tuple(t) => record(s, helpers, t), + Type::Variant(v) => variant( + s, + helpers, + &v.iter().map(|t| t.as_ref()).collect::>(), + ), + Type::Option(o) => variant(s, helpers, &[None, Some(&**o)]), + Type::Result { ok, err } => variant(s, helpers, &[ok.as_deref(), err.as_deref()]), + }; + s.push_str(")\n"); + } +} + +#[derive(Clone)] +enum FlatSource { + Local(u32), + LocalConvert { + local: u32, + from: CoreType, + to: CoreType, + }, +} + +impl fmt::Display for FlatSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FlatSource::Local(i) => write!(f, "(local.get {i})"), + FlatSource::LocalConvert { local, from, to } => { + match (from, to) { + (a, b) if a == b => write!(f, "(local.get {local})"), + (CoreType::I32, CoreType::F32) => { + write!(f, "(f32.reinterpret_i32 (local.get {local}))") + } + (CoreType::I64, CoreType::I32) => { + write!(f, "(i32.wrap_i64 (local.get {local}))") + } + (CoreType::I64, CoreType::F64) => { + write!(f, "(f64.reinterpret_i64 (local.get {local}))") + } + (CoreType::I64, CoreType::F32) => { + write!( + f, + "(f32.reinterpret_i32 (i32.wrap_i64 (local.get {local})))" + ) + } + _ => unimplemented!("convert {from:?} to {to:?}"), + } + // .. + } + } + } } fn lower_record<'a>(types: impl Iterator, vec: &mut Vec) { @@ -350,7 +752,19 @@ fn align_to(a: usize, align: u32) -> usize { (a + (align - 1)) & !(align - 1) } -fn record_size_and_alignment<'a>(types: impl Iterator) -> SizeAndAlignment { +fn record_field_offsets<'a>( + types: impl IntoIterator, +) -> impl Iterator { + let mut offset = 0; + types.into_iter().map(move |ty| { + let SizeAndAlignment { size, alignment } = ty.size_and_alignment(); + let ret = align_to(offset, alignment); + offset = ret + size; + (ret as u32, ty) + }) +} + +fn record_size_and_alignment<'a>(types: impl IntoIterator) -> SizeAndAlignment { let mut offset = 0; let mut align = 1; for ty in types { @@ -388,80 +802,336 @@ fn variant_size_and_alignment<'a>( } } -fn make_import_and_export(params: &[&Type], result: Option<&Type>) -> String { +fn variant_memory_info<'a>( + types: impl ExactSizeIterator>, +) -> (DiscriminantSize, u32) { + let discriminant_size = DiscriminantSize::from_count(types.len()).unwrap(); + let mut alignment = u32::from(discriminant_size); + for ty in types { + if let Some(ty) = ty { + let size_and_alignment = ty.size_and_alignment(); + alignment = alignment.max(size_and_alignment.alignment); + } + } + + ( + discriminant_size, + align_to(usize::from(discriminant_size), alignment) as u32, + ) +} + +/// Generates the internals of a core wasm module which imports a single +/// component function `IMPORT_FUNCTION` and exports a single component +/// function `EXPORT_FUNCTION`. +/// +/// The component function takes `params` as arguments and optionally returns +/// `result`. The `lift_abi` and `lower_abi` fields indicate the ABI in-use for +/// this operation. +fn make_import_and_export( + params: &[&Type], + result: Option<&Type>, + lift_abi: LiftAbi, + lower_abi: LowerAbi, +) -> String { let params_lowered = params .iter() .flat_map(|ty| ty.lowered()) .collect::>(); let result_lowered = result.map(|t| t.lowered()).unwrap_or(Vec::new()); - let mut core_params = String::new(); - let mut gets = String::new(); + let mut wat = String::new(); - if params_lowered.len() <= MAX_FLAT_PARAMS { - for (index, param) in params_lowered.iter().enumerate() { - write!(&mut core_params, " {param}").unwrap(); - write!(&mut gets, "local.get {index} ").unwrap(); - } - } else { - write!(&mut core_params, " i32").unwrap(); - write!(&mut gets, "local.get 0 ").unwrap(); + enum Location { + Flat, + Indirect(u32), } - let maybe_core_params = if params_lowered.is_empty() { - String::new() - } else { - format!("(param{core_params})") + // Generate the core wasm type corresponding to the imported function being + // lowered with `lower_abi`. + wat.push_str(&format!("(type $import (func")); + let max_import_params = match lower_abi { + LowerAbi::Sync => MAX_FLAT_PARAMS, + LowerAbi::Async => MAX_FLAT_ASYNC_PARAMS, + }; + let (import_params_loc, nparams) = push_params(&mut wat, ¶ms_lowered, max_import_params); + let import_results_loc = match lower_abi { + LowerAbi::Sync => { + push_result_or_retptr(&mut wat, &result_lowered, nparams, MAX_FLAT_RESULTS) + } + LowerAbi::Async => { + let loc = if result.is_none() { + Location::Flat + } else { + wat.push_str(" (param i32)"); // result pointer + Location::Indirect(nparams) + }; + wat.push_str(" (result i32)"); // status code + loc + } }; + wat.push_str("))\n"); + + // Generate the import function. + wat.push_str(&format!( + r#"(import "host" "{IMPORT_FUNCTION}" (func $host (type $import)))"# + )); + + // Do the same as above for the exported function's type which is lifted + // with `lift_abi`. + // + // Note that `export_results_loc` being `None` means that `task.return` is + // used to communicate results. + wat.push_str(&format!("(type $export (func")); + let (export_params_loc, _nparams) = push_params(&mut wat, ¶ms_lowered, MAX_FLAT_PARAMS); + let export_results_loc = match lift_abi { + LiftAbi::Sync => Some(push_group(&mut wat, "result", &result_lowered, MAX_FLAT_RESULTS).0), + LiftAbi::AsyncCallback => { + wat.push_str(" (result i32)"); // status code + None + } + LiftAbi::AsyncStackful => None, + }; + wat.push_str("))\n"); + + // If the export is async, generate `task.return` as an import as well + // which is necesary to communicate the results. + if export_results_loc.is_none() { + wat.push_str(&format!("(type $task.return (func")); + push_params(&mut wat, &result_lowered, MAX_FLAT_PARAMS); + wat.push_str("))\n"); + wat.push_str(&format!( + r#"(import "" "task.return" (func $task.return (type $task.return)))"# + )); + } + + wat.push_str(&format!( + r#" +(func (export "{EXPORT_FUNCTION}") (type $export) + (local $retptr i32) + (local $argptr i32) + "# + )); + let mut store_helpers = IndexSet::new(); + let mut load_helpers = IndexSet::new(); + + match (export_params_loc, import_params_loc) { + // flat => flat is just moving locals around + (Location::Flat, Location::Flat) => { + for (index, _) in params_lowered.iter().enumerate() { + uwrite!(wat, "local.get {index}\n"); + } + } + + // indirect => indirect is just moving locals around + (Location::Indirect(i), Location::Indirect(j)) => { + assert_eq!(j, 0); + uwrite!(wat, "local.get {i}\n"); + } + + // flat => indirect means that all parameters are stored in memory as + // if it was a record of all the parameters. + (Location::Flat, Location::Indirect(_)) => { + let SizeAndAlignment { size, alignment } = + record_size_and_alignment(params.iter().cloned()); + wat.push_str(&format!( + r#" + (local.set $argptr + (call $realloc + (i32.const 0) + (i32.const 0) + (i32.const {alignment}) + (i32.const {size}))) + local.get $argptr + "# + )); + let mut locals = (0..params_lowered.len() as u32).map(FlatSource::Local); + for (offset, ty) in record_field_offsets(params.iter().cloned()) { + ty.store_flat(&mut wat, "$argptr", offset, &mut locals, &mut store_helpers); + } + assert!(locals.next().is_none()); + } + + (Location::Indirect(_), Location::Flat) => unreachable!(), + } + + // Pass a return-pointer if necessary. + match import_results_loc { + Location::Flat => {} + Location::Indirect(_) => { + let SizeAndAlignment { size, alignment } = result.unwrap().size_and_alignment(); + + wat.push_str(&format!( + r#" + (local.set $retptr + (call $realloc + (i32.const 0) + (i32.const 0) + (i32.const {alignment}) + (i32.const {size}))) + local.get $retptr + "# + )); + } + } + + wat.push_str("call $host\n"); + + // Assert the lowered call is ready if an async code was returned. + // + // TODO: handle when the import isn't ready yet + if let LowerAbi::Async = lower_abi { + wat.push_str("i32.const 2\n"); + wat.push_str("i32.ne\n"); + wat.push_str("if unreachable end\n"); + } + + // TODO: conditionally inject a yield here + + match (import_results_loc, export_results_loc) { + // flat => flat results involves nothing, the results are already on + // the stack. + (Location::Flat, Some(Location::Flat)) => {} + + // indirect => indirect results requires returning the `$retptr` the + // host call filled in. + (Location::Indirect(_), Some(Location::Indirect(_))) => { + wat.push_str("local.get $retptr\n"); + } - if result_lowered.len() <= MAX_FLAT_RESULTS { - let mut core_results = String::new(); - for result in result_lowered.iter() { - write!(&mut core_results, " {result}").unwrap(); + // indirect => flat requires loading the result from the return pointer + (Location::Indirect(_), Some(Location::Flat)) => { + result + .unwrap() + .load_flat(&mut wat, "$retptr", 0, &mut load_helpers); } - let maybe_core_results = if result_lowered.is_empty() { - String::new() + // flat => task.return is easy, the results are already there so just + // call the function. + (Location::Flat, None) => { + wat.push_str("call $task.return\n"); + } + + // indirect => task.return needs to forward `$retptr` if the results + // are indirect, or otherwise it must be loaded from memory to a flat + // representation. + (Location::Indirect(_), None) => { + if result_lowered.len() <= MAX_FLAT_PARAMS { + result + .unwrap() + .load_flat(&mut wat, "$retptr", 0, &mut load_helpers); + } else { + wat.push_str("local.get $retptr\n"); + } + wat.push_str("call $task.return\n"); + } + + (Location::Flat, Some(Location::Indirect(_))) => unreachable!(), + } + + if let LiftAbi::AsyncCallback = lift_abi { + wat.push_str("i32.const 0\n"); // completed status code + } + + wat.push_str(")\n"); + + // Generate a `callback` function for the callback ABI. + // + // TODO: fill this in + if let LiftAbi::AsyncCallback = lift_abi { + wat.push_str( + r#" +(func (export "callback") (param i32 i32 i32) (result i32) unreachable) + "#, + ); + } + + // Fill out all store/load helpers that were needed during generation + // above. This is a fix-point-loop since each helper may end up requiring + // more helpers. + let mut i = 0; + while i < store_helpers.len() { + let ty = store_helpers[i].0; + ty.store_flat_helper(&mut wat, i, &mut store_helpers); + i += 1; + } + i = 0; + while i < load_helpers.len() { + let ty = load_helpers[i].0; + ty.load_flat_helper(&mut wat, i, &mut load_helpers); + i += 1; + } + + return wat; + + fn push_params(wat: &mut String, params: &[CoreType], max_flat: usize) -> (Location, u32) { + push_group(wat, "param", params, max_flat) + } + + fn push_group( + wat: &mut String, + name: &str, + params: &[CoreType], + max_flat: usize, + ) -> (Location, u32) { + let mut nparams = 0; + let loc = if params.is_empty() { + // nothing to emit... + Location::Flat + } else if params.len() <= max_flat { + wat.push_str(&format!(" ({name}")); + for ty in params { + wat.push_str(&format!(" {ty}")); + nparams += 1; + } + wat.push_str(")"); + Location::Flat } else { - format!("(result{core_results})") + wat.push_str(&format!(" ({name} i32)")); + nparams += 1; + Location::Indirect(0) }; + (loc, nparams) + } - format!( - r#" - (func $f (import "host" "{IMPORT_FUNCTION}") {maybe_core_params} {maybe_core_results}) + fn push_result_or_retptr( + wat: &mut String, + results: &[CoreType], + nparams: u32, + max_flat: usize, + ) -> Location { + if results.is_empty() { + // nothing to emit... + Location::Flat + } else if results.len() <= max_flat { + wat.push_str(" (result"); + for ty in results { + wat.push_str(&format!(" {ty}")); + } + wat.push_str(")"); + Location::Flat + } else { + wat.push_str(" (param i32)"); + Location::Indirect(nparams) + } + } +} - (func (export "{EXPORT_FUNCTION}") {maybe_core_params} {maybe_core_results} - {gets} +struct Helper<'a>(&'a Type); - call $f - )"# - ) - } else { - let SizeAndAlignment { size, alignment } = result.unwrap().size_and_alignment(); +impl Hash for Helper<'_> { + fn hash(&self, h: &mut H) { + std::ptr::hash(self.0, h); + } +} - format!( - r#" - (func $f (import "host" "{IMPORT_FUNCTION}") (param{core_params} i32)) - - (func (export "{EXPORT_FUNCTION}") {maybe_core_params} (result i32) - (local $base i32) - (local.set $base - (call $realloc - (i32.const 0) - (i32.const 0) - (i32.const {alignment}) - (i32.const {size}))) - {gets} - local.get $base - - call $f - - local.get $base - )"# - ) +impl PartialEq for Helper<'_> { + fn eq(&self, other: &Self) -> bool { + std::ptr::eq(self.0, other.0) } } +impl Eq for Helper<'_> {} + fn make_rust_name(name_counter: &mut u32) -> Ident { let name = format_ident!("Foo{name_counter}"); *name_counter += 1; @@ -669,7 +1339,7 @@ impl<'a> TypesBuilder<'a> { | Type::Flags(_) => { let idx = self.next; self.next += 1; - write!(dst, "$t{idx}").unwrap(); + uwrite!(dst, "$t{idx}"); self.worklist.push((idx, ty)); } } @@ -700,7 +1370,7 @@ impl<'a> TypesBuilder<'a> { Type::Record(types) => { decl.push_str("(record"); for (index, ty) in types.iter().enumerate() { - write!(decl, r#" (field "f{index}" "#).unwrap(); + uwrite!(decl, r#" (field "f{index}" "#); self.write_ref(ty, &mut decl); decl.push_str(")"); } @@ -717,7 +1387,7 @@ impl<'a> TypesBuilder<'a> { Type::Variant(types) => { decl.push_str("(variant"); for (index, ty) in types.iter().enumerate() { - write!(decl, r#" (case "C{index}""#).unwrap(); + uwrite!(decl, r#" (case "C{index}""#); if let Some(ty) = ty { decl.push_str(" "); self.write_ref(ty, &mut decl); @@ -729,7 +1399,7 @@ impl<'a> TypesBuilder<'a> { Type::Enum(count) => { decl.push_str("(enum"); for index in 0..*count { - write!(decl, r#" "E{index}""#).unwrap(); + uwrite!(decl, r#" "E{index}""#); } decl.push_str(")"); } @@ -754,13 +1424,13 @@ impl<'a> TypesBuilder<'a> { Type::Flags(count) => { decl.push_str("(flags"); for index in 0..*count { - write!(decl, r#" "F{index}""#).unwrap(); + uwrite!(decl, r#" "F{index}""#); } decl.push_str(")"); } } decl.push_str(")\n"); - writeln!(decl, "(import \"t{idx}\" (type $t{idx} (eq $t{idx}')))").unwrap(); + uwriteln!(decl, "(import \"t{idx}\" (type $t{idx} (eq $t{idx}')))"); decl } } @@ -776,12 +1446,13 @@ pub struct Declarations { pub params: Cow<'static, str>, /// Result declaration used for the imported and exported functions pub results: Cow<'static, str>, - /// A WAT fragment representing the core function import and export to use for testing - pub import_and_export: Cow<'static, str>, - /// String encoding to use for host -> component - pub encoding1: StringEncoding, - /// String encoding to use for component -> host - pub encoding2: StringEncoding, + /// Implementation of the "caller" component, which invokes the `callee` + /// composed component. + pub caller_module: Cow<'static, str>, + /// Implementation of the "callee" component, which invokes the host. + pub callee_module: Cow<'static, str>, + /// Options used for caller/calle ABI/etc. + pub options: TestCaseOptions, } impl Declarations { @@ -792,47 +1463,115 @@ impl Declarations { type_instantiation_args, params, results, - import_and_export, - encoding1, - encoding2, + caller_module, + callee_module, + options, } = self; - let mk_component = |name: &str, encoding: StringEncoding| { + let mk_component = |name: &str, + module: &str, + import_async: bool, + export_async: bool, + encoding: StringEncoding, + lift_abi: LiftAbi, + lower_abi: LowerAbi| { + let import_async = if import_async { "async" } else { "" }; + let export_async = if export_async { "async" } else { "" }; + let lower_async_option = match lower_abi { + LowerAbi::Sync => "", + LowerAbi::Async => "async", + }; + let lift_async_option = match lift_abi { + LiftAbi::Sync => "", + LiftAbi::AsyncStackful => "async", + LiftAbi::AsyncCallback => "async (callback (func $i \"callback\"))", + }; + + let mut intrinsic_defs = String::new(); + let mut intrinsic_imports = String::new(); + + match lift_abi { + LiftAbi::Sync => {} + LiftAbi::AsyncCallback | LiftAbi::AsyncStackful => { + intrinsic_defs.push_str(&format!( + r#" +(core func $task.return (canon task.return {results} + (memory $libc "memory") string-encoding={encoding})) + "#, + )); + intrinsic_imports.push_str( + r#" +(with "" (instance (export "task.return" (func $task.return)))) + "#, + ); + } + } + format!( r#" - (component ${name} - {types} - (type $sig (func {params} {results})) - (import "{IMPORT_FUNCTION}" (func $f (type $sig))) - - (core instance $libc (instantiate $libc)) - - (core func $f_lower (canon lower - (func $f) - (memory $libc "memory") - (realloc (func $libc "realloc")) - string-encoding={encoding} - )) - - (core instance $i (instantiate $m - (with "libc" (instance $libc)) - (with "host" (instance (export "{IMPORT_FUNCTION}" (func $f_lower)))) - )) - - (func (export "{EXPORT_FUNCTION}") (type $sig) - (canon lift - (core func $i "{EXPORT_FUNCTION}") - (memory $libc "memory") - (realloc (func $libc "realloc")) - string-encoding={encoding} - ) - ) - ) +(component ${name} + {types} + (type $import_sig (func {import_async} {params} {results})) + (type $export_sig (func {export_async} {params} {results})) + (import "{IMPORT_FUNCTION}" (func $f (type $import_sig))) + + (core instance $libc (instantiate $libc)) + + (core func $f_lower (canon lower + (func $f) + (memory $libc "memory") + (realloc (func $libc "realloc")) + string-encoding={encoding} + {lower_async_option} + )) + + {intrinsic_defs} + + (core module $m + (memory (import "libc" "memory") 1) + (func $realloc (import "libc" "realloc") (param i32 i32 i32 i32) (result i32)) + + {module} + ) + + (core instance $i (instantiate $m + (with "libc" (instance $libc)) + (with "host" (instance (export "{IMPORT_FUNCTION}" (func $f_lower)))) + {intrinsic_imports} + )) + + (func (export "{EXPORT_FUNCTION}") (type $export_sig) + (canon lift + (core func $i "{EXPORT_FUNCTION}") + (memory $libc "memory") + (realloc (func $libc "realloc")) + string-encoding={encoding} + {lift_async_option} + ) + ) +) "# ) }; - let c1 = mk_component("c1", *encoding2); - let c2 = mk_component("c2", *encoding1); + let c1 = mk_component( + "callee", + &callee_module, + options.host_async, + options.guest_callee_async, + options.callee_encoding, + options.callee_lift_abi, + options.callee_lower_abi, + ); + let c2 = mk_component( + "caller", + &caller_module, + options.guest_callee_async, + options.guest_caller_async, + options.caller_encoding, + options.caller_lift_abi, + options.caller_lower_abi, + ); + let host_async = if options.host_async { "async" } else { "" }; format!( r#" @@ -842,25 +1581,19 @@ impl Declarations { {REALLOC_AND_FREE} ) - (core module $m - (memory (import "libc" "memory") 1) - (func $realloc (import "libc" "realloc") (param i32 i32 i32 i32) (result i32)) - - {import_and_export} - ) {types} - (type $sig (func {params} {results})) - (import "{IMPORT_FUNCTION}" (func $f (type $sig))) + (type $host_sig (func {host_async} {params} {results})) + (import "{IMPORT_FUNCTION}" (func $f (type $host_sig))) {c1} {c2} - (instance $c1 (instantiate $c1 + (instance $c1 (instantiate $callee {type_instantiation_args} (with "{IMPORT_FUNCTION}" (func $f)) )) - (instance $c2 (instantiate $c2 + (instance $c2 (instantiate $caller {type_instantiation_args} (with "{IMPORT_FUNCTION}" (func $c1 "{EXPORT_FUNCTION}")) )) @@ -878,13 +1611,73 @@ pub struct TestCase<'a> { pub params: Vec<&'a Type>, /// The result types of the function pub result: Option<&'a Type>, - /// String encoding to use from host-to-component. - pub encoding1: StringEncoding, - /// String encoding to use from component-to-host. - pub encoding2: StringEncoding, + /// ABI options to use for this test case. + pub options: TestCaseOptions, } -impl TestCase<'_> { +/// Collection of options which configure how the caller/callee/etc ABIs are +/// all configured. +#[derive(Debug, Arbitrary, Copy, Clone)] +pub struct TestCaseOptions { + /// Whether or not the guest caller component (the entrypoint) is using an + /// `async` function type. + pub guest_caller_async: bool, + /// Whether or not the guest callee component (what the entrypoint calls) + /// is using an `async` function type. + pub guest_callee_async: bool, + /// Whether or not the host is using an async function type (what the + /// guest callee calls). + pub host_async: bool, + /// The string encoding of the caller component. + pub caller_encoding: StringEncoding, + /// The string encoding of the callee component. + pub callee_encoding: StringEncoding, + /// The ABI that the caller component is using to lift its export (the main + /// entrypoint). + pub caller_lift_abi: LiftAbi, + /// The ABI that the callee component is using to lift its export (called + /// by the caller). + pub callee_lift_abi: LiftAbi, + /// The ABI that the caller component is using to lower its import (the + /// callee's export). + pub caller_lower_abi: LowerAbi, + /// The ABI that the callee component is using to lower its import (the + /// host function). + pub callee_lower_abi: LowerAbi, +} + +#[derive(Debug, Arbitrary, Copy, Clone)] +pub enum LiftAbi { + Sync, + AsyncStackful, + AsyncCallback, +} + +#[derive(Debug, Arbitrary, Copy, Clone)] +pub enum LowerAbi { + Sync, + Async, +} + +impl<'a> TestCase<'a> { + pub fn generate(types: &'a [Type], u: &mut Unstructured<'_>) -> arbitrary::Result { + let max_params = if types.len() > 0 { 5 } else { 0 }; + let params = (0..u.int_in_range(0..=max_params)?) + .map(|_| u.choose(&types)) + .collect::>>()?; + let result = if types.len() > 0 && u.arbitrary()? { + Some(u.choose(&types)?) + } else { + None + }; + + Ok(Self { + params, + result, + options: u.arbitrary()?, + }) + } + /// Generate a `Declarations` for this `TestCase` which may be used to build a component to execute the case. pub fn declarations(&self) -> Declarations { let mut builder = TypesBuilder::default(); @@ -903,13 +1696,24 @@ impl TestCase<'_> { results.push_str(")"); } - let import_and_export = make_import_and_export(&self.params, self.result); + let caller_module = make_import_and_export( + &self.params, + self.result, + self.options.caller_lift_abi, + self.options.caller_lower_abi, + ); + let callee_module = make_import_and_export( + &self.params, + self.result, + self.options.callee_lift_abi, + self.options.callee_lower_abi, + ); let mut type_decls = Vec::new(); let mut type_instantiation_args = String::new(); while let Some((idx, ty)) = builder.worklist.pop() { type_decls.push(builder.write_decl(idx, ty)); - writeln!(type_instantiation_args, "(with \"t{idx}\" (type $t{idx}))").unwrap(); + uwriteln!(type_instantiation_args, "(with \"t{idx}\" (type $t{idx}))"); } // Note that types are printed here in reverse order since they were @@ -926,9 +1730,9 @@ impl TestCase<'_> { type_instantiation_args: type_instantiation_args.into(), params: params.into(), results: results.into(), - import_and_export: import_and_export.into(), - encoding1: self.encoding1, - encoding2: self.encoding2, + caller_module: caller_module.into(), + callee_module: callee_module.into(), + options: self.options, } } } @@ -950,6 +1754,54 @@ impl fmt::Display for StringEncoding { } } +impl ToTokens for TestCaseOptions { + fn to_tokens(&self, tokens: &mut TokenStream) { + let TestCaseOptions { + guest_caller_async, + guest_callee_async, + host_async, + caller_encoding, + callee_encoding, + caller_lift_abi, + callee_lift_abi, + caller_lower_abi, + callee_lower_abi, + } = self; + tokens.extend(quote!(wasmtime_test_util::component_fuzz::TestCaseOptions { + guest_caller_async: #guest_caller_async, + guest_callee_async: #guest_callee_async, + host_async: #host_async, + caller_encoding: #caller_encoding, + callee_encoding: #callee_encoding, + caller_lift_abi: #caller_lift_abi, + callee_lift_abi: #callee_lift_abi, + caller_lower_abi: #caller_lower_abi, + callee_lower_abi: #callee_lower_abi, + })); + } +} + +impl ToTokens for LowerAbi { + fn to_tokens(&self, tokens: &mut TokenStream) { + let me = match self { + LowerAbi::Sync => quote!(Sync), + LowerAbi::Async => quote!(Async), + }; + tokens.extend(quote!(wasmtime_test_util::component_fuzz::LowerAbi::#me)); + } +} + +impl ToTokens for LiftAbi { + fn to_tokens(&self, tokens: &mut TokenStream) { + let me = match self { + LiftAbi::Sync => quote!(Sync), + LiftAbi::AsyncCallback => quote!(AsyncCallback), + LiftAbi::AsyncStackful => quote!(AsyncStackful), + }; + tokens.extend(quote!(wasmtime_test_util::component_fuzz::LiftAbi::#me)); + } +} + impl ToTokens for StringEncoding { fn to_tokens(&self, tokens: &mut TokenStream) { let me = match self { @@ -960,3 +1812,45 @@ impl ToTokens for StringEncoding { tokens.extend(quote!(wasmtime_test_util::component_fuzz::StringEncoding::#me)); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arbtest() { + arbtest::arbtest(|u| { + let mut fuel = 100; + let types = (0..5) + .map(|_| Type::generate(u, 3, &mut fuel)) + .collect::>>()?; + let case = TestCase::generate(&types, u)?; + let decls = case.declarations(); + let component = decls.make_component(); + let wasm = wat::parse_str(&component).unwrap_or_else(|e| { + panic!("failed to parse generated component as wat: {e}\n\n{component}"); + }); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&wasm) + .unwrap_or_else(|e| { + let mut wat = String::new(); + let mut dst = wasmprinter::PrintFmtWrite(&mut wat); + let to_print = if wasmprinter::Config::new() + .print_offsets(true) + .print_operand_stack(true) + .print(&wasm, &mut dst) + .is_ok() + { + &wat[..] + } else { + &component[..] + }; + panic!("generated component is not valid wasm: {e}\n\n{to_print}"); + }); + Ok(()) + }) + .budget_ms(1_000) + // .seed(0x3c9050d4000000e9) + ; + } +} diff --git a/fuzz/build.rs b/fuzz/build.rs index c864fc0bd503..aef25daf8ba0 100644 --- a/fuzz/build.rs +++ b/fuzz/build.rs @@ -11,6 +11,7 @@ mod component { use quote::quote; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; + use std::collections::HashMap; use std::env; use std::fmt::Write; use std::fs; @@ -51,11 +52,11 @@ mod component { let mut rng = StdRng::seed_from_u64(seed); const TYPE_COUNT: usize = 50; - const MAX_ARITY: u32 = 5; const TEST_CASE_COUNT: usize = 100; let mut type_fuel = 1000; let mut types = Vec::new(); + let mut rust_type_names = Vec::new(); let name_counter = &mut 0; let mut declarations = TokenStream::new(); let mut tests = TokenStream::new(); @@ -73,38 +74,40 @@ mod component { ret })?; - let name = + let rust_ty_name = wasmtime_test_util::component_fuzz::rust_type(&ty, name_counter, &mut declarations); - types.push((name, ty)); + types.push(ty); + rust_type_names.push(rust_ty_name); } + fn hash_key(ty: &Type) -> usize { + let ty: *const Type = ty; + ty.addr() + } + + let type_to_name_map = types + .iter() + .map(hash_key) + .zip(rust_type_names.iter().cloned()) + .collect::>(); + // Next generate a set of static API test cases driven by the above // types. for index in 0..TEST_CASE_COUNT { let (case, rust_params, rust_results) = generate(&mut rng, |u| { - let mut params = Vec::new(); - let mut result = None; let mut rust_params = TokenStream::new(); let mut rust_results = TokenStream::new(); - for _ in 0..u.int_in_range(0..=MAX_ARITY)? { - let (name, ty) = u.choose(&types)?; - params.push(ty); + let case = TestCase::generate(&types, u)?; + for ty in case.params.iter() { + let name = &type_to_name_map[&hash_key(ty)]; rust_params.extend(name.clone()); rust_params.extend(quote!(,)); } - if u.arbitrary()? { - let (name, ty) = u.choose(&types)?; - result = Some(ty); + if let Some(ty) = &case.result { + let name = &type_to_name_map[&hash_key(ty)]; rust_results.extend(name.clone()); rust_results.extend(quote!(,)); } - - let case = TestCase { - params, - result, - encoding1: u.arbitrary()?, - encoding2: u.arbitrary()?, - }; Ok((case, rust_params, rust_results)) })?; @@ -113,9 +116,9 @@ mod component { type_instantiation_args, params, results, - import_and_export, - encoding1, - encoding2, + caller_module, + callee_module, + options, } = case.declarations(); let test = quote!(#index => component_types::static_api_test::<(#rust_params), (#rust_results)>( @@ -126,9 +129,9 @@ mod component { type_instantiation_args: Cow::Borrowed(#type_instantiation_args), params: Cow::Borrowed(#params), results: Cow::Borrowed(#results), - import_and_export: Cow::Borrowed(#import_and_export), - encoding1: #encoding1, - encoding2: #encoding2, + caller_module: Cow::Borrowed(#caller_module), + callee_module: Cow::Borrowed(#callee_module), + options: #options, }; &DECLS }