libafl_nyx: make aux_buffer_size configurable (#3549)

Author: tokatoka

crates/libafl_nyx/src/helper.rs

@@ -45,6 +45,7 @@ impl NyxHelper {
             Error::illegal_argument(format!("Failed to load Nyx config from share dir: {e}"))
         })?;
         nyx_config.set_input_buffer_size(settings.input_buffer_size);
+        nyx_config.set_aux_buffer_size(settings.aux_buffer_size);
         nyx_config.set_process_role(match settings.parent_cpu_id {
             None => NyxProcessRole::StandAlone,
             Some(parent_cpu_id) if parent_cpu_id == settings.cpu_id => NyxProcessRole::Parent,

crates/libafl_nyx/src/settings.rs

@@ -44,6 +44,10 @@ pub struct NyxSettings {
     #[builder(default = DEFAULT_TIMEOUT_SECS)]
     pub timeout_secs: u8,
 
+    /// The output buffer size (in bytes) used to record the stdout/stderr messages
+    #[builder(default = 32768)]
+    pub aux_buffer_size: usize,
+
     /// Additional timeout in microseconds that gets added to
     /// `timeout_secs`.
     #[builder(default = DEFAULT_TIMEOUT_MICRO_SECS)]