diff --git a/Cargo.lock b/Cargo.lock
index a3c101a7..2ed0df2e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1611,6 +1611,10 @@ dependencies = [
 name = "progenitor-impl"
 version = "0.10.0"
 dependencies = [
+ "anyhow",
+ "base64",
+ "chrono",
+ "clap",
  "dropshot",
  "expectorate",
  "futures",
@@ -1622,7 +1626,9 @@ dependencies = [
  "proc-macro2",
  "progenitor-client",
  "quote",
+ "rand",
  "regex",
+ "regress",
  "reqwest",
  "rustfmt-wrapper",
  "schemars",
@@ -1631,10 +1637,12 @@ dependencies = [
  "serde_json",
  "serde_yaml",
  "syn",
+ "test-progenitor-compilation",
  "thiserror 2.0.12",
  "tokio",
  "typify",
  "unicode-ident",
+ "uuid",
 ]
 
 [[package]]
@@ -2311,6 +2319,24 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
 
+[[package]]
+name = "test-progenitor-compilation"
+version = "0.1.0"
+dependencies = [
+ "dropshot",
+ "expectorate",
+ "http",
+ "hyper",
+ "openapiv3",
+ "proc-macro2",
+ "progenitor-impl",
+ "quote",
+ "schemars",
+ "serde_json",
+ "serde_yaml",
+ "syn",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.69"
diff --git a/Cargo.toml b/Cargo.toml
index f90a1922..3092c6c3 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
 	"progenitor-client",
 	"progenitor-impl",
 	"progenitor-macro",
+	"test-progenitor-compilation"
 ]
 
 resolver = "2"
diff --git a/progenitor-impl/Cargo.toml b/progenitor-impl/Cargo.toml
index f4a3a9af..d19a5392 100644
--- a/progenitor-impl/Cargo.toml
+++ b/progenitor-impl/Cargo.toml
@@ -24,14 +24,23 @@ typify = { workspace = true }
 unicode-ident = { workspace = true }
 
 [dev-dependencies]
+anyhow = { workspace = true }
+base64 = { workspace = true }
+clap = { workspace = true, features = ["string"] }
+chrono = { workspace = true }
 dropshot = { workspace = true }
 expectorate = { workspace = true }
 futures = { workspace = true }
 http = { workspace = true }
 hyper = { workspace = true }
 progenitor-client = { workspace = true }
+rand = { workspace = true }
+regress = { workspace = true }
 reqwest = { workspace = true }
 rustfmt-wrapper = { workspace = true }
+schemars = { workspace = true }
 semver = { workspace = true }
 serde_yaml = { workspace = true }
+test-progenitor-compilation = { path = "../test-progenitor-compilation" }
 tokio = { workspace = true }
+uuid = { workspace = true }
diff --git a/progenitor-impl/tests/test_compilation.rs b/progenitor-impl/tests/test_compilation.rs
new file mode 100644
index 00000000..e2324ca2
--- /dev/null
+++ b/progenitor-impl/tests/test_compilation.rs
@@ -0,0 +1,46 @@
+// Copyright 2022 Oxide Computer Company
+
+use test_progenitor_compilation::cli_tokens;
+
+#[test]
+fn test_keeper_compilation() {
+    cli_tokens!("keeper.json");
+}
+
+#[test]
+fn test_buildomat_compilation() {
+    cli_tokens!("buildomat.json");
+}
+
+#[test]
+fn test_nexus_compilation() {
+    cli_tokens!("nexus.json");
+}
+
+#[test]
+fn test_propolis_server_compilation() {
+    cli_tokens!("propolis-server.json");
+}
+
+#[test]
+fn test_param_override_compilation() {
+    cli_tokens!("param-overrides.json");
+}
+
+#[test]
+fn test_yaml_compilation() {
+    cli_tokens!("param-overrides.yaml");
+}
+
+#[test]
+fn test_param_collision_compilation() {
+    cli_tokens!("param-collision.json");
+}
+
+// TODO this file is full of inconsistencies and incorrectly specified types.
+// It's an interesting test to consider whether we try to do our best to
+// interpret the intent or just fail.
+// #[test]
+// fn test_github() {
+//     cli_tokens!("api.github.com.json");
+// }
diff --git a/test-progenitor-compilation/Cargo.toml b/test-progenitor-compilation/Cargo.toml
new file mode 100644
index 00000000..9fc6fa9b
--- /dev/null
+++ b/test-progenitor-compilation/Cargo.toml
@@ -0,0 +1,24 @@
+[package]
+name = "test-progenitor-compilation"
+version = "0.1.0"
+edition = "2021"
+license = "MPL-2.0"
+description = "An OpenAPI client generator - compilation test macro"
+repository = "https://github.com/oxidecomputer/progenitor.git"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+dropshot = { workspace = true }
+expectorate = { workspace = true }
+http = { workspace = true }
+hyper = { workspace = true }
+openapiv3 = { workspace = true }
+proc-macro2 = { workspace = true }
+progenitor-impl = { workspace = true }
+quote = { workspace = true }
+schemars = { workspace = true }
+serde_yaml = { workspace = true }
+serde_json = { workspace = true }
+syn = { workspace = true }
diff --git a/test-progenitor-compilation/src/lib.rs b/test-progenitor-compilation/src/lib.rs
new file mode 100644
index 00000000..36ee7e77
--- /dev/null
+++ b/test-progenitor-compilation/src/lib.rs
@@ -0,0 +1,58 @@
+// Copyright 2022 Oxide Computer Company
+
+extern crate proc_macro;
+
+use openapiv3::OpenAPI;
+use proc_macro::TokenStream;
+use progenitor_impl::{GenerationSettings, Generator, InterfaceStyle, TagStyle};
+use quote::quote;
+use std::{
+    fs::File,
+    path::{Path, PathBuf},
+};
+
+#[proc_macro]
+pub fn cli_tokens(item: TokenStream) -> TokenStream {
+    let arg = item.to_string().replace("\"", "");
+    let mut in_path = PathBuf::from("sample_openapi");
+    in_path.push(arg);
+
+    let spec = load_api(in_path);
+
+    let mut generator = Generator::new(
+        GenerationSettings::default()
+            .with_interface(InterfaceStyle::Builder)
+            .with_tag(TagStyle::Merged)
+            .with_derive("schemars::JsonSchema"),
+    );
+
+    // Builder generation with tags.
+    let sdk = generator.generate_tokens(&spec).unwrap();
+
+    // CLI generation.
+    let cli = generator.cli(&spec, "sdk").unwrap();
+
+    quote! {
+        pub mod sdk {
+            #sdk
+        }
+        use sdk::*;
+
+        #cli
+    }
+    .into()
+}
+
+fn load_api<P>(p: P) -> OpenAPI
+where
+    P: AsRef<Path> + std::clone::Clone + std::fmt::Debug,
+{
+    let mut f = File::open(p.clone()).unwrap();
+    match serde_json::from_reader(f) {
+        Ok(json_value) => json_value,
+        _ => {
+            f = File::open(p).unwrap();
+            serde_yaml::from_reader(f).unwrap()
+        }
+    }
+}