diff --git a/Cargo.lock b/Cargo.lock index 649ec9d0d..8e01e2a9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,6 +258,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -678,9 +689,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.112.0" +version = "1.113.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee73a27721035c46da0572b390a69fbdb333d0177c24f3d8f7ff952eeb96690" +checksum = "85b6f2da36f0e3ca98031dfa4fc6f075c0887b864f01397161c67e07401f438d" dependencies = [ "aws-credential-types", "aws-runtime", @@ -702,7 +713,7 @@ dependencies = [ "http 0.2.12", "http 1.3.1", "http-body 0.4.6", - "lru", + "lru 0.12.5", "percent-encoding", "regex-lite", "sha2", @@ -1405,9 +1416,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.52" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8120877db0e5c011242f96806ce3c94e0737ab8108532a76a3300a01db2ab8" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -1415,9 +1426,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.52" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02576b399397b659c26064fbc92a75fede9d18ffd5f80ca1cd74ddab167016e1" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -2878,6 +2889,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -2885,7 +2899,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash", + "ahash 0.8.12", ] [[package]] @@ -3439,7 +3453,7 @@ version = "0.11.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ - "ahash", + "ahash 0.8.12", "indexmap 2.12.0", "is-terminal", "itoa", @@ -3830,6 +3844,15 @@ dependencies = [ "imgref", ] +[[package]] +name = "lru" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "lru" version = "0.12.5" @@ -3974,6 +3997,28 @@ dependencies = [ "serde", ] +[[package]] +name = "martin-tracing-utils" +version = "0.1.0" +dependencies = [ + "clap", + "serde", + "serde_yaml", + "tempfile", + "tracing", + "tracing-log", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -4227,6 +4272,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num" version = "0.2.1" @@ -6240,6 +6294,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -6966,6 +7029,15 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "tiff" version = "0.10.3" @@ -7410,6 +7482,51 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "ahash 0.7.8", + "log", + "lru 0.7.8", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", ] [[package]] @@ -7655,6 +7772,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "varint-rs" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index bdbf7ba54..b1c2cd697 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["martin", "martin-core", "martin-tile-utils", "mbtiles"] +members = ["martin", "martin-core", "martin-tracing-utils", "martin-tile-utils", "mbtiles"] [workspace.package] edition = "2024" @@ -104,6 +104,10 @@ tiff = "0.10.1" tilejson = "0.4" tokio = { version = "1", features = ["macros"] } tokio-postgres-rustls = "0.13" +tracing = { version = "0.1.41", features = ["log"] } +tracing-log = { version ="0.2.0", features = ["interest-cache"] } +tracing-subscriber = { version = "0.3.19", features = ["env-filter", "json"] } +tracing-test = { version = "0.2.5",features = ["no-env-filter"] } url = "2.5" walkdir = "2.5.0" xxhash-rust = { version = "0.8", features = ["xxh3"] } diff --git a/martin-tracing-utils/Cargo.toml b/martin-tracing-utils/Cargo.toml new file mode 100644 index 000000000..5404eb6e7 --- /dev/null +++ b/martin-tracing-utils/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "martin-tracing-utils" +version = "0.1.0" +authors = ["Yuri Astrakhan ", "MapLibre contributors"] +description = "Utilities to help with tracing. Used by the MapLibre's Martin tile server." +keywords = ["tracing", "mvt", "tileserver"] +categories = ["science::geo", "web-programming"] +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +homepage.workspace = true + +[dependencies] +clap.workspace = true +serde.workspace = true +serde_yaml.workspace = true +tracing-log.workspace = true +tracing-subscriber.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/martin-tracing-utils/README.md b/martin-tracing-utils/README.md new file mode 100644 index 000000000..719ce4649 --- /dev/null +++ b/martin-tracing-utils/README.md @@ -0,0 +1,24 @@ +# martin-tracing-utils + +[![docs.rs docs](https://docs.rs/martin-tracing-utils/badge.svg)](https://docs.rs/martin-tracing-utils) +[![](https://img.shields.io/badge/Slack-%23maplibre--martin-blueviolet?logo=slack)](https://slack.openstreetmap.us/) +[![GitHub](https://img.shields.io/badge/github-maplibre/martin-8da0cb?logo=github)](https://github.com/maplibre/martin) +[![crates.io version](https://img.shields.io/crates/v/martin-tracing-utils.svg)](https://crates.io/crates/martin-tracing-utils) +[![CI build](https://github.com/maplibre/martin/actions/workflows/ci.yml/badge.svg)](https://github.com/maplibre/martin/actions) + +A library to help tile servers like [Martin](https://maplibre.org/martin) do tracing ("logging, but slightly fancier"). + +## License + +Licensed under either of + +* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) + at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally +submitted for inclusion in the work by you, as defined in the +Apache-2.0 license, shall be dual licensed as above, without any +additional terms or conditions. diff --git a/martin-tracing-utils/src/lib.rs b/martin-tracing-utils/src/lib.rs new file mode 100644 index 000000000..27e9f63b3 --- /dev/null +++ b/martin-tracing-utils/src/lib.rs @@ -0,0 +1,337 @@ +#![doc = include_str!("../README.md")] + +use std::collections::HashMap; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use tracing_subscriber::EnvFilter; + +#[derive(Default)] +pub struct MartinObservability { + filter: EnvFilter, + log_format: LogFormatOptions, +} +impl MartinObservability { + /// transform [`log`](https://docs.rs/log) records into [`tracing`](https://docs.rs/tracing) [`Event`](tracing::Event)s. + /// + /// # Panics + /// This function will panic if the global `log`-logger cannot be set. + /// This only happens if the global `log`-logger has already been set. + #[must_use] + pub fn with_initialised_log_tracing(self) -> Self { + tracing_log::LogTracer::builder() + .with_interest_cache(tracing_log::InterestCacheConfig::default()) + .init() + .expect("the global logger to only be set once"); + self + } + /// Set the global subscriber for the application. + /// + /// # Panics + /// This function will panic if the global subscriber cannot be set. + /// This only happens if the global subscriber has already been set. + pub fn set_global_subscriber(self) { + use tracing::subscriber::set_global_default; + use tracing_subscriber::fmt::Layer; + use tracing_subscriber::prelude::*; + let registry = tracing_subscriber::registry().with(self.filter); + match self.log_format { + LogFormatOptions::Full => set_global_default(registry.with(Layer::default())), + LogFormatOptions::Compact => { + set_global_default(registry.with(Layer::default().compact())) + } + LogFormatOptions::Bare => { + set_global_default(registry.with(Layer::default().compact().without_time())) + } + LogFormatOptions::Pretty => { + set_global_default(registry.with(Layer::default().pretty())) + } + LogFormatOptions::Json => set_global_default(registry.with(Layer::default().json())), + } + .expect("since martin has not set the global_default, no global default is set"); + } +} +impl From<(EnvFilter, LogFormatOptions)> for MartinObservability { + fn from((filter, log_format): (EnvFilter, LogFormatOptions)) -> Self { + Self { filter, log_format } + } +} + +#[derive( + PartialEq, + Eq, + Clone, + Copy, + Default, + Debug, + clap::ValueEnum, + serde::Serialize, + serde::Deserialize, +)] +pub enum LogFormatOptions { + /// Emit human-readable, single-line logs. + /// See [here for a sample](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/format/struct.Full.html#example-output) + Full, + /// A variant of the full-format, optimized for short line lengths. + /// See [here for a sample](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/format/struct.Compact.html#example-output) + #[default] + Compact, + /// the bare log without timestamps, modules. Just the level and the log + Bare, + /// Excessively pretty, multi-line logs for local development/debugging, prioritizing readability over compact storage. + /// See [here for a sample](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/format/struct.Pretty.html#example-output) + Pretty, + /// Output newline-delimited (structured) JSON logs, ***not*** optimized for human readability. + /// See [here for a sample](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/format/struct.Json.html#example-output) + Json, +} +impl LogFormatOptions { + fn from_str(key: &str) -> Option { + match std::env::var(key).unwrap_or_default().as_str() { + "full" => Some(LogFormatOptions::Full), + "pretty" | "verbose" => Some(LogFormatOptions::Pretty), + "json" | "jsonl" => Some(LogFormatOptions::Json), + "compact" => Some(LogFormatOptions::Compact), + "bare" => Some(LogFormatOptions::Bare), + _ => None, + } + } +} + +pub struct LogFormat(Option); +impl LogFormat { + /// Search for the log format (how the logs are formatted on the cli) as an argument in the CLI + /// + /// Due to [`clap`] having a help function, it is not possible to use it. + #[must_use] + pub fn from_argument(argument: &str) -> Self { + let args = std::env::args().collect::>(); + let v = get_next_after_argument(argument, &args); + if let Some(v) = v { + if let Some(v) = LogFormatOptions::from_str(&v) { + Self(Some(v)) + } else { + eprintln!( + "Ignoring specified cli argument {argument} {v} as it is not a valid log format. Can be one of full, compact, bare, pretty, json" + ); + Self(None) + } + } else { + Self(None) + } + } + /// Search for the log format (how the logs are formatted on the cli) at a path in a config file + #[must_use] + pub fn or_in_config_file(mut self, argument: &str, key: &str) -> Self { + if self.0.is_none() { + let args = std::env::args().collect::>(); + if let Some(path) = get_next_after_argument(argument, &args) { + let path = PathBuf::from(path); + if let Some(v) = read_path_in_file(&path, key) { + match LogFormatOptions::from_str(&v) { + Some(v) => self.0 = Some(v), + None => eprintln!( + "Ignoring specified option {key}: {v} inside {} as it is not a valid log format. Can be one of full, compact, bare, pretty, json", + path.display() + ), + } + } + } + } + self + } + /// Gets log format (how the logs are formatted on the cli) from an environment variable + #[must_use] + pub fn or_env_var(mut self, key: &'static str) -> Self { + if self.0.is_none() { + if let Ok(v) = std::env::var(key) { + match LogFormatOptions::from_str(&v) { + Some(v) => self.0 = Some(v), + None => eprintln!( + "Ignoring specified environment variable {key}={v} as it is not a valid log format. Can be one of full, compact, bare, pretty, json" + ), + } + } + } + self + } + /// Sets a default + #[must_use] + pub fn or_default(self, default_format: LogFormatOptions) -> LogFormatOptions { + self.0.unwrap_or(default_format) + } +} + +/// Allows configuring log directives +/// +/// See [here](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#example-syntax) for more information. +#[derive(Clone, PartialEq, Debug)] +pub struct LogLevel(Option); +impl LogLevel { + /// Search for the log level at a path in the CLI + /// + /// Due to [`clap`] having a help function, it is not possible to use it. + /// All errors during this operation are ignored as the default ([`tracing::Level::INFO`]) will print errors for this too during the regular parsing. + #[must_use] + pub fn from_argument(argument: &str) -> Self { + let args = std::env::args().collect::>(); + Self(get_next_after_argument(argument, &args)) + } + /// Search for the log level at a path in a config file + /// + /// All errors during this operation are ignored as the default ([`tracing::Level::INFO`]) will print errors for this too during the regular parsing. + #[must_use] + pub fn or_in_config_file(mut self, argument: &str, key: &str) -> Self { + if self.0.is_none() { + let args = std::env::args().collect::>(); + if let Some(path) = get_next_after_argument(argument, &args) { + let path = PathBuf::from(path); + self.0 = read_path_in_file(path.as_path(), key); + } + } + self + } + /// Get log directives from an environment variable + #[must_use] + pub fn or_env_var(mut self, key: &str) -> Self { + if self.0.is_none() { + self.0 = std::env::var(key).ok(); + } + self + } + /// Parse a [`EnvFilter`] from the directives in the string to this point, ignoring any that are invalid. + /// + /// See [here](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#example-syntax) for more information. + #[must_use] + pub fn lossy_parse_to_filter_with_default(self, default_directives: &str) -> EnvFilter { + let directives = match self.0 { + Some(directives) => directives, + None => default_directives.to_string(), + }; + EnvFilter::builder().parse_lossy(directives) + } +} + +/// Search for the argument following a certain argument in the cli +#[must_use] +fn get_next_after_argument(argument: &str, args: &[String]) -> Option { + let mut args = args.iter(); + let _ = args.next(); // first argument is binary + while let Some(arg) = args.next() { + if arg == argument { + return args.next().cloned(); + } + } + None +} +/// Reads a key from a yaml file at a path +/// +/// Supports dot notation for nested keys. +/// **ALL errors (including parsing and io) are ignored and return [`None`]**. +/// It is assumed that the user will parse the yaml file themselves after setting up tracing via this library. +#[must_use] +fn read_path_in_file(path: &Path, key: &str) -> Option { + let mut traversial_path = key.split('.').collect::>(); + + let mut config_file = Vec::new(); + let _ = File::open(path).ok()?.read_to_end(&mut config_file).ok()?; + let mut map: HashMap = serde_yaml::from_slice(&config_file).ok()?; + let final_step = traversial_path.pop()?; + for traversal_step in traversial_path { + let new_map = map.remove(traversal_step)?; + let new_map: HashMap = serde_yaml::from_value(new_map).ok()?; + map = new_map; + } + + let v = map.remove(final_step)?; + Some(v.as_str()?.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_log_level_env_var() { + // Safety: Definitively NOT safe, but works for testing purposes + unsafe { + std::env::set_var("TEST_LOG_LEVEL_DEBUG", "debug"); + } + + let log_level = LogLevel(None).or_env_var("TEST_NOT_EXISTING_VARIABLE"); + assert_eq!(log_level, LogLevel(None)); + let log_level = LogLevel(None).or_env_var("TEST_LOG_LEVEL_DEBUG"); + assert_eq!(log_level, LogLevel(Some("debug".to_string()))); + } + + #[test] + fn test_get_next_after_argument() { + let args = vec![ + "binary-path-goes-here".to_string(), + "--log-level".to_string(), + "trace".to_string(), + "--log-level2".to_string(), + ]; + let log_level = get_next_after_argument("not-found", &args); + assert_eq!(log_level, None); + let log_level = get_next_after_argument("binary-path-goes-here", &args); + assert_eq!(log_level, None); // should be skipped + let log_level = get_next_after_argument("--log-level", &args); + assert_eq!(log_level, Some("trace".to_string())); + let log_level = get_next_after_argument("--log-level2", &args); + assert_eq!(log_level, None); + } + + #[test] + fn test_read_path_in_file() { + let dir = tempfile::tempdir().unwrap(); + let config_path = dir.path().join("config.yaml"); + + let val = read_path_in_file(&config_path, "log_level"); + assert_eq!(val, None); + + let mut file = File::create(&config_path).unwrap(); + file.write_all( + r" +log_level: warn +foo: + bar: baz" + .as_bytes(), + ) + .unwrap(); + + let val = read_path_in_file(&config_path, "key_not_found"); + assert_eq!(val, None); + let val = read_path_in_file(&config_path, "log_level"); + assert_eq!(val, Some("warn".to_string())); + let val = read_path_in_file(&config_path, "foo.bar"); + assert_eq!(val, Some("baz".to_string())); + let val = read_path_in_file(&config_path, "foo.key_not_found"); + assert_eq!(val, None); + + let mut file = File::create(&config_path).unwrap(); + file.write_all( + r" +log_: : :level: warn # invalid yaml +foo: bar" + .as_bytes(), + ) + .unwrap(); + let val = read_path_in_file(&config_path, "foo"); + assert_eq!(val, None); + } + + #[test] + fn test_lossy_parse_to_filter_with_default() { + let log_level = LogLevel(Some("info".to_string())); + let filter = log_level.lossy_parse_to_filter_with_default("warn"); + assert_eq!(filter.to_string(), "info"); + let filter = + LogLevel(Some("adsdas".to_string())).lossy_parse_to_filter_with_default("warn"); + assert_eq!(filter.to_string(), "adsdas=trace"); + + let default_filter = LogLevel(None).lossy_parse_to_filter_with_default("warn"); + assert_eq!(default_filter.to_string(), "warn"); + } +}