Merge pull request #557 from sunbeam-labs/codex/simplify-logging-setup-in-sunbeam

Use config-defined log file

Author: Ulthran

sunbeam/configs/default_config.yml

@@ -5,6 +5,7 @@ all:
   samplelist_fp: "samples.csv"
   paired_end: true
   version: "{SB_VERSION}"
+  log_fp: "sunbeam.log"
 
 # Quality control
 qc:

sunbeam/logging.py

@@ -25,7 +25,7 @@ def get_sunbeam_logger() -> logging.Logger:
     return logger
 
 
-def get_pipeline_logger(log_file: Path = None) -> logging.Logger:
+def get_pipeline_logger(log_fp: Path = None) -> logging.Logger:
     """Sets up logging for the main pipeline entry point."""
     logger = logging.getLogger("sunbeam.pipeline")
     logger.setLevel(logging.DEBUG)
@@ -38,11 +38,11 @@ def get_pipeline_logger(log_file: Path = None) -> logging.Logger:
         ch.setFormatter(ConditionalLevelFormatter())
 
         # File handler
-        if log_file is None:
+        if log_fp is None:
             raise ValueError(
-                "log_file is None but the logger hasn't been initialized with a file handler yet"
+                "log_fp is None but the logger hasn't been initialized with a file handler yet"
             )
-        fh = logging.FileHandler(log_file, mode="w")
+        fh = logging.FileHandler(log_fp, mode="w")
         fh.setLevel(logging.DEBUG)
         fh.setFormatter(logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s"))
 

sunbeam/scripts/run.py

@@ -1,11 +1,11 @@
 import argparse
-import datetime
 import os
 import sys
 from pathlib import Path
 from snakemake.cli import main as snakemake_main
 from sunbeam import __version__
 from sunbeam.logging import get_pipeline_logger
+from sunbeam.project import SunbeamConfig
 
 
 def main(argv: list[str] = sys.argv):
@@ -14,19 +14,17 @@ def main(argv: list[str] = sys.argv):
     args, remaining = parser.parse_known_args(argv)
 
     profile = Path(args.profile).resolve()
+    configfile = Path(profile) / "sunbeam_config.yml"
 
-    log_file = args.log_file
-    if not log_file:
-        log_file = (
-            profile
-            / f"sunbeam_{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log"
-        )
+    sc = SunbeamConfig.from_file(configfile)
+    Cfg = sc.resolved_paths()
+    log_fp = Cfg["all"]["log_fp"]
 
     # From here on everything is considered part of the "pipeline"
     # This means all logs are handled by the pipeline logger (or pipeline extension loggers)
     # You could argue it would make more sense to start this at the actual snakemake call
     # but this way we can log some relevant setup information that might be useful on post-mortem analysis
-    logger = get_pipeline_logger(log_file)
+    logger = get_pipeline_logger(log_fp)
 
     snakefile = Path(__file__).parent.parent / "workflow" / "Snakefile"
     if not snakefile.exists():
@@ -56,8 +54,6 @@ def main(argv: list[str] = sys.argv):
 
     os.environ["SUNBEAM_DOCKER_TAG"] = args.docker_tag
 
-    configfile = Path(profile) / "sunbeam_config.yml"
-
     snakemake_args = [
         "--snakefile",
         str(snakefile),
@@ -128,10 +124,4 @@ def main_parser():
         default=__version__,
         help="The tag to use when pulling docker images for the core pipeline environments, defaults to sunbeam's current version, a good alternative is 'latest' for the latest stable release",
     )
-    parser.add_argument(
-        "--log_file",
-        default=None,
-        help="Path to a file where the pipeline log will be written. If not specified, logs will be written to `sunbeam_<current_datetime>.log` under the project directory.",
-    )
-
     return parser

sunbeam/workflow/Snakefile

@@ -15,14 +15,12 @@ from sunbeam.logging import get_pipeline_logger, get_extension_logger
 from sunbeam.project import SampleList, SunbeamConfig, output_subdir
 from sunbeam.project.post import compile_benchmarks
 
-
-logger = get_pipeline_logger()
-
 MIN_MEM_MB = int(os.getenv("SUNBEAM_MIN_MEM_MB", 8000))
 MIN_RUNTIME = int(os.getenv("SUNBEAM_MIN_RUNTIME", 15))
 
 sc = SunbeamConfig(config)
 Cfg = sc.resolved_paths()
+logger = get_pipeline_logger(Cfg["all"]["log_fp"])
 logger.debug(f"Sunbeam configuration: {Cfg}")
 
 # Load extensions

tests/unit/test_sunbeam_config.py

@@ -39,6 +39,7 @@ def test_config_from_template(tmp_path):
     sc = SunbeamConfig.from_template(template_fp, root_fp, ext_dir)
     assert sc.config["all"]["root"] == str(root_fp.resolve())
     assert sc.config["all"]["version"] == __version__
+    assert sc.config["all"]["log_fp"] == "sunbeam.log"
 
 
 def test_config_with_extension(test_extension):
@@ -57,13 +58,17 @@ def test_config_from_file(tmp_path):
     all:
         root: /path/to/root
         version: 1.0.0
+        log_fp: sunbeam.log
     """
     config_fp = tmp_path / "config.yml"
     with open(config_fp, "w") as f:
         f.write(config_str)
 
     sc = SunbeamConfig.from_file(config_fp)
     assert sc.config["all"]["root"] == "/path/to/root"
+    assert sc.config["all"]["log_fp"] == "sunbeam.log"
+    res = sc.resolved_paths()
+    assert res["all"]["log_fp"] == Path("/path/to/root") / "sunbeam.log"
 
 
 def test_config_to_file(tmp_path):