Test all metadata (openproblems-bio#813)

scottgigante-immunai · web-flow · commit 0401028f9882 · 2023-02-02T14:07:52.000-05:00
* make sure metadata tests run

* transfer meta

* test all metadata

* fix docker

* fix path

* undo rename

* fix

* separate metadata tests

* bump test coverage
diff --git a/.coveragerc b/.coveragerc
@@ -1,8 +1,12 @@
 [run]
 parallel = true
 branch = true
-source = openproblems
-omit = */__init__.py
+source =
+    openproblems
+    test
+omit =
+    */__init__.py
+    test/utils/*.py
 
 [report]
 exclude_lines =
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -313,13 +313,13 @@ jobs:
       if: "needs.build_images.result == 'skipped'"
       run: |
         cd workflow
-        snakemake -j $(nproc) docker_pull
+        snakemake -j $(nproc) docker
         cd ..
 
     - name: Run tests
       timeout-minutes: 60
       run: |
-        pytest --cov=openproblems --cov-report=xml -vv --durations=15 --tb=native -k "${{ matrix.tests }}"
+        pytest --cov=openproblems --cov=test --cov-report=xml -vv --durations=15 --tb=native -k "${{ matrix.tests }}"
         mkdir -p coverage
         mv coverage.xml "$(echo 'coverage_${{ matrix.tests }}.xml' | sed 's/[^a-z0-9\.]/_/g')"
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/ari.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/ari.py
@@ -14,12 +14,7 @@
 """
 
 
-@metric(
-    metric_name="ARI",
-    maximize=True,
-    paper_reference="luecken2022benchmarking",
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.ari.metadata)
 def ari(adata):
     from scanpy.pp import neighbors
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/graph_connectivity.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/graph_connectivity.py
@@ -20,12 +20,7 @@
 """
 
 
-@metric(
-    metric_name="Graph connectivity",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.graph_connectivity.metadata)
 def graph_connectivity(adata):
     from scanpy.pp import neighbors
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_f1.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/iso_label_f1.py
@@ -25,12 +25,7 @@
 """
 
 
-@metric(
-    metric_name="Isolated label F1",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.isolated_labels_f1.metadata)
 def isolated_labels_f1(adata):
     from scanpy.pp import neighbors
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/nmi.py b/openproblems/tasks/_batch_integration/batch_integration_embed/metrics/nmi.py
@@ -13,12 +13,7 @@
 """
 
 
-@metric(
-    metric_name="NMI",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.nmi.metadata)
 def nmi(adata):
     from scanpy.pp import neighbors
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/ari.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/ari.py
@@ -14,12 +14,7 @@
 """
 
 
-@metric(
-    metric_name="ARI",
-    maximize=True,
-    paper_reference="luecken2022benchmarking",
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.ari.metadata)
 def ari(adata):
     from scanpy.pp import neighbors
     from scanpy.tl import pca
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/cc_score.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/cc_score.py
@@ -19,12 +19,7 @@
 the preintegration variance contribution reduces the score."""
 
 
-@metric(
-    metric_name="Cell Cycle Score",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**embed_metrics.cc_score.metadata)
 def cc_score(adata, test=False):
     from scanpy.tl import pca
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/graph_connectivity.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/graph_connectivity.py
@@ -20,12 +20,7 @@
 """
 
 
-@metric(
-    metric_name="Graph connectivity",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.graph_connectivity.metadata)
 def graph_connectivity(adata):
     from scanpy.pp import neighbors
     from scanpy.tl import pca
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/iso_label_f1.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/iso_label_f1.py
@@ -25,12 +25,7 @@
 """
 
 
-@metric(
-    metric_name="Isolated label F1",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.isolated_labels_f1.metadata)
 def isolated_labels_f1(adata):
     from scanpy.pp import neighbors
     from scanpy.tl import pca
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/iso_label_sil.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/iso_label_sil.py
@@ -13,12 +13,7 @@
 """
 
 
-@metric(
-    metric_name="Isolated label Silhouette",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**embed_metrics.isolated_labels_sil.metadata)
 def isolated_labels_sil(adata):
     from scanpy.tl import pca
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/kBET.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/kBET.py
@@ -25,12 +25,7 @@
 """
 
 
-@metric(
-    metric_name="kBET",
-    paper_reference="bttner2018test",
-    maximize=True,
-    image="openproblems-r-extras",
-)
+@metric(**embed_metrics.kBET.metadata)
 def kBET(adata):
     from scanpy.tl import pca
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/nmi.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/nmi.py
@@ -13,12 +13,7 @@
 """
 
 
-@metric(
-    metric_name="NMI",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**graph_metrics.nmi.metadata)
 def nmi(adata):
     from scanpy.pp import neighbors
     from scanpy.tl import pca
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/pcr.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/pcr.py
@@ -16,12 +16,7 @@
 component."""
 
 
-@metric(
-    metric_name="PC Regression",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**embed_metrics.pcr.metadata)
 def pcr(adata):
     from scanpy.tl import pca
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/sil_batch.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/sil_batch.py
@@ -22,12 +22,7 @@
 Here, M is the set of unique cell labels."""
 
 
-@metric(
-    metric_name="Batch ASW",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**embed_metrics.silhouette_batch.metadata)
 def silhouette_batch(adata):
     from scanpy.tl import pca
 
diff --git a/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/silhouette.py b/openproblems/tasks/_batch_integration/batch_integration_feature/metrics/silhouette.py
@@ -10,12 +10,7 @@
 For information about the batch silhouette score, check sil_batch."""
 
 
-@metric(
-    metric_name="Silhouette",
-    paper_reference="luecken2022benchmarking",
-    maximize=True,
-    image="openproblems-r-pytorch",
-)
+@metric(**embed_metrics.silhouette.metadata)
 def silhouette(adata):
     from scanpy.tl import pca
 
diff --git a/openproblems/tasks/denoising/methods/dca.py b/openproblems/tasks/denoising/methods/dca.py
@@ -27,7 +27,7 @@ def _dca(adata, test=False, epochs=None):
 @method(
     method_name="DCA",
     paper_name="Single-cell RNA-seq denoising using a deep count autoencoder",
-    paper_reference="https://www.nature.com/articles/s41467-018-07931-2",
+    paper_reference="eraslan2019single",
     paper_year=2019,
     code_url="https://github.com/theislab/dca",
     image="openproblems-python-tensorflow",
diff --git a/openproblems/tasks/denoising/methods/magic.py b/openproblems/tasks/denoising/methods/magic.py
@@ -9,7 +9,7 @@
     method,
     paper_name="Recovering Gene Interactions from Single-Cell Data "
     "Using Data Diffusion",
-    paper_reference="https://doi.org/10.1016/j.cell.2018.05.061",
+    paper_reference="van2018recovering",
     paper_year=2018,
     code_url="https://github.com/KrishnaswamyLab/MAGIC",
     image="openproblems-python-extras",
diff --git a/scripts/install_renv.R b/scripts/install_renv.R
@@ -40,7 +40,7 @@ strip_comments <- function(remote) {
 }
 
 with_retries <- function(func,
-                         attempts = 5,
+                         attempts = 10,
                          sleep_once = 3,
                          sleep_multiple = 60,
                          backoff = 2,
diff --git a/test/test_core_metadata.py b/test/test_core_metadata.py
@@ -0,0 +1,87 @@
+import openproblems
+import openproblems.utils
+import parameterized
+import utils
+import utils.asserts
+import utils.cache
+import utils.git
+import utils.name
+
+DATASET_SUMMARY_MINLEN = 40
+DATASET_SUMMARY_MAXLEN = 1000
+
+
+@parameterized.parameterized.expand(
+    [(dataset,) for task in openproblems.TASKS for dataset in task.DATASETS],
+    name_func=utils.name.name_test,
+)
+def test_dataset_metadata(dataset):
+    """Test for existence of dataset metadata."""
+    assert hasattr(dataset, "metadata")
+    for attr in [
+        "dataset_name",
+        "data_url",
+        "data_reference",
+        "dataset_summary",
+        "image",
+    ]:
+        assert attr in dataset.metadata
+        assert dataset.metadata[attr] is not None
+
+    assert isinstance(dataset.metadata["dataset_name"], str)
+    assert isinstance(dataset.metadata["image"], str)
+    assert dataset.metadata["image"].startswith("openproblems")
+    assert isinstance(dataset.metadata["dataset_summary"], str)
+    assert len(dataset.metadata["dataset_summary"]) > DATASET_SUMMARY_MINLEN
+    assert len(dataset.metadata["dataset_summary"]) < DATASET_SUMMARY_MAXLEN
+    assert isinstance(dataset.metadata["data_url"], str)
+    assert utils.asserts.assert_url_accessible(dataset.metadata["data_url"])
+    assert isinstance(dataset.metadata["data_reference"], str)
+    assert utils.asserts.assert_valid_reference(dataset.metadata["data_reference"])
+
+
+@parameterized.parameterized.expand(
+    [(method,) for task in openproblems.TASKS for method in task.METHODS],
+    name_func=utils.name.name_test,
+)
+def test_method_metadata(method):
+    """Test for existence of method metadata."""
+    assert hasattr(method, "metadata")
+    for attr in [
+        "method_name",
+        "paper_name",
+        "paper_reference",
+        "paper_year",
+        "code_url",
+        "image",
+        "is_baseline",
+    ]:
+        assert attr in method.metadata
+
+    assert isinstance(method.metadata["image"], str)
+    assert method.metadata["image"].startswith("openproblems")
+    assert isinstance(method.metadata["method_name"], str)
+    assert isinstance(method.metadata["paper_name"], str)
+    assert isinstance(method.metadata["paper_year"], int)
+    assert isinstance(method.metadata["paper_reference"], str)
+    assert utils.asserts.assert_valid_reference(method.metadata["paper_reference"])
+    assert isinstance(method.metadata["code_url"], str)
+    assert utils.asserts.assert_url_accessible(method.metadata["code_url"])
+    assert isinstance(method.metadata["is_baseline"], bool)
+
+
+@parameterized.parameterized.expand(
+    [(metric,) for task in openproblems.TASKS for metric in task.METRICS],
+    name_func=utils.name.name_test,
+)
+def test_metric_metadata(metric):
+    """Test for existence of metric metadata."""
+    assert hasattr(metric, "metadata")
+    for attr in ["metric_name", "maximize", "image"]:
+        assert attr in metric.metadata
+    assert isinstance(metric.metadata["maximize"], bool)
+    assert isinstance(metric.metadata["metric_name"], str)
+    assert isinstance(metric.metadata["image"], str)
+    assert metric.metadata["image"].startswith("openproblems")
+    assert isinstance(metric.metadata["paper_reference"], str)
+    assert utils.asserts.assert_valid_reference(metric.metadata["paper_reference"])
diff --git a/test/test_core_utils.py b/test/test_core_utils.py
@@ -10,7 +10,7 @@ def test_temporary_version_missing():
     """Test temporary decorator behavior with missing version."""
 
     @openproblems.utils.temporary
-    def test_fn():
+    def test_fn():  # pragma: nocover
         pass
 
     np.testing.assert_raises_regex(
@@ -27,7 +27,7 @@ def test_temporary_version_future():
         temp_version = "{}.{}".format(version.major - 1, 0)
 
     @openproblems.utils.temporary(version=temp_version)
-    def test_fn():
+    def test_fn():  # pragma: nocover
         pass
 
     np.testing.assert_raises_regex(
diff --git a/test/test_task_1_load_data.py b/test/test_task_1_load_data.py
@@ -21,7 +21,7 @@
     skip_on_empty=True,
 )
 @utils.docker.docker_test(retries=2)
-def test_load_dataset(task_name, dataset_name, test, tempdir, image):
+def test_load_dataset(task_name, dataset_name, test, tempdir, image):  # pragma: nocover
     """Test loading and caching of a dataset."""
     import utils.asserts
     import utils.cache
diff --git a/test/test_task_2_datasets.py b/test/test_task_2_datasets.py
diff --git a/test/test_task_cell_cell_communication_source_target.py b/test/test_task_cell_cell_communication_source_target.py
diff --git a/test/test_task_dimensionality_reduction.py b/test/test_task_dimensionality_reduction.py
diff --git a/test/test_task_methods.py b/test/test_task_methods.py
diff --git a/test/test_task_metrics.py b/test/test_task_metrics.py
diff --git a/test/utils/asserts.py b/test/utils/asserts.py
diff --git a/test/utils/name.py b/test/utils/name.py
