Merge pull request #34 from automl/improvement/monotonicity_conditions_fallback

Improvement/monotonicity conditions fallback

Author: mwever

CHANGELOG.md

@@ -1,3 +1,9 @@
+# v0.0.6
+
+## Improvements
+- Added fallback for configuration spaces with conditions resulting in all configurations being filtered out.
+- Added caching and a function in RandomConfigSpaceSearcher to ensure monotonicity of the value function.
+
 # v0.0.5
 
 ## Features

src/hypershap/utils.py

@@ -161,8 +161,67 @@ def search(self, coalition: np.ndarray) -> float:
                 )
 
             # predict performance values with the help of the surrogate model for the filtered configurations
-            vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples))
+            if len(filtered_samples) > 0:
+                vals: np.ndarray = np.array(
+                    self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples),
+                )
+            else:
+                logger.warning(
+                    "WARNING: After filtering for conditions, no configurations were left, thus, using baseline value.",
+                )
+                vals = np.array([self.search(np.array([False] * len(coalition)))])
         else:
             vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample))
 
-        return evaluate_aggregation(self.mode, vals)
+        # determine the final, aggregated value of the coalition
+        res = evaluate_aggregation(self.mode, vals)
+
+        # in case we are maximizing or minimizing, ensure that the value function is monotone
+        if self.mode in (Aggregation.MAX, Aggregation.MIN):
+            res = self._ensure_monotonicity(coalition, res)
+
+        # cache the coalition's value
+        self.coalition_cache[str(coalition.tolist())] = res
+
+        return res
+
+    def _ensure_monotonicity(self, coalition: np.ndarray, value: float) -> float:
+        """Ensure that the value function is monotonically increasing/decreasing depending on whether we want to maximize or minimize respectively.
+
+        Args:
+            coalition: The current coalition.
+            value: The value of the coalition as determined by searching.
+
+        Returns: The monotonicity-ensured value of the coalition.
+
+        """
+        monotone_value = value
+        checked_one = False
+
+        for i in range(len(coalition)):
+            if coalition[i]:  # check whether the entry is True and set it to False to check for a cached result
+                temp_coalition = coalition.copy()
+                temp_coalition[i] = False
+                if str(temp_coalition.tolist()) in self.coalition_cache:
+                    checked_one = True
+                    monotone_value = evaluate_aggregation(
+                        self.mode,
+                        np.array([
+                            monotone_value,
+                            self.coalition_cache[str(temp_coalition.tolist())],
+                        ]),
+                    )
+
+        if not checked_one and coalition.any():
+            logger.warning(
+                "Could not ensure monotonicity as none of the coalitions with one player less has been cached so far.",
+            )
+
+        if value < monotone_value:  # pragma: no cover
+            logger.debug(
+                "Ensured monotonicity with a sub-coalition's value. Increased the value of the current coalition from %s to %s.",
+                value,
+                monotone_value,
+            )
+
+        return monotone_value

tests/fixtures/simple_setup.py

@@ -3,7 +3,13 @@
 from __future__ import annotations
 
 import pytest
-from ConfigSpace import Configuration, ConfigurationSpace, LessThanCondition, UniformFloatHyperparameter
+from ConfigSpace import (
+    Configuration,
+    ConfigurationSpace,
+    GreaterThanCondition,
+    LessThanCondition,
+    UniformFloatHyperparameter,
+)
 
 from hypershap import ExplanationTask
 
@@ -88,10 +94,34 @@ def simple_cond_config_space() -> ConfigurationSpace:
     return config_space
 
 
+@pytest.fixture(scope="session")
+def simple_act_config_space() -> ConfigurationSpace:
+    """Return a simple config space with activation structure for testing."""
+    config_space = ConfigurationSpace()
+    config_space.seed(42)
+
+    a = UniformFloatHyperparameter("a", 0, 1, 0)
+    b = UniformFloatHyperparameter("b", 0, 1, 0)
+    config_space.add(a)
+    config_space.add(b)
+
+    config_space.add(GreaterThanCondition(b, a, 0.3))
+    return config_space
+
+
 @pytest.fixture(scope="session")
 def simple_cond_base_et(
     simple_cond_config_space: ConfigurationSpace,
     simple_blackbox_function: SimpleBlackboxFunction,
 ) -> ExplanationTask:
     """Return a base explanation task for the simple setup with conditions."""
     return ExplanationTask.from_function(simple_cond_config_space, simple_blackbox_function.evaluate)
+
+
+@pytest.fixture(scope="session")
+def simple_act_base_et(
+    simple_act_config_space: ConfigurationSpace,
+    simple_blackbox_function: SimpleBlackboxFunction,
+) -> ExplanationTask:
+    """Return a base explanation task for the simple setup with conditions."""
+    return ExplanationTask.from_function(simple_act_config_space, simple_blackbox_function.evaluate)

tests/test_extended_settings.py

@@ -79,3 +79,10 @@ def test_tunability_with_conditions(simple_cond_base_et: ExplanationTask) -> Non
     hypershap = HyperSHAP(simple_cond_base_et)
     iv = hypershap.tunability(simple_cond_base_et.config_space.get_default_configuration())
     assert iv is not None, "Interaction values should not be none."
+
+
+def test_tunability_with_activation_structures(simple_act_base_et: ExplanationTask) -> None:
+    """Test the tunability task with a configuration space that has conditions."""
+    hypershap = HyperSHAP(simple_act_base_et)
+    iv = hypershap.tunability(simple_act_base_et.config_space.get_default_configuration())
+    assert iv is not None, "Interaction values should not be none."

tests/test_utils.py

@@ -12,6 +12,8 @@
     from hypershap import ExplanationTask
     from tests.fixtures.simple_setup import SimpleBlackboxFunction
 
+from ConfigSpace import Configuration
+
 from hypershap.task import BaselineExplanationTask
 from hypershap.utils import Aggregation, RandomConfigSpaceSearcher, evaluate_aggregation
 
@@ -144,3 +146,31 @@ def test_evaluate_aggregation() -> None:
     assert evaluate_aggregation(Aggregation.MAX, vals) == AGG_LIST[2]
     assert evaluate_aggregation(Aggregation.AVG, vals) == np.array(AGG_LIST).mean()
     assert abs(evaluate_aggregation(Aggregation.VAR, vals) - np.array(AGG_LIST).var()) < EPSILON
+
+
+def test_fallback_unfulfilled_conditions(simple_act_base_et: ExplanationTask) -> None:
+    """Test the fallback strategy when no configurations are left in random sample after filtering for conditions."""
+    bet = BaselineExplanationTask(
+        simple_act_base_et.config_space,
+        simple_act_base_et.surrogate_model,
+        simple_act_base_et.config_space.get_default_configuration(),
+    )
+    rcss = RandomConfigSpaceSearcher(bet)
+    rcss.random_sample = np.array([
+        Configuration(
+            configuration_space=simple_act_base_et.config_space,
+            values={
+                "a": 0.4,
+                "b": 0.1,
+            },
+        ).get_array(),
+        Configuration(
+            configuration_space=simple_act_base_et.config_space,
+            values={
+                "a": 0.5,
+                "b": 0.1,
+            },
+        ).get_array(),
+    ])
+    value = rcss.search(np.array([False, True]))
+    assert value is not None