Merge pull request #30 from automl/feature/constraints

Constraints and Auto-Configure Approximators

Author: mwever

pyproject.toml

@@ -17,11 +17,12 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-    "shapiq>=1.2.3",
+    "shapiq==1.4.1",
     "numpy>=2.2.6",
     "scikit-learn>=1.7.1",
     "matplotlib>=3.10.5",
-    "networkx>=3.4.2"
+    "networkx>=3.4.2",
+    "ConfigSpace>=1.2.1",
 ]
 
 [project.urls]
@@ -50,8 +51,6 @@ docs = [
 ]
 
 dev = [
-    "shapiq>=1.2.0",
-    "ConfigSpace>=1.2.1",
     "numpy>=1.26.4",
     "tox-uv>=1.11.3",
     "deptry>=0.23.0",

src/hypershap/hypershap.py

@@ -12,6 +12,7 @@
 
 if TYPE_CHECKING:
     from ConfigSpace import Configuration
+    from shapiq import ValidApproximationIndices
 
     from hypershap.utils import ConfigSpaceSearcher
 
@@ -20,7 +21,8 @@
 import matplotlib.pyplot as plt
 import networkx as nx
 import numpy as np
-from shapiq import SHAPIQ, ExactComputer, InteractionValues, KernelSHAPIQ
+from shapiq import ExactComputer, InteractionValues
+from shapiq.explainer.configuration import setup_approximator_automatically
 
 from hypershap.games import (
     AblationGame,
@@ -66,13 +68,13 @@ class HyperSHAP:
         __init__(explanation_task: ExplanationTask):
             Initializes the HyperSHAP instance with an explanation task.
 
-        ablation(config_of_interest: Configuration, baseline_config: Configuration, index: str = "FSII", order: int = 2) -> InteractionValues:
+        ablation(config_of_interest: Configuration, baseline_config: Configuration, index: ValidApproximationIndices = "FSII", order: int = 2) -> InteractionValues:
             Computes and returns the interaction values for ablation analysis.
 
-        tunability(baseline_config: Configuration | None, index: str = "FSII", order: int = 2) -> InteractionValues:
+        tunability(baseline_config: Configuration | None, index: ValidApproximationIndices = "FSII", order: int = 2) -> InteractionValues:
             Computes and returns the interaction values for tunability analysis.
 
-        optimizer_bias(optimizer_of_interest: ConfigSpaceSearcher, optimizer_ensemble: list[ConfigSpaceSearcher], index: str = "FSII", order: int = 2) -> InteractionValues:
+        optimizer_bias(optimizer_of_interest: ConfigSpaceSearcher, optimizer_ensemble: list[ConfigSpaceSearcher], index: ValidApproximationIndices = "FSII", order: int = 2) -> InteractionValues:
             Computes and returns the interaction values for optimizer bias analysis.
 
         plot_si_graph(interaction_values: InteractionValues | None = None, save_path: str | None = None):
@@ -116,19 +118,22 @@ def __init__(
         )
         self.verbose = verbose
 
-    def __get_interaction_values(self, game: AbstractHPIGame, index: str = "FSII", order: int = 2) -> InteractionValues:
+    def __get_interaction_values(
+        self,
+        game: AbstractHPIGame,
+        index: ValidApproximationIndices = "FSII",
+        order: int = 2,
+        seed: int | None = 0,
+    ) -> InteractionValues:
         if game.n_players <= EXACT_MAX_HYPERPARAMETERS:
             # instantiate exact computer if number of hyperparameters is small enough
             ec = ExactComputer(n_players=game.get_num_hyperparameters(), game=game)  # pyright: ignore
 
             # compute interaction values with the given index and order
             interaction_values = ec(index=index, order=order)
         else:
-            # instantiate kernel
-            if index == "FSII":
-                approx = SHAPIQ(n=game.n_players, max_order=2, index=index)
-            else:
-                approx = KernelSHAPIQ(n=game.n_players, max_order=2, index=index)
+            # instantiate approximator
+            approx = setup_approximator_automatically(index, order, game.n_players, seed)
 
             # approximate interaction values with the given index and order
             interaction_values = approx(budget=self.approximation_budget, game=game)
@@ -142,15 +147,15 @@ def ablation(
         self,
         config_of_interest: Configuration,
         baseline_config: Configuration,
-        index: str = "FSII",
+        index: ValidApproximationIndices = "FSII",
         order: int = 2,
     ) -> InteractionValues:
         """Compute and return the interaction values for ablation analysis.
 
         Args:
             config_of_interest (Configuration): The configuration of interest.
             baseline_config (Configuration): The baseline configuration.
-            index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
+            index (ValidApproximationIndices, optional): The index to use for computing interaction values. Defaults to "FSII".
             order (int, optional): The order of the interaction values. Defaults to 2.
 
         Returns:
@@ -191,7 +196,7 @@ def ablation_multibaseline(
         config_of_interest: Configuration,
         baseline_configs: list[Configuration],
         aggregation: Aggregation = Aggregation.AVG,
-        index: str = "FSII",
+        index: ValidApproximationIndices = "FSII",
         order: int = 2,
     ) -> InteractionValues:
         """Compute and return the interaction values for multi-baseline ablation analysis.
@@ -200,7 +205,7 @@ def ablation_multibaseline(
             config_of_interest (Configuration): The configuration of interest.
             baseline_configs (list[Configuration]): The list of baseline configurations.
             aggregation (Aggregation): The aggregation method to use for computing interaction values.
-            index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
+            index (ValidApproximationIndices, optional): The index to use for computing interaction values. Defaults to "FSII".
             order (int, optional): The order of the interaction values. Defaults to 2.
 
         Returns:
@@ -240,7 +245,7 @@ def ablation_multibaseline(
     def tunability(
         self,
         baseline_config: Configuration | None = None,
-        index: str = "FSII",
+        index: ValidApproximationIndices = "FSII",
         order: int = 2,
         n_samples: int = 10_000,
         seed: int | None = 0,
@@ -298,7 +303,7 @@ def tunability(
     def sensitivity(
         self,
         baseline_config: Configuration | None = None,
-        index: str = "FSII",
+        index: ValidApproximationIndices = "FSII",
         order: int = 2,
         n_samples: int = 10_000,
         seed: int | None = 0,
@@ -356,7 +361,7 @@ def sensitivity(
     def mistunability(
         self,
         baseline_config: Configuration | None = None,
-        index: str = "FSII",
+        index: ValidApproximationIndices = "FSII",
         order: int = 2,
         n_samples: int = 10_000,
         seed: int | None = 0,
@@ -365,7 +370,7 @@ def mistunability(
 
         Args:
             baseline_config (Configuration | None, optional): The baseline configuration. Defaults to None.
-            index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
+            index (ValidApproximationIndices, optional): The index to use for computing interaction values. Defaults to "FSII".
             order (int, optional): The order of the interaction values. Defaults to 2.
             n_samples (int, optional): The number of samples to use for simulating HPO. Defaults to 10_000.
             seed (int, optiona): The random seed for simulating HPO. Defaults to 0.
@@ -414,15 +419,15 @@ def optimizer_bias(
         self,
         optimizer_of_interest: ConfigSpaceSearcher,
         optimizer_ensemble: list[ConfigSpaceSearcher],
-        index: str = "FSII",
+        index: ValidApproximationIndices = "FSII",
         order: int = 2,
     ) -> InteractionValues:
         """Compute and return the interaction values for optimizer bias analysis.
 
         Args:
             optimizer_of_interest (ConfigSpaceSearcher): The optimizer of interest.
             optimizer_ensemble (list[ConfigSpaceSearcher]): The ensemble of optimizers.
-            index (str, optional): The index to use for computing interaction values. Defaults to "FSII".
+            index (ValidApproximationIndices, optional): The index to use for computing interaction values. Defaults to "FSII".
             order (int, optional): The order of the interaction values. Defaults to 2.
 
         Returns:

src/hypershap/utils.py

@@ -5,15 +5,23 @@
 
 from __future__ import annotations
 
+import logging
 from abc import ABC, abstractmethod
 from copy import deepcopy
 from enum import Enum
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from hypershap.task import BaselineExplanationTask
-
 import numpy as np
+from ConfigSpace.exceptions import (
+    ActiveHyperparameterNotSetError,
+    ForbiddenValueError,
+    IllegalVectorizedValueError,
+    InactiveHyperparameterSetError,
+)
+
+logger = logging.getLogger(__name__)
 
 
 class Aggregation(Enum):
@@ -106,6 +114,20 @@ def __init__(
         # cache coalition values to ensure monotonicity for min/max
         self.coalition_cache = {}
 
+    def _is_valid(self, config: np.ndarray) -> bool:
+        """Check whether a configuration is valid with respect to conditions of the configuration space."""
+        try:
+            self.explanation_task.config_space.check_configuration_vector_representation(config)
+        except (
+            ActiveHyperparameterNotSetError,
+            IllegalVectorizedValueError,
+            InactiveHyperparameterSetError,
+            ForbiddenValueError,
+        ):
+            return False
+        else:
+            return True
+
     def search(self, coalition: np.ndarray) -> float:
         """Search the configuration space based on the coalition.
 
@@ -125,6 +147,22 @@ def search(self, coalition: np.ndarray) -> float:
         column_index = np.where(blind_coalition)
         temp_random_sample[:, column_index] = self.explanation_task.baseline_config.get_array()[column_index]
 
-        # predict performance values with the help of the surrogate model
-        vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample))
+        # in case of conditions in the config space, it might happen that through blinding hyperparameter values
+        # configurations might become invalid and those should not be considered for calculating vals
+        if len(self.explanation_task.config_space.conditions) > 0:
+            # filter invalid configurations
+            validity = np.apply_along_axis(self._is_valid, axis=1, arr=temp_random_sample)
+            filtered_samples = temp_random_sample[validity]
+
+            if len(filtered_samples) < 0.05 * len(temp_random_sample):  # pragma: no cover
+                logger.warning(
+                    "WARNING: Due to blinding less than 5% of the samples in the random search remain valid. "
+                    "Consider increasing the sampling budget of the random search.",
+                )
+
+            # predict performance values with the help of the surrogate model for the filtered configurations
+            vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(filtered_samples))
+        else:
+            vals: np.ndarray = np.array(self.explanation_task.get_single_surrogate_model().evaluate(temp_random_sample))
+
         return evaluate_aggregation(self.mode, vals)

tests/fixtures/simple_setup.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 import pytest
-from ConfigSpace import Configuration, ConfigurationSpace, UniformFloatHyperparameter
+from ConfigSpace import Configuration, ConfigurationSpace, LessThanCondition, UniformFloatHyperparameter
 
 from hypershap import ExplanationTask
 
@@ -41,7 +41,7 @@ def evaluate(self, x: Configuration) -> float:
         Returns: The value of the configuration.
 
         """
-        return self.value(x["a"], x["b"])
+        return self.value(x["a"], x.get("b", 0))
 
     def value(self, a: float, b: float) -> float:
         """Evaluate the value of a configuration.
@@ -71,3 +71,27 @@ def simple_base_et(
 ) -> ExplanationTask:
     """Return a base explanation task for the simple setup."""
     return ExplanationTask.from_function(simple_config_space, simple_blackbox_function.evaluate)
+
+
+@pytest.fixture(scope="session")
+def simple_cond_config_space() -> ConfigurationSpace:
+    """Return a simple config space with conditions for testing."""
+    config_space = ConfigurationSpace()
+    config_space.seed(42)
+
+    a = UniformFloatHyperparameter("a", 0, 1, 0)
+    b = UniformFloatHyperparameter("b", 0, 1, 0)
+    config_space.add(a)
+    config_space.add(b)
+
+    config_space.add(LessThanCondition(b, a, 0.3))
+    return config_space
+
+
+@pytest.fixture(scope="session")
+def simple_cond_base_et(
+    simple_cond_config_space: ConfigurationSpace,
+    simple_blackbox_function: SimpleBlackboxFunction,
+) -> ExplanationTask:
+    """Return a base explanation task for the simple setup with conditions."""
+    return ExplanationTask.from_function(simple_cond_config_space, simple_blackbox_function.evaluate)

tests/test_extended_settings.py

@@ -72,3 +72,10 @@ def test_multi_data_sensitivity(multi_data_baseline_config: Configuration, hyper
     """Test the multi-data sesntivity task."""
     iv = hypershap_inst.sensitivity(baseline_config=multi_data_baseline_config)
     assert iv is not None, "Interaction values should not be none."
+
+
+def test_tunability_with_conditions(simple_cond_base_et: ExplanationTask) -> None:
+    """Test the tunability task with a configuration space that has conditions."""
+    hypershap = HyperSHAP(simple_cond_base_et)
+    iv = hypershap.tunability(simple_cond_base_et.config_space.get_default_configuration())
+    assert iv is not None, "Interaction values should not be none."