Merge pull request #23 from automl/feature/pseudo-randomness

Feature/pseudo randomness

Author: mwever

CHANGELOG.md

@@ -1,3 +1,7 @@
+# v0.0.4
+- Added pseudorandomization
+- Added index-specific approximation
+
 # v0.0.3
 - Added multi-baseline ablation game. This game computes ablation paths with respect to multiple baseline configurations and aggregates values for different paths via mean, min, max or variance.
 - Added waterfall plots to the HyperSHAP interface.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hypershap"
-version = "0.0.2"
+version = "0.0.4"
 description = "HyperSHAP is a post-hoc explanation method for hyperparameter optimization."
 authors = [{ name = "Marcel Wever", email = "[email protected]" }]
 readme = "README.md"

src/hypershap/surrogate_model.py

@@ -182,6 +182,7 @@ def __init__(
         config_space: ConfigurationSpace,
         data: list[tuple[Configuration, float]],
         base_model: BaseEstimator | None = None,
+        seed: int | None = 0,
     ) -> None:
         """Initialize the DataBasedSurrogateModel with data and an optional base model.
 
@@ -191,13 +192,14 @@ def __init__(
                   is a tuple of (Configuration, float).
             base_model: The base model to be used for fitting the surrogate model.
                         If None, a RandomForestRegressor is used.
+            seed: The random seed for pseudo-randomization of the surrogate model. Defaults to 0.
 
         """
         train_x = np.array([obs[0].get_array() for obs in data])
         train_y = np.array([obs[1] for obs in data])
 
         if base_model is None:
-            base_model = RandomForestRegressor()
+            base_model = RandomForestRegressor(random_state=seed)
 
         pipeline = cast("SklearnRegressorProtocol", base_model)
         pipeline.fit(train_x, train_y)

src/hypershap/task.py

@@ -17,6 +17,8 @@
 
     from hypershap import ConfigSpaceSearcher
 
+from copy import deepcopy
+
 from sklearn.ensemble import RandomForestRegressor
 
 from hypershap.surrogate_model import DataBasedSurrogateModel, ModelBasedSurrogateModel, SurrogateModel
@@ -132,6 +134,7 @@ def from_function(
         function: Callable[[Configuration], float],
         n_samples: int = 1_000,
         base_model: BaseEstimator | None = None,
+        seed: int | None = 0,
     ) -> ExplanationTask:
         """Create an ExplanationTask from a function that evaluates configurations.
 
@@ -140,17 +143,21 @@ def from_function(
             function: A callable that takes a configuration and returns its performance.
             n_samples: The number of configurations to sample for training the surrogate model. Defaults to 1000.
             base_model: The base model to use for training the surrogate model. Defaults to RandomForestRegressor.
+            seed: The seed for the random number generator, it is used to seed a deep copy of the config space.
 
         Returns:
             An ExplanationTask instance.
 
         """
-        samples: list[Configuration] = config_space.sample_configuration(n_samples)
+        cs = deepcopy(config_space)
+        if seed is not None:
+            cs.seed(seed)
+        samples: list[Configuration] = cs.sample_configuration(n_samples)
         values: list[float] = [function(config) for config in samples]
         data: list[tuple[Configuration, float]] = list(zip(samples, values, strict=False))
-        base_model = base_model if base_model is not None else RandomForestRegressor()
+        base_model = base_model if base_model is not None else RandomForestRegressor(random_state=seed)
 
-        return ExplanationTask.from_data(config_space=config_space, data=data, base_model=base_model)
+        return ExplanationTask.from_data(config_space=cs, data=data, base_model=base_model)
 
     @staticmethod
     def from_function_multidata(

src/hypershap/utils.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from copy import deepcopy
 from enum import Enum
 from typing import TYPE_CHECKING
 
@@ -83,6 +84,7 @@ def __init__(
         explanation_task: BaselineExplanationTask,
         mode: Aggregation = Aggregation.MAX,
         n_samples: int = 10_000,
+        seed: int | None = 0,
     ) -> None:
         """Initialize the random configuration space searcher.
 
@@ -91,11 +93,14 @@ def __init__(
                 space and surrogate model.
             mode: The aggregation mode for performance values.
             n_samples: The number of configurations to sample.
+            seed: The random seed for sampling configurations from the config space.
 
         """
         super().__init__(explanation_task, mode=mode)
-
-        sampled_configurations = self.explanation_task.config_space.sample_configuration(size=n_samples)
+        cs = deepcopy(explanation_task.config_space)
+        if seed is not None:
+            cs.seed(seed)
+        sampled_configurations = cs.sample_configuration(size=n_samples)
         self.random_sample = np.array([config.get_array() for config in sampled_configurations])
 
         # cache coalition values to ensure monotonicity for min/max

tests/test_extended_settings.py

@@ -26,6 +26,14 @@ def test_large_ablation(large_base_et: ExplanationTask) -> None:
     hypershap.ablation(comparison, baseline)
 
 
+def test_large_ablation_kernelshap(large_base_et: ExplanationTask) -> None:
+    """Test HyperSHAP with large config space."""
+    baseline = large_base_et.config_space.sample_configuration()
+    comparison = large_base_et.config_space.sample_configuration()
+    hypershap = HyperSHAP(explanation_task=large_base_et, approximation_budget=2**7)
+    hypershap.ablation(comparison, baseline, index="k-SII")
+
+
 def test_multi_data_ablation(
     multi_data_baseline_config: Configuration,
     multi_data_config_space: ConfigurationSpace,