Source code for optuna.importance._fanova._evaluator

from collections import OrderedDict
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional

import numpy

from optuna._transform import _SearchSpaceTransform
from optuna.importance._base import _get_distributions
from optuna.importance._base import BaseImportanceEvaluator
from optuna.importance._fanova._fanova import _Fanova
from import Study
from optuna.trial import FrozenTrial
from optuna.trial import TrialState

[docs]class FanovaImportanceEvaluator(BaseImportanceEvaluator): """fANOVA importance evaluator. Implements the fANOVA hyperparameter importance evaluation algorithm in `An Efficient Approach for Assessing Hyperparameter Importance <>`_. Given a study, fANOVA fits a random forest regression model that predicts the objective value given a parameter configuration. The more accurate this model is, the more reliable the importances assessed by this class are. .. note:: Requires the `sklearn <>`_ Python package. .. note:: Pairwise and higher order importances are not supported through this class. They can be computed using :class:`~optuna.importance._fanova._fanova._Fanova` directly but is not recommended as interfaces may change without prior notice. .. note:: The performance of fANOVA depends on the prediction performance of the underlying random forest model. In order to obtain high prediction performance, it is necessary to cover a wide range of the hyperparameter search space. It is recommended to use an exploration-oriented sampler such as :class:`~optuna.samplers.RandomSampler`. .. note:: For how to cite the original work, please refer to Args: n_trees: The number of trees in the forest. max_depth: The maximum depth of the trees in the forest. seed: Controls the randomness of the forest. For deterministic behavior, specify a value other than :obj:`None`. """ def __init__( self, *, n_trees: int = 64, max_depth: int = 64, seed: Optional[int] = None ) -> None: self._evaluator = _Fanova( n_trees=n_trees, max_depth=max_depth, min_samples_split=2, min_samples_leaf=1, seed=seed, )
[docs] def evaluate( self, study: Study, params: Optional[List[str]] = None, *, target: Optional[Callable[[FrozenTrial], float]] = None, ) -> Dict[str, float]: if target is None and study._is_multi_objective(): raise ValueError( "If the `study` is being used for multi-objective optimization, " "please specify the `target`. For example, use " "`target=lambda t: t.values[0]` for the first objective value." ) distributions = _get_distributions(study, params) if len(distributions) == 0: return OrderedDict() trials = [] for trial in study.trials: if trial.state != TrialState.COMPLETE: continue if any(name not in trial.params for name in distributions.keys()): continue trials.append(trial) trans = _SearchSpaceTransform(distributions, transform_log=False, transform_step=False) n_trials = len(trials) trans_params = numpy.empty((n_trials, trans.bounds.shape[0]), dtype=numpy.float64) trans_values = numpy.empty(n_trials, dtype=numpy.float64) for trial_idx, trial in enumerate(trials): trans_params[trial_idx] = trans.transform(trial.params) trans_values[trial_idx] = trial.value if target is None else target(trial) trans_bounds = trans.bounds column_to_encoded_columns = trans.column_to_encoded_columns if trans_params.size == 0: # `params` were given but as an empty list. return OrderedDict() # Many (deep) copies of the search spaces are required during the tree traversal and using # Optuna distributions will create a bottleneck. # Therefore, search spaces (parameter distributions) are represented by a single # `numpy.ndarray`, coupled with a list of flags that indicate whether they are categorical # or not. evaluator = self._evaluator X=trans_params, y=trans_values, search_spaces=trans_bounds, column_to_encoded_columns=column_to_encoded_columns, ) importances = {} for i, name in enumerate(distributions.keys()): importance, _ = evaluator.get_importance((i,)) importances[name] = importance total_importance = sum(importances.values()) for name in importances: importances[name] /= total_importance sorted_importances = OrderedDict( reversed( sorted(importances.items(), key=lambda name_and_importance: name_and_importance[1]) ) ) return sorted_importances