From 5714caa37717d93103a243bb7501628900e5b434 Mon Sep 17 00:00:00 2001 From: dzhang Date: Mon, 12 Mar 2018 18:35:42 -0400 Subject: [PATCH 1/2] add test_weight_score argument to allow weighted cv --- sklearn/model_selection/_search.py | 32 +++++++++++++------ sklearn/model_selection/_validation.py | 43 +++++++++++++++++++------- 2 files changed, 53 insertions(+), 22 deletions(-) diff --git a/sklearn/model_selection/_search.py b/sklearn/model_selection/_search.py index d9e7d3166b8ff..ec488d6c1c781 100644 --- a/sklearn/model_selection/_search.py +++ b/sklearn/model_selection/_search.py @@ -273,7 +273,7 @@ def __len__(self): def fit_grid_point(X, y, estimator, parameters, train, test, scorer, - verbose, error_score='raise-deprecating', **fit_params): + verbose, error_score='raise-deprecating', test_score_weight=None, **fit_params): """Run fit on one set of parameters. Parameters @@ -318,6 +318,10 @@ def fit_grid_point(X, y, estimator, parameters, train, test, scorer, step, which will always raise the error. Default is 'raise' but from version 0.22 it will change to np.nan. + test_score_weight : string, optional, default: None + If specified, this is the sample weight key in the `fit_params` dict. + If there's no such key found, we will assume unweighted test score. + Returns ------- score : float @@ -337,7 +341,8 @@ def fit_grid_point(X, y, estimator, parameters, train, test, scorer, test, verbose, parameters, fit_params=fit_params, return_n_test_samples=True, - error_score=error_score) + error_score=error_score, + test_score_weight=test_score_weight) return scores, parameters, n_samples_test @@ -392,7 +397,8 @@ class BaseSearchCV(six.with_metaclass(ABCMeta, BaseEstimator, def __init__(self, estimator, scoring=None, fit_params=None, n_jobs=1, iid='warn', refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs', - error_score='raise-deprecating', return_train_score=True): + error_score='raise-deprecating', return_train_score=True, + test_score_weight=None): self.scoring = scoring self.estimator = estimator @@ -405,12 +411,13 @@ def __init__(self, estimator, scoring=None, self.pre_dispatch = pre_dispatch self.error_score = error_score self.return_train_score = return_train_score + self.test_score_weight = test_score_weight @property def _estimator_type(self): return self.estimator._estimator_type - def score(self, X, y=None): + def score(self, X, y=None, sample_weight=None): """Returns the score on the given data, if the estimator has been refit. This uses the score defined by ``scoring`` where provided, and the @@ -426,6 +433,9 @@ def score(self, X, y=None): Target relative to X for classification or regression; None for unsupervised learning. + sample_weight : array-like of shape = (n_samples), optional + Sample weights. + Returns ------- score : float @@ -436,7 +446,8 @@ def score(self, X, y=None): "and the estimator doesn't provide one %s" % self.best_estimator_) score = self.scorer_[self.refit] if self.multimetric_ else self.scorer_ - return score(self.best_estimator_, X, y) + score_kwgs = {} if sample_weight is None else {'sample_weight': sample_weight} + return score(self.best_estimator_, X, y, **score_kwgs) def _check_is_fitted(self, method_name): if not self.refit: @@ -636,7 +647,8 @@ def fit(self, X, y=None, groups=None, **fit_params): return_train_score=self.return_train_score, return_n_test_samples=True, return_times=True, return_parameters=False, - error_score=self.error_score) + error_score=self.error_score, + test_score_weight=self.test_score_weight) for parameters, (train, test) in product(candidate_params, cv.split(X, y, groups))) @@ -1088,12 +1100,12 @@ class GridSearchCV(BaseSearchCV): def __init__(self, estimator, param_grid, scoring=None, fit_params=None, n_jobs=1, iid='warn', refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs', error_score='raise-deprecating', - return_train_score="warn"): + return_train_score="warn", test_score_weight=None): super(GridSearchCV, self).__init__( estimator=estimator, scoring=scoring, fit_params=fit_params, n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score, - return_train_score=return_train_score) + return_train_score=return_train_score, test_score_weight=test_score_weight) self.param_grid = param_grid _check_param_grid(param_grid) @@ -1389,7 +1401,7 @@ class RandomizedSearchCV(BaseSearchCV): def __init__(self, estimator, param_distributions, n_iter=10, scoring=None, fit_params=None, n_jobs=1, iid='warn', refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs', random_state=None, - error_score='raise-deprecating', return_train_score="warn"): + error_score='raise-deprecating', return_train_score="warn", test_score_weight=None): self.param_distributions = param_distributions self.n_iter = n_iter self.random_state = random_state @@ -1397,7 +1409,7 @@ def __init__(self, estimator, param_distributions, n_iter=10, scoring=None, estimator=estimator, scoring=scoring, fit_params=fit_params, n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score, - return_train_score=return_train_score) + return_train_score=return_train_score, test_score_weight=test_score_weight) def _get_param_iterator(self): """Return ParameterSampler instance for the given distributions""" diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py index 03bf0c92c8b07..fcd26f27bdfb5 100644 --- a/sklearn/model_selection/_validation.py +++ b/sklearn/model_selection/_validation.py @@ -361,7 +361,7 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score=False, return_parameters=False, return_n_test_samples=False, return_times=False, return_estimator=False, - error_score='raise-deprecating'): + error_score='raise-deprecating', test_score_weight=None): """Fit estimator and compute scores for a given dataset split. Parameters @@ -402,6 +402,10 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, step, which will always raise the error. Default is 'raise' but from version 0.22 it will change to np.nan. + test_score_weight : string, optional, default: None + If specified, this is the sample weight key in the `fit_params` dict. + If there's no such key found, we will assume unweighted test score. + parameters : dict or None Parameters to be set on the estimator. @@ -455,8 +459,22 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, for k, v in parameters.items())) print("[CV] %s %s" % (msg, (64 - len(msg)) * '.')) - # Adjust length of sample weights + # Init to empty dict if pass in as None fit_params = fit_params if fit_params is not None else {} + + sample_weight_key, weighted_test_score = None, False + + if test_score_weight is not None and test_score_weight in fit_params: + sample_weight_key, weighted_test_score = test_score_weight, True + + if weighted_test_score: + train_sample_weight = _index_param_value(X, fit_params[sample_weight_key], train) + test_sample_weight = _index_param_value(X, fit_params[sample_weight_key], test) + else: + train_sample_weight = None + test_sample_weight = None + + # Adjust length of sample weights fit_params = dict([(k, _index_param_value(X, v, train)) for k, v in fit_params.items()]) @@ -516,11 +534,11 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, else: fit_time = time.time() - start_time # _score will return dict if is_multimetric is True - test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric) + test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric, sample_weight=test_sample_weight) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer, - is_multimetric) + is_multimetric, sample_weight=train_sample_weight) if verbose > 2: if is_multimetric: @@ -546,19 +564,20 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, return ret -def _score(estimator, X_test, y_test, scorer, is_multimetric=False): +def _score(estimator, X_test, y_test, scorer, is_multimetric=False, sample_weight=None): """Compute the score(s) of an estimator on a given test set. Will return a single float if is_multimetric is False and a dict of floats, if is_multimetric is True """ if is_multimetric: - return _multimetric_score(estimator, X_test, y_test, scorer) + return _multimetric_score(estimator, X_test, y_test, scorer, sample_weight=sample_weight) else: + score_kwgs = {} if sample_weight is None else {'sample_weight': sample_weight} if y_test is None: - score = scorer(estimator, X_test) + score = scorer(estimator, X_test, **score_kwgs) else: - score = scorer(estimator, X_test, y_test) + score = scorer(estimator, X_test, y_test, **score_kwgs) if hasattr(score, 'item'): try: @@ -575,15 +594,15 @@ def _score(estimator, X_test, y_test, scorer, is_multimetric=False): return score -def _multimetric_score(estimator, X_test, y_test, scorers): +def _multimetric_score(estimator, X_test, y_test, scorers, sample_weight=None): """Return a dict of score for multimetric scoring""" scores = {} - + score_kwgs = {} if sample_weight is None else {'sample_weight': sample_weight} for name, scorer in scorers.items(): if y_test is None: - score = scorer(estimator, X_test) + score = scorer(estimator, X_test, **score_kwgs) else: - score = scorer(estimator, X_test, y_test) + score = scorer(estimator, X_test, y_test, **score_kwgs) if hasattr(score, 'item'): try: From 48d5c80dad9ca0c1c7de9eb5378b37bc1cd77092 Mon Sep 17 00:00:00 2001 From: dzhang Date: Tue, 13 Mar 2018 10:48:23 -0400 Subject: [PATCH 2/2] some formatting --- sklearn/model_selection/_search.py | 18 ++++++++++------ sklearn/model_selection/_validation.py | 29 ++++++++++++++++++-------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/sklearn/model_selection/_search.py b/sklearn/model_selection/_search.py index ec488d6c1c781..41c02ed5a2830 100644 --- a/sklearn/model_selection/_search.py +++ b/sklearn/model_selection/_search.py @@ -273,7 +273,8 @@ def __len__(self): def fit_grid_point(X, y, estimator, parameters, train, test, scorer, - verbose, error_score='raise-deprecating', test_score_weight=None, **fit_params): + verbose, error_score='raise-deprecating', + test_score_weight=None, **fit_params): """Run fit on one set of parameters. Parameters @@ -342,7 +343,8 @@ def fit_grid_point(X, y, estimator, parameters, train, test, scorer, fit_params=fit_params, return_n_test_samples=True, error_score=error_score, - test_score_weight=test_score_weight) + test_score_weight=test_score_weight + ) return scores, parameters, n_samples_test @@ -446,7 +448,8 @@ def score(self, X, y=None, sample_weight=None): "and the estimator doesn't provide one %s" % self.best_estimator_) score = self.scorer_[self.refit] if self.multimetric_ else self.scorer_ - score_kwgs = {} if sample_weight is None else {'sample_weight': sample_weight} + score_kwgs = {} if sample_weight is None else \ + {'sample_weight': sample_weight} return score(self.best_estimator_, X, y, **score_kwgs) def _check_is_fitted(self, method_name): @@ -1105,7 +1108,8 @@ def __init__(self, estimator, param_grid, scoring=None, fit_params=None, estimator=estimator, scoring=scoring, fit_params=fit_params, n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score, - return_train_score=return_train_score, test_score_weight=test_score_weight) + return_train_score=return_train_score, + test_score_weight=test_score_weight) self.param_grid = param_grid _check_param_grid(param_grid) @@ -1401,7 +1405,8 @@ class RandomizedSearchCV(BaseSearchCV): def __init__(self, estimator, param_distributions, n_iter=10, scoring=None, fit_params=None, n_jobs=1, iid='warn', refit=True, cv=None, verbose=0, pre_dispatch='2*n_jobs', random_state=None, - error_score='raise-deprecating', return_train_score="warn", test_score_weight=None): + error_score='raise-deprecating', return_train_score="warn", + test_score_weight=None): self.param_distributions = param_distributions self.n_iter = n_iter self.random_state = random_state @@ -1409,7 +1414,8 @@ def __init__(self, estimator, param_distributions, n_iter=10, scoring=None, estimator=estimator, scoring=scoring, fit_params=fit_params, n_jobs=n_jobs, iid=iid, refit=refit, cv=cv, verbose=verbose, pre_dispatch=pre_dispatch, error_score=error_score, - return_train_score=return_train_score, test_score_weight=test_score_weight) + return_train_score=return_train_score, + test_score_weight=test_score_weight) def _get_param_iterator(self): """Return ParameterSampler instance for the given distributions""" diff --git a/sklearn/model_selection/_validation.py b/sklearn/model_selection/_validation.py index fcd26f27bdfb5..b92b9e117c60d 100644 --- a/sklearn/model_selection/_validation.py +++ b/sklearn/model_selection/_validation.py @@ -468,8 +468,12 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, sample_weight_key, weighted_test_score = test_score_weight, True if weighted_test_score: - train_sample_weight = _index_param_value(X, fit_params[sample_weight_key], train) - test_sample_weight = _index_param_value(X, fit_params[sample_weight_key], test) + train_sample_weight = _index_param_value(X, + fit_params[sample_weight_key], + train) + test_sample_weight = _index_param_value(X, + fit_params[sample_weight_key], + test) else: train_sample_weight = None test_sample_weight = None @@ -534,11 +538,13 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, else: fit_time = time.time() - start_time # _score will return dict if is_multimetric is True - test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric, sample_weight=test_sample_weight) + test_scores = _score(estimator, X_test, y_test, scorer, is_multimetric, + sample_weight=test_sample_weight) score_time = time.time() - start_time - fit_time if return_train_score: train_scores = _score(estimator, X_train, y_train, scorer, - is_multimetric, sample_weight=train_sample_weight) + is_multimetric, + sample_weight=train_sample_weight) if verbose > 2: if is_multimetric: @@ -564,16 +570,19 @@ def _fit_and_score(estimator, X, y, scorer, train, test, verbose, return ret -def _score(estimator, X_test, y_test, scorer, is_multimetric=False, sample_weight=None): +def _score(estimator, X_test, y_test, scorer, is_multimetric=False, + sample_weight=None): """Compute the score(s) of an estimator on a given test set. Will return a single float if is_multimetric is False and a dict of floats, if is_multimetric is True """ if is_multimetric: - return _multimetric_score(estimator, X_test, y_test, scorer, sample_weight=sample_weight) + return _multimetric_score(estimator, X_test, y_test, scorer, + sample_weight=sample_weight) else: - score_kwgs = {} if sample_weight is None else {'sample_weight': sample_weight} + score_kwgs = {} if sample_weight is None else \ + {'sample_weight': sample_weight} if y_test is None: score = scorer(estimator, X_test, **score_kwgs) else: @@ -594,10 +603,12 @@ def _score(estimator, X_test, y_test, scorer, is_multimetric=False, sample_weigh return score -def _multimetric_score(estimator, X_test, y_test, scorers, sample_weight=None): +def _multimetric_score(estimator, X_test, y_test, scorers, + sample_weight=None): """Return a dict of score for multimetric scoring""" scores = {} - score_kwgs = {} if sample_weight is None else {'sample_weight': sample_weight} + score_kwgs = {} if sample_weight is None else \ + {'sample_weight': sample_weight} for name, scorer in scorers.items(): if y_test is None: score = scorer(estimator, X_test, **score_kwgs)