From 7f02c949ddb92484f63b8d3ecdfdcea3248eab7a Mon Sep 17 00:00:00 2001 From: Vlad Niculae Date: Thu, 10 Jan 2013 17:26:01 +0000 Subject: [PATCH 1/3] Add failing test for issue #1389 --- sklearn/linear_model/tests/test_ridge.py | 33 +++++++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/sklearn/linear_model/tests/test_ridge.py b/sklearn/linear_model/tests/test_ridge.py index f2ba15de25d60..d058720314f56 100644 --- a/sklearn/linear_model/tests/test_ridge.py +++ b/sklearn/linear_model/tests/test_ridge.py @@ -231,7 +231,7 @@ def _test_ridge_loo(filter_): assert_array_almost_equal(np.vstack((y_pred, y_pred)).T, Y_pred, decimal=5) - return ret + yield ret def _test_ridge_cv(filter_): @@ -256,7 +256,7 @@ def _test_ridge_cv(filter_): def _test_ridge_diabetes(filter_): ridge = Ridge(fit_intercept=False) ridge.fit(filter_(X_diabetes), y_diabetes) - return np.round(ridge.score(filter_(X_diabetes), y_diabetes), 5) + yield np.round(ridge.score(filter_(X_diabetes), y_diabetes), 5) def _test_multi_ridge_diabetes(filter_): @@ -303,20 +303,39 @@ def _test_tolerance(filter_): assert_true(score >= score2) +def _test_intercept_values(filter_): + """Tests that the learned model doesn't change if the array is sparse + + Issue #1389 + """ + for n_samples, n_features in ((5, 3), (3, 5)): # Test both tall and wide + rng = np.random.RandomState(0) + X = rng.randn(n_samples, n_features) + y = rng.randn(n_samples) + for normalize in (False, True): + ridge = Ridge(alpha=1e-10, fit_intercept=True, normalize=normalize) + # almost no regularization, but not singular + ridge.fit(filter_(X), y) + yield ridge.coef_ + yield ridge.intercept_ + + def test_dense_sparse(): for test_func in (_test_ridge_loo, _test_ridge_cv, _test_ridge_diabetes, _test_multi_ridge_diabetes, _test_ridge_classifiers, - _test_tolerance): + _test_tolerance, + _test_intercept_values): # test dense matrix - ret_dense = test_func(DENSE_FILTER) + dense_returns = test_func(DENSE_FILTER) # test sparse matrix - ret_sparse = test_func(SPARSE_FILTER) + sparse_returns = test_func(SPARSE_FILTER) # test that the outputs are the same - if ret_dense is not None and ret_sparse is not None: - assert_array_almost_equal(ret_dense, ret_sparse, decimal=3) + if dense_returns is not None: + for ret_dense, ret_sparse in zip(dense_returns, sparse_returns): + assert_array_almost_equal(ret_dense, ret_sparse, decimal=3) def test_class_weights(): From 4ae5640760f021dae4bbe506c9a9e1ec88ac5ae0 Mon Sep 17 00:00:00 2001 From: Vlad Niculae Date: Thu, 10 Jan 2013 17:50:54 +0000 Subject: [PATCH 2/3] Fit intercept in Ridge with dummy feature. Wide tests fail --- sklearn/linear_model/ridge.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/sklearn/linear_model/ridge.py b/sklearn/linear_model/ridge.py index 0a4e43bdf111e..5a0ee80443e8e 100644 --- a/sklearn/linear_model/ridge.py +++ b/sklearn/linear_model/ridge.py @@ -20,7 +20,7 @@ from ..base import RegressorMixin from ..utils.extmath import safe_sparse_dot from ..utils import safe_asarray -from ..preprocessing import LabelBinarizer +from ..preprocessing import LabelBinarizer, StandardScaler, add_dummy_feature from ..grid_search import GridSearchCV @@ -192,16 +192,39 @@ def fit(self, X, y, sample_weight=1.0, solver=None): X = safe_asarray(X, dtype=np.float) y = np.asarray(y, dtype=np.float) - X, y, X_mean, y_mean, X_std = self._center_data( - X, y, self.fit_intercept, self.normalize, self.copy_X) - + is_sparse = not hasattr(X, '__array__') + if self.fit_intercept: + if is_sparse: + if self.normalize: + scaler = StandardScaler(with_mean=False, with_std=True, + copy=False) + X = scaler.fit_transform(X) + X_std = scaler.std_ + X = add_dummy_feature(X, value=1.0) # TODO: intercept weight + else: + X, y, X_mean, y_mean, X_std = self._center_data( + X, y, True, self.normalize, self.copy_X) self.coef_ = ridge_regression(X, y, alpha=self.alpha, sample_weight=sample_weight, solver=solver, max_iter=self.max_iter, tol=self.tol) - self._set_intercept(X_mean, y_mean, X_std) + if self.fit_intercept: + if is_sparse: + self.coef_ = np.atleast_2d(self.coef_) + self.intercept_ = self.coef_[:, 0] + self.coef_ = self.coef_[:, 1:] + self.intercept_ = np.squeeze(self.intercept_) + self.coef_ = np.squeeze(self.coef_) + if self.intercept_.ndim == 0: + self.intercept_ = np.float64(self.intercept_) + if self.normalize: + self.coef_ /= X_std + else: + self._set_intercept(X_mean, y_mean, X_std) + else: + self.intercept_ = 0.0 return self From 0f1de922cbcae5adb921a789cee5b6a3b3a9be3c Mon Sep 17 00:00:00 2001 From: Vlad Niculae Date: Thu, 10 Jan 2013 18:05:55 +0000 Subject: [PATCH 3/3] Reduce branches, leverage `_set_intercept` --- sklearn/linear_model/ridge.py | 43 +++++++++++++++-------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/sklearn/linear_model/ridge.py b/sklearn/linear_model/ridge.py index 5a0ee80443e8e..4423902ba8df4 100644 --- a/sklearn/linear_model/ridge.py +++ b/sklearn/linear_model/ridge.py @@ -193,38 +193,31 @@ def fit(self, X, y, sample_weight=1.0, solver=None): y = np.asarray(y, dtype=np.float) is_sparse = not hasattr(X, '__array__') - if self.fit_intercept: - if is_sparse: - if self.normalize: - scaler = StandardScaler(with_mean=False, with_std=True, - copy=False) - X = scaler.fit_transform(X) - X_std = scaler.std_ - X = add_dummy_feature(X, value=1.0) # TODO: intercept weight + if self.fit_intercept and is_sparse: + if self.normalize: + scaler = StandardScaler(with_mean=False, with_std=True, + copy=False) + X = scaler.fit_transform(X) + X_std = scaler.std_ else: - X, y, X_mean, y_mean, X_std = self._center_data( - X, y, True, self.normalize, self.copy_X) + X_std = np.ones(X.shape[1]) + X_mean = np.zeros(X.shape[1]) + y_mean = 0. + X = add_dummy_feature(X, value=1.0) # TODO: intercept weight + else: + X, y, X_mean, y_mean, X_std = self._center_data( + X, y, self.fit_intercept, self.normalize, self.copy_X) self.coef_ = ridge_regression(X, y, alpha=self.alpha, sample_weight=sample_weight, solver=solver, max_iter=self.max_iter, tol=self.tol) - if self.fit_intercept: - if is_sparse: - self.coef_ = np.atleast_2d(self.coef_) - self.intercept_ = self.coef_[:, 0] - self.coef_ = self.coef_[:, 1:] - self.intercept_ = np.squeeze(self.intercept_) - self.coef_ = np.squeeze(self.coef_) - if self.intercept_.ndim == 0: - self.intercept_ = np.float64(self.intercept_) - if self.normalize: - self.coef_ /= X_std - else: - self._set_intercept(X_mean, y_mean, X_std) - else: - self.intercept_ = 0.0 + if self.fit_intercept and is_sparse: + coef = np.atleast_2d(self.coef_) + self.coef_ = coef[:, 1:].squeeze() + y_mean = coef[:, 0].squeeze() + self._set_intercept(X_mean, y_mean, X_std) return self