Skip to content

[MRG+1] idf_ setter for TfidfTransformer. #10899

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Apr 5, 2018
Merged

[MRG+1] idf_ setter for TfidfTransformer. #10899

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
639a6a0
idf_ setter for TfidfTransformer. Fixes #7102
serega Apr 1, 2018
b753a56
Do not re-fit TfidfTransformer. Changed fit_transform to transform.
serega Apr 1, 2018
8cb346b
Validate the length of idf argument in TfidfVectorizer.
serega Apr 2, 2018
fe2a303
Line length is less then 80 chars
serega Apr 2, 2018
0ceafad
Made the line even shorter
serega Apr 2, 2018
78cecea
Using setattr(copy) instead of copy.__setattr__
serega Apr 3, 2018
cd0e19b
idf_ is not defined when use_idf is false.
serega Apr 3, 2018
91460f1
docs corrections as suggested by rth.
serega Apr 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions sklearn/feature_extraction/tests/test_text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -942,6 +942,35 @@ def test_pickling_transformer():
orig.fit_transform(X).toarray())


def test_transformer_idf_setter():
X = CountVectorizer().fit_transform(JUNK_FOOD_DOCS)
orig = TfidfTransformer().fit(X)
copy = TfidfTransformer()
copy.idf_ = orig.idf_
assert_array_equal(
copy.transform(X).toarray(),
orig.transform(X).toarray())


def test_tfidf_vectorizer_setter():
orig = TfidfVectorizer(use_idf=True)
orig.fit(JUNK_FOOD_DOCS)
copy = TfidfVectorizer(vocabulary=orig.vocabulary_, use_idf=True)
copy.idf_ = orig.idf_
assert_array_equal(
copy.transform(JUNK_FOOD_DOCS).toarray(),
orig.transform(JUNK_FOOD_DOCS).toarray())


def test_tfidfvectorizer_invalid_idf_attr():
vect = TfidfVectorizer(use_idf=True)
vect.fit(JUNK_FOOD_DOCS)
copy = TfidfVectorizer(vocabulary=vect.vocabulary_, use_idf=True)
expected_idf_len = len(vect.idf_)
invalid_idf = [1.0] * (expected_idf_len + 1)
assert_raises(ValueError, setattr, copy, 'idf_', invalid_idf)


def test_non_unique_vocab():
vocab = ['a', 'b', 'c', 'a', 'a']
vect = CountVectorizer(vocabulary=vocab)
Expand Down
29 changes: 26 additions & 3 deletions sklearn/feature_extraction/text.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1062,6 +1062,12 @@ class TfidfTransformer(BaseEstimator, TransformerMixin):
sublinear_tf : boolean, default=False
Apply sublinear tf scaling, i.e. replace tf with 1 + log(tf).

Attributes
----------
idf_ : array, shape (n_features)
The inverse document frequency (IDF) vector; only defined
if ``use_idf`` is True.

References
----------

Expand Down Expand Up @@ -1157,6 +1163,13 @@ def idf_(self):
# which means hasattr(self, "idf_") is False
return np.ravel(self._idf_diag.sum(axis=0))

@idf_.setter
def idf_(self, value):
value = np.asarray(value, dtype=np.float64)
n_features = value.shape[0]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

n_features is the vocabulary size, we should check here that the provided shape is consistent with it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TfidfTransformer does not have the vocabulary, TfidfVectorizer does. I added validation to TfidfVectorizer

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, right, thanks.

self._idf_diag = sp.spdiags(value, diags=0, m=n_features,
n=n_features, format='csr')


class TfidfVectorizer(CountVectorizer):
"""Convert a collection of raw documents to a matrix of TF-IDF features.
Expand Down Expand Up @@ -1295,9 +1308,9 @@ class TfidfVectorizer(CountVectorizer):
vocabulary_ : dict
A mapping of terms to feature indices.

idf_ : array, shape = [n_features], or None
The learned idf vector (global term weights)
when ``use_idf`` is set to True, None otherwise.
idf_ : array, shape (n_features)
The inverse document frequency (IDF) vector; only defined
if ``use_idf`` is True.

stop_words_ : set
Terms that were ignored because they either:
Expand Down Expand Up @@ -1386,6 +1399,16 @@ def sublinear_tf(self, value):
def idf_(self):
return self._tfidf.idf_

@idf_.setter
def idf_(self, value):
self._validate_vocabulary()
if hasattr(self, 'vocabulary_'):
if len(self.vocabulary_) != len(value):
raise ValueError("idf length = %d must be equal "
"to vocabulary size = %d" %
(len(value), len(self.vocabulary)))
self._tfidf.idf_ = value

def fit(self, raw_documents, y=None):
"""Learn vocabulary and idf from training set.

Expand Down