from __future__ import print_function

import logging
import numpy as np
from optparse import OptionParser
import sys
from time import time
import matplotlib.pyplot as plt

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.grid_search import GridSearchCV
from sklearn.linear_model import SGDClassifier
from sklearn.cross_validation import permutation_test_score

categories = ['sci.space', 'comp.graphics']
data_train = fetch_20newsgroups(subset='train', categories=categories,
                                shuffle=True, random_state=42)


y_train = data_train.target
vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,
                             stop_words='english')
X_train = vectorizer.fit_transform(data_train.data)
estimator = SGDClassifier(loss='log', penalty= 'l2',l1_ratio=0.5, n_iter=20, shuffle=True, verbose=False, 
                   n_jobs=6, alpha=1e-6, average=False, class_weight='balanced')
parameters = {'alpha': [10 ** a for a in range(-10, -1)]}
clf = GridSearchCV(estimator, parameters, cv=5, scoring='f1_macro', n_jobs=5)
clf.fit(X_train, y_train)
score, permutation_scores, pvalue = permutation_test_score(clf.best_estimator_, X_train, y_train, cv=5, n_jobs=5,
                                                   n_permutations=100, scoring='accuracy')
print ('socre %f p-value %f' %( score, pvalue))