import argparse
import codecs
import sys
import matplotlib.pyplot as plt
import numpy as np

from sklearn.cross_validation import ShuffleSplit
from sklearn import cross_validation
from sklearn.datasets import load_files
from sklearn.feature_extraction.text import TfidfVectorizer, HashingVectorizer
from sklearn.linear_model import PassiveAggressiveClassifier, SGDClassifier
from sklearn.metrics import SCORERS
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC


def randomized_train_test(estimator, data, target, test_size, n_iter, scorer, random_state, n_jobs):
    cv = ShuffleSplit(len(data), n_iter=n_iter, test_size=test_size, random_state=random_state)

    scores = cross_validation.cross_val_score(estimator, data, target, scorer, cv, n_jobs)
    score_mean = scores.mean()
    score_std = scores.std()
    return score_mean, score_std, scores


def plot(scorer_name, results, grid_w=None, grid_h=None):
    all_scores = list()
    for _, (_, _, scores) in results:
        all_scores.extend(scores)

    count = len(results)

    bins = np.histogram(all_scores, bins=10 + (len(all_scores) /
                                               (count * np.max((
                                                   1, np.log2(len(all_scores) / count))))))[1]

    if grid_w is None:
        grid_w = int(np.ceil(np.sqrt(count)))
    if grid_h is None:
        grid_h = int(np.ceil(count / grid_w))

    subplots = list()

    for i, (name, (score_mean, score_std, scores)) in enumerate(results):
        if len(subplots) > 0:
            subplot = plt.subplot2grid((grid_h, grid_w),
                                       (int(i / grid_w), int(i % grid_w)),
                                       sharex=subplots[0],
                                       sharey=subplots[0])
        else:
            subplot = plt.subplot2grid((grid_h, grid_w),
                                       (int(i / grid_w), int(i % grid_w)))

        subplot.set_title('%s mean %s=%0.3f std=%0.3f' % (name, scorer_name, score_mean, score_std))
        subplot.hist(scores, alpha=0.5, bins=bins)
        subplots.append(subplot)

    ylim = subplots[0].get_ylim()

    for (name, (score_mean, score_std, _)), subplot in zip(results, subplots):
        subplot.plot([score_mean for _ in ylim], ylim, '--k', linewidth=2)
        subplot.plot([score_mean - score_std for _ in ylim], ylim, ':k', linewidth=2)
        subplot.plot([score_mean + score_std for _ in ylim], ylim, ':k', linewidth=2)

# download demo data from:
# https://www.cs.cornell.edu/people/pabo/movie-review-data/review_polarity.tar.gz

def main():
    sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer)
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('-p', '--path', help='Input path', required=True)
    parser.add_argument('-r', '--random', help='Randomization seed (default: 0)', type=int, default=0)
    parser.add_argument('-s', '--scorer',
                        help='Evaluation function (default: f1, values: %s)' % ' '.join(SCORERS.keys()), type=str,
                        default='f1')
    parser.add_argument('-i', '--iter', help='Number of randomized runs (default: 100)', type=int, default=100)
    parser.add_argument('-t', '--test', help='Portions of examples to be used as test set (default: 0.2)', type=float,
                        default=0.2)
    args = parser.parse_args()

    test_size = args.test
    random_state = args.random
    n_iter = args.iter
    n_jobs = -1
    scorer = SCORERS[args.scorer]

    examples = load_files(args.path, shuffle=False)

    data = examples.data
    target = examples.target

    print("Total %i Train/test split %0.2f/%0.2f %i/%i" % (len(data),
                                                           (1 - args.test), args.test,
                                                           int(len(data) * (1 - args.test)),
                                                           int(len(data) * args.test)))

    experiments = list()

    experiments.append(('pa-tfidf', Pipeline([
        ('vec', TfidfVectorizer()),
        ('clf', PassiveAggressiveClassifier(random_state=random_state)),
    ])))

    experiments.append(('pa-hash-no-average', Pipeline([
        ('vec', HashingVectorizer()),
        ('clf', PassiveAggressiveClassifier(random_state=random_state)),
    ])))

    experiments.append(('pa-hash-average-100', Pipeline([
        ('vec', HashingVectorizer()),
        ('clf', PassiveAggressiveClassifier(random_state=random_state, average=100)),
    ])))


    experiments.append(('sgdc-tfidf', Pipeline([
        ('vec', TfidfVectorizer()),
        ('clf', SGDClassifier(random_state=random_state)),
    ])))

    experiments.append(('sgdc-hash-no-average', Pipeline([
        ('vec', HashingVectorizer()),
        ('clf', SGDClassifier(random_state=random_state)),
    ])))

    experiments.append(('sgdc-hash-average-100', Pipeline([
        ('vec', HashingVectorizer()),
        ('clf', SGDClassifier(random_state=random_state, average=100)),
    ])))

    results = list()

    for name, estimator in experiments:
        print(name)
        results.append((name, list(randomized_train_test(estimator, data, target, test_size, n_iter,
                                                         scorer,
                                                         random_state, n_jobs))))

    print(results)

    plt.figure()
    plot(args.scorer, results, grid_w=3, grid_h=2)
    plt.show()


if __name__ == "__main__":
    main()
'''

<!-- Reviewable:start -->

---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/scikit-learn/scikit-learn/4939)
<!-- Reviewable:end -->