From c4f683787974eb0cd6483669fe011bd2dcc1bcb6 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:01:44 +0900 Subject: [PATCH 01/26] Add prev, next and fit_predict, fit_transform to pipeline, Add returnType to mlLibrary --- visualpython/css/m_ml/pipeline.css | 56 ++++++++- visualpython/data/m_ml/mlLibrary.js | 52 +++++++- visualpython/html/m_ml/pipeline.html | 4 + visualpython/js/m_ml/Pipeline.js | 173 ++++++++++++++++++++++----- 4 files changed, 253 insertions(+), 32 deletions(-) diff --git a/visualpython/css/m_ml/pipeline.css b/visualpython/css/m_ml/pipeline.css index 0bbe46c7..7e670421 100644 --- a/visualpython/css/m_ml/pipeline.css +++ b/visualpython/css/m_ml/pipeline.css @@ -15,7 +15,7 @@ } .vp-pp-right-box { display: grid; - grid-template-rows: 35px 10px calc(100% - 45px); + grid-template-rows: 35px 10px calc(100% - 80px) 35px; padding-left: 5px; overflow: auto; } @@ -45,7 +45,13 @@ position: absolute; transition: all 0.5s ease-in-out; } -.vp-pp-item:not(:last-child):before { +.vp-pp-item:last-child:before, +.vp-pp-item:last-child:after, +.vp-pp-item.vp-last-visible:before, +.vp-pp-item.vp-last-visible:after { + display: none; +} +.vp-pp-item:before { border: 0.25px solid var(--vp-gray-color); border-width: 0 4px 4px 0; display: inline-block; @@ -55,7 +61,7 @@ top: 47px; left: 100px; } -.vp-pp-item:not(:last-child):after { +.vp-pp-item:after { border: 0.25px solid var(--vp-gray-color); border-width: 0 4px 0 0; height: 15px; @@ -129,4 +135,48 @@ } .vp-pp-step-content:empty::after { content: 'Select the template and Follow the pipeline to generate simple ML code.' +} +.vp-pp-step-footer { + border-top: 0.25px solid var(--vp-border-gray-color); + width: 100%; + height: 35px; +} +.vp-pp-step-move-btn { + width: 70px; + height: 30px; + background-color: var(--vp-background-color); + border: 0.25px solid var(--vp-border-gray-color); + box-sizing: border-box; + box-shadow: 0.5px 0.5px 0.5px rgb(0 0 0 / 10%); + border-radius: 3px; + line-height: 30px; + vertical-align: middle; + font-family: 'AppleSDGothicNeo'; + font-size: 13px; + text-align: center; + color: var(--vp-font-primary); + cursor: pointer; + -webkit-touch-callout: none; + -webkit-user-select: none; + -khtml-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + + display: inline-block; + position: absolute; + bottom: 10px; +} +.vp-pp-step-move-btn:hover { + background-color: var(--vp-light-gray-color); +} +.vp-pp-step-move-btn.disabled { + background-color: var(--vp-gray-color); + cursor: not-allowed; +} +.vp-pp-step-prev { + left: 270px; +} +.vp-pp-step-next{ + right: 15px; } \ No newline at end of file diff --git a/visualpython/data/m_ml/mlLibrary.js b/visualpython/data/m_ml/mlLibrary.js index 78528fed..beeae0ec 100644 --- a/visualpython/data/m_ml/mlLibrary.js +++ b/visualpython/data/m_ml/mlLibrary.js @@ -129,6 +129,7 @@ define([ name: 'OneHotEncoder', import: 'from sklearn.preprocessing import OneHotEncoder', code: 'OneHotEncoder(${sparse}${handle_unknown}${etc})', + returnType: 'OneHotEncoder', options: [ { name: 'sparse', component: ['bool_select'], default: 'False', usePair: true }, { name: 'handle_unknown', component: ['option_suggest'], usePair: true, @@ -139,6 +140,7 @@ define([ name: 'LabelEncoder', import: 'from sklearn.preprocessing import LabelEncoder', code: 'LabelEncoder()', + returnType: 'LabelEncoder', options: [ ] @@ -147,6 +149,7 @@ define([ name: 'OrdinalEncoder', import: 'from sklearn.preprocessing import OrdinalEncoder', code: 'OrdinalEncoder(${handle_unknown}${unknown_values}${etc})', + returnType: 'OrdinalEncoder', options: [ { name: 'handle_unknown', component: ['option_suggest'], usePair: true, options: ['error', 'use_encoded_value'], default: 'error' }, @@ -158,6 +161,7 @@ define([ install: '!pip install category_encoders', import: 'from category_encoders.target_encoder import TargetEncoder', code: 'TargetEncoder(${cols}${handle_missing}${handle_unknown}${smoothing}${etc})', + returnType: 'TargetEncoder', options: [ { name: 'cols', component: ['var_suggest', '1darr'], usePair: true }, { name: 'handle_missing', component: ['option_suggest'], usePair: true, @@ -172,6 +176,7 @@ define([ install: '!pip install imblearn', import: 'from imblearn.over_sampling import SMOTE', code: 'SMOTE(${random_state}${k_neighbors}${etc})', + returnType: 'SMOTE', options: [ { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, { name: 'k_neighbors', component: ['input_number'], default: 5, usePair: true } @@ -182,6 +187,7 @@ define([ name: 'StandardScaler', import: 'from sklearn.preprocessing import StandardScaler', code: 'StandardScaler(${with_mean}${with_std}${etc})', + returnType: 'StandardScaler', options: [ { name: 'with_mean', component: ['bool_select'], default: 'True', usePair: true }, { name: 'with_std', component: ['bool_select'], default: 'True', usePair: true } @@ -191,6 +197,7 @@ define([ name: 'RobustScaler', import: 'from sklearn.preprocessing import RobustScaler', code: 'RobustScaler(${with_centering}${with_scaling}${etc})', + returnType: 'RobustScaler', options: [ { name: 'with_centering', component: ['bool_select'], default: 'True', usePair: true }, { name: 'with_scaling', component: ['bool_select'], default: 'True', usePair: true } @@ -200,6 +207,7 @@ define([ name: 'MinMaxScaler', import: 'from sklearn.preprocessing import MinMaxScaler', code: 'MinMaxScaler(${feature_range}${etc})', + returnType: 'MinMaxScaler', options: [ { name: 'feature_range', component: ['input'], placeholder: '(min, max)', default: '(0, 1)', usePair: true } ] @@ -208,6 +216,7 @@ define([ name: 'Normalizer', import: 'from sklearn.preprocessing import Normalizer', code: 'Normalizer(${norm}${etc})', + returnType: 'Normalizer', options: [ { name: 'norm', component: ['option_suggest'], usePair: true, options: ['l1', 'l2', 'max'], default: 'l2' }, @@ -217,6 +226,7 @@ define([ name: 'Log Scaling', import: 'from sklearn.preprocessing import FunctionTransformer', code: 'FunctionTransformer(np.log1p${etc})', + returnType: 'FunctionTransformer', options: [ ] @@ -225,6 +235,7 @@ define([ name: 'Exponential Scaling', import: 'from sklearn.preprocessing import FunctionTransformer', code: 'FunctionTransformer(np.expm1${etc})', + returnType: 'FunctionTransformer', options: [ ] @@ -233,6 +244,7 @@ define([ name: 'Polynomial Features', import: 'from sklearn.preprocessing import PolynomialFeatures', code: 'PolynomialFeatures(${etc})', + returnType: 'PolynomialFeatures', options: [ ] @@ -241,6 +253,7 @@ define([ name: 'KBins Discretizer', import: 'from sklearn.preprocessing import KBinsDiscretizer', code: 'KBinsDiscretizer(${n_bins}${strategy}${encode}${etc})', + returnType: 'KBinsDiscretizer', options: [ { name: 'n_bins', component: ['input_number'], default: 5, usePair: true }, { name: 'strategy', component: ['option_select'], type: 'text', default: 'quantile', usePair: true, @@ -253,6 +266,7 @@ define([ name: 'SimpleImputer', import: 'from sklearn.impute import SimpleImputer', code: 'SimpleImputer(${missing_values}${strategy}${fill_value}${copy}${add_indicator}${etc})', + returnType: 'SimpleImputer', options: [ { name: 'missing_values', component: ['input'], placeholder: 'np.nan', usePair: true }, { name: 'strategy', component: ['option_select'], type: 'text', default: 'quantile', usePair: true, @@ -266,6 +280,7 @@ define([ name: 'MakeColumnTransformer', import: 'from sklearn.compose import make_column_transformer', code: 'make_column_transformer(${mct_code})', + returnType: 'ColumnTransformer', options: [ ] @@ -275,6 +290,7 @@ define([ name: 'LinearRegression', import: 'from sklearn.linear_model import LinearRegression', code: 'LinearRegression(${fit_intercept}${etc})', + returnType: 'LinearRegression', options: [ { name: 'fit_intercept', component: ['bool_select'], default: 'True', usePair: true } ] @@ -283,6 +299,7 @@ define([ name: 'Ridge', import: 'from sklearn.linear_model import Ridge', code: 'Ridge(${alpha}${etc})', + returnType: 'Ridge', options: [ { name: 'alpha', component: ['input_number'], default: 1.0, usePair: true } ] @@ -291,6 +308,7 @@ define([ name: 'Lasso', import: 'from sklearn.linear_model import Lasso', code: 'Lasso(${alpha}${etc})', + returnType: 'Lasso', options: [ { name: 'alpha', component: ['input_number'], default: 1.0, usePair: true } ] @@ -299,6 +317,7 @@ define([ name: 'ElasticNet', import: 'from sklearn.linear_model import ElasticNet', code: 'ElasticNet(${alpha}${l1_ratio}${etc})', + returnType: 'ElasticNet', options: [ { name: 'alpha', component: ['input_number'], default: 1.0, usePair: true }, { name: 'l1_ratio', component: ['input_number'], default: 0.5, usePair: true } @@ -308,6 +327,7 @@ define([ name: 'SVR', import: 'from sklearn.svm import SVR', code: 'SVR(${C}${kernel}${degree}${gamma}${coef0}${random_state}${etc})', + returnType: 'SVR', options: [ { name: 'C', component: ['input_number'], placeholder: '1.0', usePair: true, step: 0.1, min: 0 }, { name: 'kernel', component: ['option_select'], type: 'text', usePair: true, @@ -323,6 +343,7 @@ define([ name: 'DecisionTreeRegressor', import: 'from sklearn.tree import DecisionTreeRegressor', code: 'DecisionTreeRegressor(${criterion}${max_depth}${min_samples_split}${random_state}${etc})', + returnType: 'DecisionTreeRegressor', options: [ { name: 'criterion', component: ['option_select'], type: 'text', default: 'squared_error', type:'text', options: ['squared_error', 'friedman_mse', 'absolute_error', 'poisson'] }, @@ -335,6 +356,7 @@ define([ name: 'RandomForestRegressor', import: 'from sklearn.ensemble import RandomForestRegressor', code: 'RandomForestRegressor(${n_estimators}${criterion}${max_depth}${min_samples_split}${n_jobs}${random_state}${etc})', + returnType: 'RandomForestRegressor', options: [ { name: 'n_estimators', component: ['input_number'], default: 100, usePair: true }, { name: 'criterion', component: ['option_select'], type: 'text', default: 'squared_error', type:'text', usePair: true, @@ -349,6 +371,7 @@ define([ name: 'GradientBoostingRegressor', import: 'from sklearn.ensemble import GradientBoostingRegressor', code: 'GradientBoostingRegressor(${loss}${learning_rate}${n_estimators}${criterion}${random_state}${etc})', + returnType: 'GradientBoostingRegressor', options: [ { name: 'loss', component: ['option_select'], type: 'text', default: 'squared_error', type:'text', usePair: true, options: ['squared_error', 'absolute_error', 'huber', 'quantile'] }, @@ -364,6 +387,7 @@ define([ install: '!pip install xgboost', import: 'from xgboost import XGBRegressor', code: 'XGBRegressor(${n_estimators}${max_depth}${learning_rate}${gamma}${random_state}${etc})', + returnType: 'XGBRegressor', options: [ { name: 'n_estimators', component: ['input_number'], default: 100, usePair: true }, { name: 'max_depth', component: ['input_number'], placeholder: 'None', usePair: true }, @@ -377,6 +401,7 @@ define([ install: '!pip install lightgbm', import: 'from lightgbm import LGBMRegressor', code: 'LGBMRegressor(${boosting_type}${max_depth}${learning_rate}${n_estimators}${random_state}${etc})', + returnType: 'LGBMRegressor', options: [ { name: 'boosting_type', component: ['option_select'], type: 'text', default: 'gbdt', type: 'text', usePair: true, options: ['gbdt', 'dart', 'goss', 'rf']}, @@ -391,6 +416,7 @@ define([ install: '!pip install catboost', import: 'from catboost import CatBoostRegressor', code: 'CatBoostRegressor(${learning_rate}${loss_function}${task_type}${max_depth}${n_estimators}${random_state}${etc})', + returnType: 'CatBoostRegressor', options: [ { name: 'learning_rate', component: ['input_number'], placeholder: 'None', usePair: true }, { name: 'loss_function', component: ['option_select'], type: 'text', default: 'RMSE', type:'text', usePair: true, @@ -407,6 +433,7 @@ define([ name: 'LogisticRegression', import: 'from sklearn.linear_model import LogisticRegression', code: 'LogisticRegression(${penalty}${C}${random_state}${etc})', + returnType: 'LogisticRegression', options: [ { name: 'penalty', component: ['option_select'], type: 'text', default: 'l2', usePair: true, options: ['l1', 'l2', 'elasticnet', 'none']}, { name: 'C', component: ['input_number'], placeholder: '1.0', usePair: true, step: 0.1, min: 0 }, @@ -417,6 +444,7 @@ define([ name: 'BernoulliNB', import: 'from sklearn.naive_bayes import BernoulliNB', code: 'BernoulliNB(${etc})', + returnType: 'BernoulliNB', options: [ //TODO: ] @@ -425,6 +453,7 @@ define([ name: 'MultinomialNB', import: 'from sklearn.naive_bayes import MultinomialNB', code: 'MultinomialNB(${etc})', + returnType: 'MultinomialNB', options: [ //TODO: ] @@ -433,6 +462,7 @@ define([ name: 'GaussianNB', import: 'from sklearn.naive_bayes import GaussianNB', code: 'GaussianNB(${etc})', + returnType: 'GaussianNB', options: [ //TODO: ] @@ -441,6 +471,7 @@ define([ name: 'SVC', import: 'from sklearn.svm import SVC', code: 'SVC(${C}${kernel}${degree}${gamma}${coef0}${random_state}${etc})', + returnType: 'SVC', options: [ { name: 'C', component: ['input_number'], placeholder: '1.0', usePair: true, step: 0.1, min: 0 }, { name: 'kernel', component: ['option_select'], type: 'text', usePair: true, @@ -456,6 +487,7 @@ define([ name: 'DecisionTreeClassifier', import: 'from sklearn.tree import DecisionTreeClassifier', code: 'DecisionTreeClassifier(${criterion}${max_depth}${min_samples_split}${random_state}${etc})', + returnType: 'DecisionTreeClassifier', options: [ { name: 'criterion', component: ['option_select'], type: 'text', default: 'squared_error', type:'text', options: ['squared_error', 'friedman_mse', 'absolute_error', 'poisson'], usePair: true }, @@ -468,6 +500,7 @@ define([ name: 'RandomForestClassifier', import: 'from sklearn.ensemble import RandomForestClassifier', code: 'RandomForestClassifier(${n_estimators}${criterion}${max_depth}${min_samples_split}${n_jobs}${random_state}${etc})', + returnType: 'RandomForestClassifier', options: [ { name: 'n_estimators', component: ['input_number'], default: 100, usePair: true }, { name: 'criterion', component: ['option_select'], type: 'text', default: 'gini', type:'text', usePair: true, @@ -482,6 +515,7 @@ define([ name: 'GradientBoostingClassifier', import: 'from sklearn.ensemble import GradientBoostingClassifier', code: 'GradientBoostingClassifier(${loss}${learning_rate}${n_estimators}${criterion}${random_state}${etc})', + returnType: 'GradientBoostingClassifier', options: [ { name: 'loss', component: ['option_select'], type: 'text', default: 'deviance', type: 'text', usePair: true, options: ['deviance', 'exponential'] }, @@ -497,6 +531,7 @@ define([ install: '!pip install xgboost', import: 'from xgboost import XGBClassifier', code: 'XGBClassifier(${n_estimators}${max_depth}${learning_rate}${gamma}${random_state}${etc})', + returnType: 'XGBClassifier', options: [ { name: 'n_estimators', component: ['input_number'], default: 100, usePair: true }, { name: 'max_depth', component: ['input_number'], placeholder: 'None', usePair: true }, @@ -510,6 +545,7 @@ define([ install: '!pip install lightgbm', import: 'from lightgbm import LGBMClassifier', code: 'LGBMClassifier(${boosting_type}${max_depth}${learning_rate}${n_estimators}${random_state}${etc})', + returnType: 'LGBMClassifier', options: [ { name: 'boosting_type', component: ['option_select'], type: 'text', default: 'gbdt', type: 'text', usePair: true, options: ['gbdt', 'dart', 'goss', 'rf']}, @@ -524,6 +560,7 @@ define([ install: '!pip install catboost', import: 'from catboost import CatBoostClassifier', code: 'CatBoostClassifier(${learning_rate}${loss_function}${task_type}${max_depth}${n_estimators}${random_state}${etc})', + returnType: 'CatBoostClassifier', options: [ { name: 'learning_rate', component: ['input_number'], placeholder: 'None', usePair: true }, { name: 'loss_function', component: ['option_select'], type: 'text', default: 'RMSE', type:'text', usePair: true, @@ -542,6 +579,7 @@ define([ import: 'from autosklearn.regression import AutoSklearnRegressor', link: 'https://automl.github.io/auto-sklearn/master/api.html#regression', code: 'AutoSklearnRegressor(${etc})', + returnType: 'AutoSklearnRegressor', options: [ ] @@ -551,6 +589,7 @@ define([ install: '!pip install tpot', import: 'from tpot import TPOTRegressor', code: 'TPOTRegressor(${generation}${population_size}${cv}${random_state}${etc})', + returnType: 'TPOTRegressor', options: [ { name: 'generation', component: ['input_number'], default: 100, usePair: true }, { name: 'population_size', component: ['input_number'], default: 100, usePair: true }, @@ -564,6 +603,7 @@ define([ import: 'from autosklearn.classification import AutoSklearnClassifier', link: 'https://automl.github.io/auto-sklearn/master/api.html#classification', code: 'AutoSklearnClassifier(${etc})', + returnType: 'AutoSklearnClassifier', options: [ ] @@ -573,6 +613,7 @@ define([ install: '!pip install tpot', import: 'from tpot import TPOTClassifier', code: 'TPOTClassifier(${generation}${population_size}${cv}${random_state}${etc})', + returnType: 'TPOTClassifier', options: [ { name: 'generation', component: ['input_number'], default: 100, usePair: true }, { name: 'population_size', component: ['input_number'], default: 100, usePair: true }, @@ -585,6 +626,7 @@ define([ name: 'KMeans', import: 'from sklearn.cluster import KMeans', code: 'KMeans(${n_clusters}${random_state}${etc})', + returnType: 'KMeans', options: [ { name: 'n_clusters', component: ['input_number'], default: 8, usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true } @@ -594,6 +636,7 @@ define([ name: 'AgglomerativeClustering', import: 'from sklearn.cluster import AgglomerativeClustering', code: 'AgglomerativeClustering(${n_clusters}${random_state}${etc})', + returnType: 'AgglomerativeClustering', options: [ { name: 'n_clusters', component: ['input_number'], default: 2, usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true } @@ -603,6 +646,7 @@ define([ name: 'GaussianMixture', import: 'from sklearn.mixture import GaussianMixture', code: 'GaussianMixture(${n_components}${random_state}${etc})', + returnType: 'GaussianMixture', options: [ { name: 'n_components', component: ['input_number'], default: 1, usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true } @@ -612,6 +656,7 @@ define([ name: 'DBSCAN', import: 'from sklearn.cluster import DBSCAN', code: 'DBSCAN(${eps}${min_samples}${etc})', + returnType: 'DBSCAN', options: [ { name: 'eps', component: ['input_number'], default: 0.5, usePair: true }, { name: 'min_samples', component: ['input_number'], default: 5, usePair: true } @@ -622,6 +667,7 @@ define([ name: 'PCA(Principal Component Analysis)', import: 'from sklearn.decomposition import PCA', code: 'PCA(${n_components}${random_state}${etc})', + returnType: 'PCA', options: [ { name: 'n_components', component: ['input_number'], placeholder: 'None', usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true } @@ -631,6 +677,7 @@ define([ name: 'LDA(Linear Discriminant Analysis)', import: 'from sklearn.discriminant_analysis import LinearDiscriminantAnalysis', code: 'LinearDiscriminantAnalysis(${n_components}${etc})', + returnType: 'LinearDiscriminantAnalysis', options: [ { name: 'n_components', component: ['input_number'], placeholder: 'None', usePair: true } ] @@ -639,6 +686,7 @@ define([ name: 'Truncated SVD', import: 'from sklearn.decomposition import TruncatedSVD', code: 'TruncatedSVD(${n_components}${random_state}${etc})', + returnType: 'TruncatedSVD', options: [ { name: 'n_components', component: ['input_number'], default: 2, usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true } @@ -648,6 +696,7 @@ define([ name: 'NMF(Non-Negative Matrix Factorization)', import: 'from sklearn.decomposition import NMF', code: 'NMF(${n_components}${random_state}${etc})', + returnType: 'NMF', options: [ { name: 'n_components', component: ['input_number'], placeholder: 'None', usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true } @@ -657,6 +706,7 @@ define([ name: 'TSNE(T-distributed Stochastic Neighbor Embedding)', import: 'from sklearn.manifold import TSNE', code: 'TSNE(${n_components}${learning_rate}${random_state}${etc})', + returnType: 'TSNE', options: [ { name: 'n_components', component: ['input_number'], placeholder: 'None', usePair: true }, { name: 'learning_rate', component: ['input_number'], default: 200.0, usePair: true }, @@ -667,7 +717,7 @@ define([ 'grid-search': { name: 'GridSearch', import: 'from sklearn.model_selection import GridSearchCV', - code: 'GridSearchCV(${estimator}, ${param_grid}${n_jobs}${cv}${verbose}${etc})', + returnType: 'GridSearchCV', options: [ { name: 'estimator', component: ['data_select'], placeholder: 'Select model'}, { name: 'param_grid', component: ['input'], placeholder: 'Enter parameters'}, diff --git a/visualpython/html/m_ml/pipeline.html b/visualpython/html/m_ml/pipeline.html index 464f27fa..f9d5570e 100644 --- a/visualpython/html/m_ml/pipeline.html +++ b/visualpython/html/m_ml/pipeline.html @@ -20,6 +20,10 @@
+
+
< Prev
+
Next >
+
\ No newline at end of file diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js index da0f7181..49e75265 100644 --- a/visualpython/js/m_ml/Pipeline.js +++ b/visualpython/js/m_ml/Pipeline.js @@ -107,6 +107,7 @@ define([ { name: 'ml_clustering', label: 'Clustering', useApp: true, child: ['pp_fit', 'pp_predict', 'pp_transform'] }, { name: 'pp_fit', label: 'Fit' }, { name: 'pp_predict', label: 'Predict' }, + { name: 'pp_fit_predict', label: 'Fit and Predict' }, { name: 'pp_transform', label: 'Transform' }, { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'cls' } }, ] @@ -117,7 +118,8 @@ define([ step: [ { name: 'ml_dimensionReduction', label: 'Dimension Reduction', useApp: true, child: ['pp_fit', 'pp_transform'] }, { name: 'pp_fit', label: 'Fit' }, - { name: 'pp_transform', label: 'Transform' } + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_transform', label: 'Fit and Transform' }, ] }, 'gridSearch': { @@ -184,6 +186,8 @@ define([ $(that.wrapSelector('.vp-pp-step-content')).html(''); $(that.wrapSelector('.vp-pp-step-title')).text(''); that.state.pipeline = []; + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); that.handleChangeTemplate(type); }); @@ -192,7 +196,7 @@ define([ $(document).on('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]'), function() { if (!$(this).hasClass('selected')) { let title = $(this).data('label'); - let stepSeq = parseInt($(this).data('seq')); // 0 ~ n + let stepSeq = parseInt($(this).data('step')); // 0 ~ n let name = $(this).data('name'); let ppObj = that.state.pipeline[stepSeq]; // set title @@ -210,6 +214,20 @@ define([ // check selected $(that.wrapSelector('.vp-pp-item')).removeClass('selected'); $(this).addClass('selected'); + + // check prev/next + let prevTagList = $(this).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(this).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } } }); @@ -232,6 +250,25 @@ define([ $(that.wrapSelector('.vp-pp-step-title')).text(''); $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); $(itemTag).removeClass('selected'); + + // disable prev/next + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + // check prev/next + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } } }); @@ -239,26 +276,75 @@ define([ // model type change event $(document).on('change', this.wrapSelector(`#modelType`), function() { let name = $(this).closest('.vp-pp-step-page').data('name'); - - let modelType = $(this).val(); - let modelObj = that.modelConfig[modelType]; - let modelTypeName = modelObj.code.split('(')[0]; - - that.state.modelType = modelType; - that.state.modelTypeName = modelTypeName; - - // show fit / predict / transform depends on model selection - let defaultActions = ['fit', 'predict', 'transform']; - let actions = that.modelEditor.getAction(modelTypeName); - defaultActions.forEach(actKey => { - if (actions[actKey] === undefined) { - // if undefined, hide step - $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); - } else { - $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).show(); + let modelCreatorList = ['ml_dataPrep', 'ml_regression', 'ml_classification', 'ml_clustering', 'ml_dimensionReduction', 'ml_gridSearch']; + if (modelCreatorList.includes(name)) { + let modelType = $(this).val(); + let modelObj = that.modelConfig[modelType]; + let modelTypeName = modelObj.returnType; + + that.state.modelType = modelType; + that.state.modelTypeName = modelTypeName; + + // show fit / predict / transform depends on model selection + let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform']; + let actions = that.modelEditor.getAction(modelTypeName); + defaultActions.forEach(actKey => { + if (actions[actKey] === undefined) { + // if undefined, hide step + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).show(); + if (actKey === 'fit_predict') { + if (actions['fit'] === undefined || actions['predict'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_predict"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_transform') { + if (actions['fit'] === undefined || actions['transform'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } + } + $(that.wrapSelector('.vp-pp-item')).removeClass('vp-last-visible'); + $(that.wrapSelector('.vp-pp-item:visible:last')).addClass('vp-last-visible'); + }); + + // change evaluation page's default selection + if (name === 'ml_gridSearch') { + let modelCategory = that.modelEditor.getModelCategory(modelTypeName); // Regression / Classification / Clustering + let evalModelTypeValue = 'rgs'; + switch (modelCategory) { + case 'Regression': + evalModelTypeValue = 'rgs'; + break; + case 'Classification': + evalModelTypeValue = 'clf'; + break; + case 'Clustering': + evalModelTypeValue = 'cls'; + break; + } + let evalPageTag = $(that.wrapSelector('.vp-pp-step-page[data-name="ml_evaluation"]')); + if (evalPageTag.is(':empty') === true) { + // evaluation page is not opened yet + // set state for evaluation page + let stepSeq = $(evalPageTag).data('step'); + let ppObj = that.state.pipeline[stepSeq]; + if (ppObj.app) { + ppObj.app.state.modelType = evalModelTypeValue; + } + } else { + let evalModelTypeTag = $(evalPageTag).find('#modelType'); + $(evalModelTypeTag).val(evalModelTypeValue); + $(evalModelTypeTag).trigger('change'); + } } - }); - + } }); // model allocation variable change @@ -267,6 +353,20 @@ define([ let modelAllocation = $(this).val(); that.state.model = modelAllocation; }); + + // click prev button + $(this.wrapSelector('.vp-pp-step-prev:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(prevTagList[0]).trigger('click'); + }); + + // click next button + $(this.wrapSelector('.vp-pp-step-next:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible') + $(nextTagList[0]).trigger('click'); + }); } /** @@ -284,7 +384,7 @@ define([ // load pipeline items tplObj.step.forEach((stepObj, idx) => { let { name, label, useApp=false, child=[], state={} } = stepObj; - ppTag.appendFormatLine(`
+ ppTag.appendFormatLine(`
{3}
@@ -354,7 +454,8 @@ define([ that.handleAppView(name, mlComponent); // select first step - $(that.wrapSelector('.vp-pp-item[data-seq="0"]')).click(); + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); // end of DUP AREA: pp-1 } }); @@ -390,7 +491,8 @@ define([ that.handleAppView(name, mlComponent); // select first step - $(that.wrapSelector('.vp-pp-item[data-seq="0"]')).click(); + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); // end of DUP AREA: pp-1 } }) @@ -436,6 +538,9 @@ define([ // load state if (this.state.templateType !== '') { $(this.wrapSelector('#templateType')).trigger('change'); + } else { + $(this.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(this.wrapSelector('.vp-pp-step-next')).addClass('disabled'); } } @@ -469,6 +574,12 @@ define([ case 'pp_transform': tag = this.templateForOptionPage(actions['transform']); break; + case 'pp_fit_predict': + tag = this.templateForOptionPage(actions['fit_predict']); + break; + case 'pp_fit_transform': + tag = this.templateForOptionPage(actions['fit_transform']); + break; } $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(`
${tag}
@@ -505,8 +616,8 @@ define([ var { name, label, useApp, app } = ppObj; let requiredList = []; result = true; - let isVisible = $(that.wrapSelector(`.vp-pp-item[data-seq="${idx}"]`)).is(':visible') === true; - let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-seq="${idx}"]`)).attr('data-flag') === 'enabled'; + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; if (isVisible && isEnabled) { switch (name) { case 'ml_dataSplit': @@ -563,6 +674,12 @@ define([ case 'pp_transform': actObj = actions['transform']; break; + case 'pp_fit_predict': + actObj = actions['fit_predict']; + break; + case 'pp_fit_transform': + actObj = actions['fit_transform']; + break; } let code = new com_String(); @@ -597,8 +714,8 @@ define([ let { name, label, useApp, app } = ppObj; // check disabled - let isVisible = $(that.wrapSelector(`.vp-pp-item[data-seq="${idx}"]`)).is(':visible') === true; - let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-seq="${idx}"]`)).attr('data-flag') === 'enabled'; + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; if (isVisible && isEnabled) { if (code.toString() !== '') { code.appendLine(); From 1aa5cf8eb20f84e23ee9cd4b3af4f84ff2b8e0ae Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:02:37 +0900 Subject: [PATCH 02/26] Add scoring option to GridSearch --- visualpython/data/m_ml/mlLibrary.js | 2 + visualpython/html/m_ml/gridSearch.html | 2 +- visualpython/js/m_ml/GridSearch.js | 62 +++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/visualpython/data/m_ml/mlLibrary.js b/visualpython/data/m_ml/mlLibrary.js index beeae0ec..5268cdb4 100644 --- a/visualpython/data/m_ml/mlLibrary.js +++ b/visualpython/data/m_ml/mlLibrary.js @@ -717,10 +717,12 @@ define([ 'grid-search': { name: 'GridSearch', import: 'from sklearn.model_selection import GridSearchCV', + code: 'GridSearchCV(${estimator}, ${param_grid}${scoring}${n_jobs}${cv}${verbose}${etc})', returnType: 'GridSearchCV', options: [ { name: 'estimator', component: ['data_select'], placeholder: 'Select model'}, { name: 'param_grid', component: ['input'], placeholder: 'Enter parameters'}, + { name: 'scoring', component: ['input'], placeholder: 'None', usePair: true }, // https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter { name: 'n_jobs', component: ['input'], placeholder: 'None', usePair: true }, { name: 'cv', component: ['input'], placeholder: 'None', usePair: true }, { name: 'verbose', component: ['input_number'], placeholder: 'Input number', usePair: true } diff --git a/visualpython/html/m_ml/gridSearch.html b/visualpython/html/m_ml/gridSearch.html index 976cd69c..ee276db9 100644 --- a/visualpython/html/m_ml/gridSearch.html +++ b/visualpython/html/m_ml/gridSearch.html @@ -22,7 +22,6 @@
-
@@ -36,6 +35,7 @@
+
\ No newline at end of file diff --git a/visualpython/js/m_ml/GridSearch.js b/visualpython/js/m_ml/GridSearch.js index dd1a1eb5..32dbc967 100644 --- a/visualpython/js/m_ml/GridSearch.js +++ b/visualpython/js/m_ml/GridSearch.js @@ -81,6 +81,8 @@ define([ that.hideInstallButton(); } + that.handleScoringOptions(modelType); + // reset model param set $(that.wrapSelector('.vp-param-grid-box')).html(''); $(that.wrapSelector('.vp-param-grid-box')).html(that.templateForParamSet()); @@ -140,6 +142,10 @@ define([ let parentTag = $(thisTag).parent(); let paramIsText = $(parentTag).find('.vp-param-val').data('type') === 'text'; // text / var let paramVal = $(parentTag).find('.vp-param-val').val(); + let reservedKeywordList = ['None', 'True', 'False', 'np.nan', 'np.NaN']; + if (reservedKeywordList.includes(paramVal)) { + paramIsText = false; + } // check , and split it let paramSplit = paramVal.split(','); paramSplit && paramSplit.forEach(val => { @@ -195,6 +201,58 @@ define([ }); } + handleScoringOptions(modelType) { + let options = { + 'Classification': [ + "'accuracy'", + "'balanced_accuracy'", + "'top_k_accuracy'", + "'average_precision'", + "'neg_brier_score'", + "'f1'", + "'f1_micro'", + "'f1_macro'", + "'f1_weighted'", + "'f1_samples'", + "'neg_log_loss'", + "'precision' etc.", + "'recall' etc.", + "'jaccard' etc.", + "'roc_auc'", + "'roc_auc_ovr'", + "'roc_auc_ovo'", + "'roc_auc_ovr_weighted'", + "'roc_auc_ovo_weighted'", + ], + 'Regression': [ + "'explained_variance'", + "'max_error'", + "'neg_mean_absolute_error'", + "'neg_mean_squared_error'", + "'neg_root_mean_squared_error'", + "'neg_mean_squared_log_error'", + "'neg_median_absolute_error'", + "'r2'", + "'neg_mean_poisson_deviance'", + "'neg_mean_gamma_deviance'", + "'neg_mean_absolute_percentage_error'", + "'d2_absolute_error_score'", + "'d2_pinball_score'", + "'d2_tweedie_score'" + ] + } + let modelCategory = this.modelTypeList['Regression'].includes(modelType)?'Regression':'Classification'; + + // Set suggestInput on scoring option + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('scoring'); + suggestInput.setPlaceholder('Select option'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return options[modelCategory]; }); + suggestInput.setNormalFilter(true); + $(this.wrapSelector('#scoring')).replaceWith(suggestInput.toTagString()); + } + templateForParamSet() { let paramSetNo = 1; // set param set number @@ -378,6 +436,7 @@ define([ let thisTag = $(that.wrapSelector('.' + suggestInput.uuid)); that.handleAddParamValue($(thisTag)); $(thisTag).val(''); + return false; }); paramSet.appendLine(suggestInput.toTagString()); } @@ -394,6 +453,8 @@ define([ // Model Editor this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + this.handleScoringOptions(this.state.modelType); } generateInstallCode() { @@ -432,7 +493,6 @@ define([ state['estimator'] = estimator; state['param_grid'] = '{}'; - let reservedKeywordList = ['None', 'True', 'False', 'np.nan', 'np.NaN']; let paramGrid = []; // generate param_grid $(this.wrapSelector('.vp-param-set-box')).each((i, tag) => { From d3434e9b0b2a98b62829e698313e8b70bc854bf4 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:03:28 +0900 Subject: [PATCH 03/26] Edit SaveLoad to show model types for target and Fixed labels --- visualpython/data/m_ml/mlLibrary.js | 12 ++++++------ visualpython/js/m_ml/SaveLoad.js | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/visualpython/data/m_ml/mlLibrary.js b/visualpython/data/m_ml/mlLibrary.js index 5268cdb4..e5bf2205 100644 --- a/visualpython/data/m_ml/mlLibrary.js +++ b/visualpython/data/m_ml/mlLibrary.js @@ -732,19 +732,19 @@ define([ 'model_save': { name: 'Save model', import: 'import joblib', - code: "joblib.dump(${target}, ${savePath}${etc})", + code: "joblib.dump(${target}, ${save_path}${etc})", options: [ - { name: 'target', component: ['data_select'], placeholder: 'Select model'}, - { name: 'savePath', component: ['file-save'], placeholder: 'Select path'} + { name: 'target', component: ['data_select'], comp_type: 'ml', placeholder: 'Select model'}, + { name: 'save_path', component: ['file-save'], placeholder: 'Select path'} ] }, 'model_load': { name: 'Load model', import: 'import joblib', - code: "${allocateTo} = joblib.load(${loadPath}${etc})", + code: "${allocate_to} = joblib.load(${load_path}${etc})", options: [ - { name: 'loadPath', component: ['file-open'], placeholder: 'Select path'}, - { name: 'allocateTo', component: ['data_select'], placeholder: 'New variable to load'} + { name: 'load_path', component: ['file-open'], placeholder: 'Select path'}, + { name: 'allocate_to', component: ['input'], placeholder: 'New variable to load'} ] } } diff --git a/visualpython/js/m_ml/SaveLoad.js b/visualpython/js/m_ml/SaveLoad.js index 0940b028..454f468c 100644 --- a/visualpython/js/m_ml/SaveLoad.js +++ b/visualpython/js/m_ml/SaveLoad.js @@ -86,7 +86,7 @@ define([ // render tag config.options.forEach(opt => { optBox.appendFormatLine('' - , opt.name, opt.name, opt.name); + , opt.name, opt.name, com_util.optionToLabel(opt.name)); let content = com_generator.renderContent(this, opt.component[0], opt, state); optBox.appendLine(content[0].outerHTML); }); From d4e9c346f51c01f4ddcc09e71e216a81cdfe0eea Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:05:43 +0900 Subject: [PATCH 04/26] Edit Codeview, DataView not to save its content after hide --- .../html/component/popupComponent.html | 4 +- .../js/com/component/PopupComponent.js | 48 +++++++++++++++++-- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/visualpython/html/component/popupComponent.html b/visualpython/html/component/popupComponent.html index 825a51c9..4c1ddec2 100644 --- a/visualpython/html/component/popupComponent.html +++ b/visualpython/html/component/popupComponent.html @@ -91,11 +91,11 @@
-
+
-
+
diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index ff48a5bd..a4d5950d 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -308,6 +308,22 @@ define([ _bindEvent() { var that = this; + // Popup click / focus event + $(this.wrapSelector()).on('click focus', function(evt) { + // Close on blur + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-codeview-box') + && $(that.wrapSelector('.vp-popup-codeview-box')).find(evt.target).length == 0) { + that.closeView('code'); + } + } + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-dataview-box') + && $(that.wrapSelector('.vp-popup-dataview-box')).find(evt.target).length == 0) { + that.closeView('data'); + } + } + }); // Close popup event $(this.wrapSelector('.vp-popup-close')).on('click', function(evt) { if (that.getTaskType() === 'task') { @@ -514,6 +530,14 @@ define([ } }); + // blur on code, dataview + $(this.wrapSelector('.vp-popup-codeview-box')).on('hide', function() { + that.closeView('code'); + }); + $(this.wrapSelector('.vp-popup-dataview-box')).on('hide', function() { + that.closeView('data'); + }); + // focus on data selector input $(this.wrapSelector('.vp-data-selector')).on('focus', function(evt) { @@ -1154,20 +1178,36 @@ define([ setTimeout(function() { that.cmCodeview.refresh(); }, 1); - $(this.wrapSelector('.vp-popup-dataview-box')).hide(); + this.closeView('data'); $(this.wrapSelector('.vp-popup-codeview-box')).show(); } else if (viewType === 'data') { this.renderDataView(); - $(this.wrapSelector('.vp-popup-codeview-box')).hide(); + this.closeView('code'); $(this.wrapSelector('.vp-popup-dataview-box')).show(); } else if (viewType === 'help') { - $(this.wrapSelector('.vp-popup-codeview-box')).hide(); - $(this.wrapSelector('.vp-popup-dataview-box')).hide(); + this.closeView('code'); + this.closeView('data'); this.openHelpView(); } } closeView(viewType) { + if (viewType === 'code') { + // reset codeview + if (this.cmCodeview) { + this.cmCodeview.setValue(''); + this.cmCodeview.save(); + + var that = this; + setTimeout(function() { + that.cmCodeview.refresh(); + }, 1); + } + } else if (viewType === 'data') { + // reset dataview + $(this.wrapSelector('.vp-popup-dataview-box')).html(''); + } + // hide view $(this.wrapSelector('.vp-popup-'+viewType+'view-box')).hide(); } From 5fee3528f60c32cdde8bc17e4d42908e98aff989 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:07:37 +0900 Subject: [PATCH 05/26] Fix not to save app's state on task which is not intended --- visualpython/js/com/component/PopupComponent.js | 15 ++++++++++----- visualpython/js/menu/TaskItem.js | 5 +---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index a4d5950d..759f31bd 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -342,10 +342,7 @@ define([ // Toggle operation (minimize) $(this.wrapSelector('.vp-popup-toggle')).on('click', function(evt) { evt.stopPropagation(); - $(that.eventTarget).trigger({ - type: 'close_option_page', - component: that - }); + that.hide(); }); // Maximize operation $(this.wrapSelector('.vp-popup-maximize')).on('click', function(evt) { @@ -1071,7 +1068,15 @@ define([ close() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'close popup', this); this.saveState(); - this.hide(); + if (this.getTaskType() === 'task') { + $(this.eventTarget).trigger({ + type: 'remove_option_page', + component: this + }); + } else { + // if it's block, just hide it + this.hide(); + } } save() { diff --git a/visualpython/js/menu/TaskItem.js b/visualpython/js/menu/TaskItem.js index dcb63181..c8e2be64 100644 --- a/visualpython/js/menu/TaskItem.js +++ b/visualpython/js/menu/TaskItem.js @@ -39,10 +39,7 @@ define([ if (isOpen) { // hide task if it's already opened // open task - $('#vp_wrapper').trigger({ - type: 'close_option_page', - component: that.state.task - }); + that.state.task.hide(); } else { // open task $('#vp_wrapper').trigger({ From 8f98b057011a4ddcf5fd66897412f9b44500bb50 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:09:33 +0900 Subject: [PATCH 06/26] Add col_multi component to autoGen and apply it to by option in sort_values --- visualpython/data/m_library/pandasLibrary.js | 5 +- visualpython/js/com/com_generatorV2.js | 88 +++++++++++++++++-- .../js/com/component/MultiSelector.js | 2 +- 3 files changed, 86 insertions(+), 9 deletions(-) diff --git a/visualpython/data/m_library/pandasLibrary.js b/visualpython/data/m_library/pandasLibrary.js index 8491177b..315290fa 100644 --- a/visualpython/data/m_library/pandasLibrary.js +++ b/visualpython/data/m_library/pandasLibrary.js @@ -1916,7 +1916,10 @@ define([ { "name": "by", "label": "Sort By", - "usePair": true + "usePair": true, + "component": [ + "col_multi" + ] }, { "name": "axis", diff --git a/visualpython/js/com/com_generatorV2.js b/visualpython/js/com/com_generatorV2.js index 13cce5e5..d8cb8cf8 100644 --- a/visualpython/js/com/com_generatorV2.js +++ b/visualpython/js/com/com_generatorV2.js @@ -14,8 +14,9 @@ define([ 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/com/component/FileNavigation' -], function (com_util, com_makeDom, SuggestInput, VarSelector2, DataSelector, FileNavigation) { +], function (com_util, com_makeDom, SuggestInput, VarSelector2, DataSelector, MultiSelector, FileNavigation) { /** * show result after code executed */ @@ -39,6 +40,7 @@ define([ 'var_select': 'Select Variable', 'var_multi': 'Select N-Variables', 'col_select': 'Select Column', + 'col_multi': 'Select Columns', 'textarea': 'Input textarea', 'input_number': 'Input number', 'input_text': 'Input text', @@ -149,6 +151,7 @@ define([ } + bindMultiSelector(pageThis); bindAutoComponentEvent(pageThis); } @@ -174,7 +177,7 @@ define([ } else if (output === true) { requiredFontStyle = 'vp-bold'; } - var lblTag = $(``).attr({ + var lblTag = $(``).attr({ 'for': name, 'class': requiredFontStyle, 'title': '(' + name + ')' @@ -315,7 +318,8 @@ define([ case 'option_select': var optSlct = $('').attr({ 'class':'vp-select option-select vp-state', - 'id':obj.name + 'id':obj.name, + 'title': (obj.help==undefined?'':obj.help), }); obj.options.forEach((opt, idx, arr) => { var label = (obj.options_label != undefined? obj.options_label[idx]:opt); @@ -368,6 +372,7 @@ define([ let dataSelector = new DataSelector({ pageThis: pageThis, id: obj.name, + type: obj.comp_type || 'data', allowDataType: obj.var_type, placeholder: obj.placeholder || 'Select data', value: value, @@ -382,7 +387,8 @@ define([ id: obj.name, class: 'vp-input vp-state', placeholder: obj.placeholder || 'Select data', - required: obj.required === true + required: obj.required === true, + title: (obj.help==undefined?'':obj.help), }); vp_generateVarSuggestInput(pageThis.wrapSelector(), obj); content = tag; @@ -393,7 +399,8 @@ define([ 'id': obj.name, 'class': 'vp-select var-multi vp-state', // multiple selection true - 'multiple': true + 'multiple': true, + 'title': (obj.help==undefined?'':obj.help), }); vp_generateVarSelect(tag, obj.var_type, obj.value); content = tag; @@ -402,10 +409,28 @@ define([ var tag = $('').attr({ 'type': 'text', 'id': obj.name, - 'class': 'vp-input vp-state' + 'class': 'vp-input vp-state', + 'title': (obj.help==undefined?'':obj.help), }); content = tag; break; + case 'col_multi': + var tag1 = $('
').attr({ + 'id': obj.name + '_auto_comp', + 'class': 'vp-auto-multi-column', + 'data-id': obj.name, + 'data-target': obj.comp_target || 'i0', + 'style': 'height: 150px;', + 'title': (obj.help==undefined?'':obj.help), + }); + var tag2 = $('').attr({ + type: 'hidden', + id: obj.name, + class: 'vp-input vp-state', + }) + var tag = $('
').append(tag1).append(tag2); + content = tag; + break; case 'textarea': var textarea = $(``); // cell metadata test @@ -710,6 +735,12 @@ define([ case 'dtype': value = $(pageThis.wrapSelector(parent + ' #'+obj.name)).val(); break; + case 'col_multi': + let colList = pageThis.autoGen[obj.name].getDataList(); + pageThis.state[obj.name] = colList.map(data => { return data.code }); + value = colList.map(data => { return data.code }).join(','); + $(pageThis.wrapSelector('#'+obj.name)).val(value); + break; case 'file-open': case 'file-save': case 'table': @@ -1236,6 +1267,50 @@ define([ return $(''); } + var bindMultiSelector = function(pageThis) { + //==================================================================== + // for column multi selector + //==================================================================== + $(pageThis.wrapSelector('.vp-auto-multi-column')).each((idx, tag) => { + let compId = tag.id; + let id = $(tag).data('id'); + let targetId = $(tag).data('target'); + let colSelector = new MultiSelector( + pageThis.wrapSelector('#' + compId), + { mode: 'columns', parent: (pageThis.state[targetId] || ''), selectedList: pageThis.state[compId] } + ); + pageThis.autoGen = { + [id]: colSelector, + ...pageThis.autoGen + }; + $(pageThis.wrapSelector('#' + targetId)).on('change', function() { + let targetVariable = $(this).val(); + let colSelector = new MultiSelector( + pageThis.wrapSelector('#' + compId), + { + mode: 'columns', parent: targetVariable, selectedList: pageThis.state[compId], + change: function(type, list) { + let value = list.map(data => { return data.code }).join(','); + if (list.length == 0) { + value = ''; + } else if (list.length > 1) { + value = '[' + value + ']'; + } + pageThis.state[compId] = list.map(data => { return data.code }); + pageThis.state[id] = value; + $(pageThis.wrapSelector('#'+id)).val(value); + } + }, + ); + pageThis.autoGen = { + [id]: colSelector, + ...pageThis.autoGen + }; + }); + + }); + } + var bindAutoComponentEvent = function(pageThis) { let selector = pageThis.wrapSelector(); // Auto-component selector @@ -1499,7 +1574,6 @@ define([ $(pageThis.wrapSelector('#'+id)).val(code); }); - //==================================================================== // Event for tabBlock //==================================================================== diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index 516365f8..9ec8ae02 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -90,7 +90,7 @@ define([ var that = this; - if (parent === '') { + if (parent == null || parent === '' || (Array.isArray(parent) && parent.length == 0)) { this._executeCallback([]); return; } From 6db2412e36fa224ced43712100f46a6f2ad20695 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:12:17 +0900 Subject: [PATCH 07/26] Fix not to add block by default --- .../js/com/component/PackageManager.js | 52 +++++++++++-------- visualpython/js/m_apps/Information.js | 21 ++++---- visualpython/js/m_apps/Snippets.js | 19 ++++--- visualpython/js/m_visualize/Seaborn.js | 19 ++++--- 4 files changed, 64 insertions(+), 47 deletions(-) diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js index 884b76a6..6907a8d1 100644 --- a/visualpython/js/com/component/PackageManager.js +++ b/visualpython/js/com/component/PackageManager.js @@ -17,11 +17,12 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(ifHtml, ifCss, com_util, com_Const, com_String, SuggestInput, PopupComponent, FileNavigation, LoadingSpinner) { +], function(ifHtml, ifCss, com_util, com_Const, com_String, com_interface, SuggestInput, PopupComponent, FileNavigation, LoadingSpinner) { /** * PackageManager @@ -197,28 +198,32 @@ define([ if (vpConfig.extensionType === 'lite') { code = com_util.formatString("%pip uninstall {0}", pipName); } + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); } else if (menu === 'upgrade') { var pipName = that.packageLib[key].pipName; var code = com_util.formatString("!pip install --upgrade {0}", pipName); if (vpConfig.extensionType === 'lite') { code = com_util.formatString("%pip install {0}", pipName); } + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); } else if (menu === 'delete') { $(item).remove(); delete that.packageLib[key]; @@ -331,14 +336,17 @@ define([ return false; } } + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + break; } diff --git a/visualpython/js/m_apps/Information.js b/visualpython/js/m_apps/Information.js index ff310874..145a9e1a 100644 --- a/visualpython/js/m_apps/Information.js +++ b/visualpython/js/m_apps/Information.js @@ -17,10 +17,11 @@ define([ __VP_CSS_LOADER__('vp_base/css/m_apps/information'), // INTEGRATION: unified version of css loader 'vp_base/js/com/com_String', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/DataSelector', 'vp_base/js/com/component/LoadingSpinner' -], function(varHtml, varCss, com_String, com_util, PopupComponent, DataSelector, LoadingSpinner) { +], function(varHtml, varCss, com_String, com_util, com_interface, PopupComponent, DataSelector, LoadingSpinner) { /** * Information @@ -406,15 +407,17 @@ define([ // click run button $(this.wrapSelector('.vp-information-run-button')).click(function(event) { // get code - var code = that.generateCode(); + var code = that.generateCodeForInfo(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Data Info'); }); } diff --git a/visualpython/js/m_apps/Snippets.js b/visualpython/js/m_apps/Snippets.js index 3c031163..fb3d5f21 100644 --- a/visualpython/js/m_apps/Snippets.js +++ b/visualpython/js/m_apps/Snippets.js @@ -18,10 +18,11 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(snHtml, snCss, com_util, com_Const, com_String, PopupComponent, FileNavigation, LoadingSpinner) { +], function(snHtml, snCss, com_util, com_Const, com_String, com_interface, PopupComponent, FileNavigation, LoadingSpinner) { /** * Snippets @@ -471,14 +472,16 @@ define([ let cmCode = that.codemirrorList[title]; cmCode.save(); var code = cmCode.getValue(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Snippets'); } else if (menu == 'duplicate') { var dupNo = 1; var timestamp = new Date().getTime(); diff --git a/visualpython/js/m_visualize/Seaborn.js b/visualpython/js/m_visualize/Seaborn.js index 7bb2759d..0554276a 100644 --- a/visualpython/js/m_visualize/Seaborn.js +++ b/visualpython/js/m_visualize/Seaborn.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/data/m_visualize/seabornLibrary', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_generator, com_util, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { +], function(chartHTml, chartCss, com_String, com_generator, com_util, com_interface, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { class Seaborn extends PopupComponent { _init() { @@ -770,14 +771,16 @@ define([ handleInnerOk() { // generateImportCode var code = this.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Seaborn'); this.closeInnerPopup(); From 31b28a1e467b18c731c27be0c0d2c294bcb26a53 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:12:48 +0900 Subject: [PATCH 08/26] Add display() to evaluation --- visualpython/js/m_ml/evaluation.js | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/visualpython/js/m_ml/evaluation.js b/visualpython/js/m_ml/evaluation.js index 6b36f6ba..138c0ef7 100644 --- a/visualpython/js/m_ml/evaluation.js +++ b/visualpython/js/m_ml/evaluation.js @@ -199,6 +199,7 @@ define([ sizeOfClusters, silhouetteScore, ari, nmi, clusteredIndex, featureData2, targetData2 } = this.state; + let needDisplay = false; //==================================================================== // Classfication @@ -207,7 +208,8 @@ define([ if (confusion_matrix) { code = new com_String(); code.appendLine("# Confusion Matrix"); - code.appendFormat('pd.crosstab({0}, {1}, margins=True)', targetData, predictData); + code.appendFormat('display(pd.crosstab({0}, {1}, margins=True))', targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (report) { @@ -219,25 +221,29 @@ define([ if (accuracy) { code = new com_String(); code.appendLine("# Accuracy"); - code.appendFormat('metrics.accuracy_score({0}, {1})', targetData, predictData); + code.appendFormat('display(metrics.accuracy_score({0}, {1}))', targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (precision) { code = new com_String(); code.appendLine("# Precision"); - code.appendFormat("metrics.precision_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("display(metrics.precision_score({0}, {1}, average='weighted'))", targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (recall) { code = new com_String(); code.appendLine("# Recall"); - code.appendFormat("metrics.recall_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("display(metrics.recall_score({0}, {1}, average='weighted'))", targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (f1_score) { code = new com_String(); code.appendLine("# F1-score"); - code.appendFormat("metrics.f1_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("display(metrics.f1_score({0}, {1}, average='weighted'))", targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } // if (roc_curve) { @@ -272,13 +278,15 @@ define([ if (r_squared) { code = new com_String(); code.appendLine("# R square"); - code.appendFormat('metrics.r2_score({0}, {1})', targetData, predictData); + code.appendFormat('display(metrics.r2_score({0}, {1}))', targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (mae) { code = new com_String(); code.appendLine("# MAE(Mean Absolute Error)"); - code.appendFormat('metrics.mean_absolute_error({0}, {1})', targetData, predictData); + code.appendFormat('display(metrics.mean_absolute_error({0}, {1}))', targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (mape) { @@ -287,13 +295,15 @@ define([ code.appendLine('def MAPE(y_test, y_pred):'); code.appendLine(' return np.mean(np.abs((y_test - pred) / y_test)) * 100'); code.appendLine(); - code.appendFormat('MAPE({0}, {1})', targetData, predictData); + code.appendFormat('display(MAPE({0}, {1}))', targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (rmse) { code = new com_String(); code.appendLine("# RMSE(Root Mean Squared Error)"); - code.appendFormat('metrics.mean_squared_error({0}, {1})**0.5', targetData, predictData); + code.appendFormat('display(metrics.mean_squared_error({0}, {1})**0.5)', targetData, predictData); + needDisplay = true; codeCells.push(code.toString()); } if (scatter_plot) { @@ -333,6 +343,12 @@ define([ codeCells.push(code.toString()); } } + if (needDisplay === true) { + codeCells = [ + "from IPython.display import display", + ...codeCells + ]; + } // return as seperated cells return codeCells; } From e93adcf019dcfc67ce7176eec4fdabf271becdc6 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:13:28 +0900 Subject: [PATCH 09/26] Fix PackageManager to load package after install or uninstall it --- .../js/com/component/PackageManager.js | 19 ++++++++++++------- visualpython/python/variableCommand.py | 1 + 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js index 6907a8d1..00afe14c 100644 --- a/visualpython/js/com/component/PackageManager.js +++ b/visualpython/js/com/component/PackageManager.js @@ -52,19 +52,19 @@ define([ 'plotly': { pipName: 'plotly' }, 'wordcloud': { pipName: 'wordcloud' }, 'sklearn': { pipName: 'scikit-learn' }, - 'scikit-posthocs': { pipName: 'scikit-posthocs' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, 'scipy': { pipName: 'scipy' }, 'statsmodels': { pipName: 'statsmodels' }, - 'factor-analyzer': { pipName: 'factor-analyzer' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, 'pingouin': { pipName: 'pingouin' }, 'category_encoders': { pipName: 'category_encoders' }, 'imblearn': { pipName: 'imblearn' }, 'xgboost': { pipName: 'xgboost' }, 'lightgbm': { pipName: 'lightgbm' }, 'catboost': { pipName: 'catboost' }, - 'auto-sklearn': { pipName: 'auto-sklearn' }, + 'autosklearn': { pipName: 'auto-sklearn' }, 'tpot': { pipName: 'tpot' }, - 'PyMuPDF': { pipName: 'PyMuPDF' }, + 'pymupdf': { pipName: 'pymupdf' }, 'sweetviz': { pipName: 'sweetviz' }, } @@ -76,16 +76,16 @@ define([ 'seaborn': { pipName: 'seaborn' }, 'plotly': { pipName: 'plotly' }, 'sklearn': { pipName: 'scikit-learn' }, - 'scikit-posthocs': { pipName: 'scikit-posthocs' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, 'scipy': { pipName: 'scipy' }, 'statsmodels': { pipName: 'statsmodels' }, - 'factor-analyzer': { pipName: 'factor-analyzer' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, 'category_encoders': { pipName: 'category_encoders' }, 'imblearn': { pipName: 'imblearn' }, 'xgboost': { pipName: 'xgboost' }, 'lightgbm': { pipName: 'lightgbm' }, 'catboost': { pipName: 'catboost' }, - 'auto-sklearn': { pipName: 'auto-sklearn' }, + 'autosklearn': { pipName: 'auto-sklearn' }, 'sweetviz': { pipName: 'sweetviz' }, } } @@ -208,6 +208,7 @@ define([ // afterAction: 'run' // }); com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); } else if (menu === 'upgrade') { var pipName = that.packageLib[key].pipName; var code = com_util.formatString("!pip install --upgrade {0}", pipName); @@ -224,11 +225,13 @@ define([ // afterAction: 'run' // }); com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); } else if (menu === 'delete') { $(item).remove(); delete that.packageLib[key]; vpConfig.removeData('packageList', 'vppackman').then(function() { vpConfig.setData({ 'packageList': that.packageLib }, 'vppackman'); + that.loadPackageList(); }); } evt.stopPropagation(); @@ -347,6 +350,8 @@ define([ // }); com_interface.insertCell('code', code, true, 'Package Manager'); + // load package list + this.loadPackageList(); break; } diff --git a/visualpython/python/variableCommand.py b/visualpython/python/variableCommand.py index 6069dda9..1fea9968 100644 --- a/visualpython/python/variableCommand.py +++ b/visualpython/python/variableCommand.py @@ -136,6 +136,7 @@ def _vp_check_package_list(pack_list): for pack in pack_list: try: _vp_pack = _vp_ilib.import_module(pack) + _vp_ilib.reload(_vp_pack) _pack_info[pack] = { 'name': _vp_pack.__name__, 'installed': True, 'version': _vp_pack.__version__, 'path': _vp_pack.__path__ } except: _pack_info[pack] = { 'name': pack, 'installed': False } From 4844f881200f060ab1fe76650d6476e798ad8f86 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:14:07 +0900 Subject: [PATCH 10/26] Edit DataInfo to show simple value_counts code for single column --- visualpython/js/m_apps/Information.js | 37 +++++++++++++++++++-------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/visualpython/js/m_apps/Information.js b/visualpython/js/m_apps/Information.js index 145a9e1a..6baa701a 100644 --- a/visualpython/js/m_apps/Information.js +++ b/visualpython/js/m_apps/Information.js @@ -104,6 +104,7 @@ define([ \n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'category'): _value_counts.index})], axis=1)\ \n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'count'): _value_counts.values})], axis=1)\ \n_dfr.replace(np.nan,'')", + code2: "${data}.value_counts()", dtype: ['DataFrame', 'Series'], toframe: true }, ] }, @@ -429,6 +430,7 @@ define([ this.state.selected = []; this.state.selection = { start: -1, end: -1 }; this.renderMenu(); + $(this.wrapSelector('.' + VP_FE_TABLE)).html(''); this.loadCode(data); this.loadInfo(data, this.state.menu); } @@ -534,6 +536,10 @@ define([ } generateCode() { + return this.tempCode; + } + + generateCodeForInfo() { let { data, dtype, menu, menuItem } = this.state; var selected = []; @@ -600,18 +606,26 @@ define([ // only one method selected if (menuItem.length > 0 && infoObj.child) { let childObj = infoObj.child.find(obj=>obj.id === menuItem[0]); - if (childObj.toframe === true) { - if (dtype === 'Series') { - dataVar = new com_String(); - dataVar.appendFormat("{0}.to_frame()", data); - currentDtype = 'DataFrame'; - } else if (currentDtype === 'Series') { - dataVar = new com_String(); - dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); - currentDtype = 'DataFrame'; + if (menuItem[0] === 'value_counts') { + if (currentDtype === 'Series') { + codePattern = childObj.code2; + } else { + codePattern = childObj.code; + } + } else { + if (childObj.toframe === true) { + if (dtype === 'Series') { + dataVar = new com_String(); + dataVar.appendFormat("{0}.to_frame()", data); + currentDtype = 'DataFrame'; + } else if (currentDtype === 'Series') { // DataFrame with single column selected + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } } + codePattern = childObj.code; } - codePattern = childObj.code; } else { codePattern = infoObj.code; } @@ -841,7 +855,8 @@ define([ // load preview content let $infoPreviewTag = $(this.wrapSelector('#informationPreview')); $infoPreviewTag.html(''); - let code = this.generateCode(); + let code = this.generateCodeForInfo(); + this.tempCode = code; // use default pandas option // let defaultPOCode = new com_String(); From 64585de47f7a601fd4678059ba05e1c22d5a3401 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:15:20 +0900 Subject: [PATCH 11/26] Fix task bluring, focusing popup issue on Popup --- visualpython/js/com/component/PopupComponent.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 759f31bd..78a81e29 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -1046,7 +1046,7 @@ define([ this._bindCodemirror(); - if (targetFrame !== undefined) { + if (targetFrame == undefined) { $(this.eventTarget).trigger({ type: 'focus_option_page', component: this @@ -1112,6 +1112,10 @@ define([ blur() { $(this.wrapSelector()).removeClass('vp-focused'); + // blur on its block + if (this.taskItem) { + this.taskItem.blurItem(); + } } show() { From c83e8fb2c0906d16d003caa25217a4a2450099be Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:15:44 +0900 Subject: [PATCH 12/26] Add helpviewer to include import codes --- visualpython/js/com/component/HelpViewer.js | 8 ++++++-- visualpython/js/com/component/PopupComponent.js | 13 ++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/visualpython/js/com/component/HelpViewer.js b/visualpython/js/com/component/HelpViewer.js index ac8a8e68..84ccfe44 100644 --- a/visualpython/js/com/component/HelpViewer.js +++ b/visualpython/js/com/component/HelpViewer.js @@ -167,14 +167,18 @@ define([ return sbSelector.toString(); } - open(content, useHelp=true) { + open(content, useHelp=true, importCode='') { this.show(); let that = this; let code = content; if (useHelp === true) { - code = `print(help(${content}))`; + if (importCode !== '') { + code = importCode + '\n' + `print(help(${content}))`; + } else { + code = `print(help(${content}))`; + } } let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 78a81e29..d0d2d5c0 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -122,8 +122,9 @@ define([ checkModules: [], // module aliases or function names docs: 'https://visual-python.gitbook.io/docs/getting-started/welcome-to-visual-python', helpInfo: { - content: '', // method to show using help() ex) pd.DataFrame - useHelp: true // custom text to show on help viewer + content: '', // method to show using help() ex) pd.DataFrame + useHelp: true, // custom text to show on help viewer + importCode: '' // import code for help }, ...restConfig }; @@ -1227,7 +1228,7 @@ define([ openHelpView() { this.closeHelpView(); this.helpViewer = new HelpViewer(); - this.helpViewer.open(this.config.helpInfo.content, this.config.helpInfo.useHelp); + this.helpViewer.open(this.config.helpInfo.content, this.config.helpInfo.useHelp, this.config.helpInfo.importCode); } /** @@ -1243,11 +1244,13 @@ define([ * Set HelpViewer content * @param {string} content * @param {boolean} useHelp + * @param {string} importCode */ - setHelpContent(content, useHelp=true) { + setHelpContent(content, useHelp=true, importCode='') { this.config.helpInfo = { content: content, - useHelp: useHelp + useHelp: useHelp, + importCode: importCode }; } From c42729bd01226e7c792661b8baff01df0706d7b0 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:16:06 +0900 Subject: [PATCH 13/26] Fix undefined code on DataPrep --- visualpython/js/m_ml/DataPrep.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index 5324f8b9..e01e7201 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -450,7 +450,7 @@ define([ // generate mct code if (modelType === 'make-column-transformer') { let mctCodes = []; - let { mct_estimator1, mct_columns1, mct_estimator2, mct_columns2 } = state; + let { mct_estimator1, mct_columns1='', mct_estimator2, mct_columns2='' } = state; if (mct_estimator1 != undefined && mct_estimator1 != '') { code.appendLine(this.modelConfig[mct_estimator1].import); let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], state, (userOption != ''? ', ' + userOption : '')); From fd51c78fa829b10bbce487c2fa2bbaa80fe98f7f Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:16:49 +0900 Subject: [PATCH 14/26] Add ml mode to DataSelector --- visualpython/js/com/component/DataSelector.js | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/visualpython/js/com/component/DataSelector.js b/visualpython/js/com/component/DataSelector.js index d6d34108..7439a7e9 100644 --- a/visualpython/js/com/component/DataSelector.js +++ b/visualpython/js/com/component/DataSelector.js @@ -50,7 +50,7 @@ define([ super._init(); this.prop = { - type: 'data', // selector type : data / column + type: 'data', // selector type : data / column / ml pageThis: null, // target's page object id: '', // target id value: null, // pre-defined value @@ -58,6 +58,7 @@ define([ finish: null, // callback after selection (value, dtype) select: null, // callback after selection from suggestInput (value, dtype) allowDataType: null, // list of allowed data types + dataCategory: null, // list of data category (use it for ml categories) // additional options boxClasses: '', classes: '', @@ -71,9 +72,20 @@ define([ // If null, define default allow data type if (!this.prop.allowDataType) { // default allow data types - this.prop.allowDataType = [ - 'DataFrame', 'Series', 'ndarray', 'list', 'dict' - ] + if (this.prop.type === 'ml') { + this.prop.allowDataType = vpConfig.getMLDataTypes(); + } else { + this.prop.allowDataType = [ + 'DataFrame', 'Series', 'ndarray', 'list', 'dict' + ] + } + } + if (!this.prop.dataCategory) { + if (this.prop.type === 'ml') { + this.prop.dataCategory = vpConfig.getMLCategories(); + } else { + this.prop.dataCategory = this.prop.allowDataType; + } } this.state = { @@ -282,7 +294,11 @@ define([ that.renderVariableBox(that._varList.filter(obj => !that.prop.allowDataType.includes(obj.dtype))); } else { // filter variable list - that.renderVariableBox(that._varList.filter(obj => obj.dtype == type)); + if (that.prop.type === 'ml') { + that.renderVariableBox(that._varList.filter(obj => vpConfig.getMLDataDict(type).includes(obj.dtype))); + } else { + that.renderVariableBox(that._varList.filter(obj => obj.dtype == type)); + } } }); @@ -530,7 +546,7 @@ define([ let varTags = new com_String(); let types = [ 'All', - ...this.prop.allowDataType, + ...this.prop.dataCategory, 'Others' ]; // Add Data Types to filter @@ -551,10 +567,17 @@ define([ let that = this; let varTags = new com_String(); varTags = new com_String(); - varList && varList.forEach((obj, idx) => { - varTags.appendFormatLine('
{4}
' - , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label); - }); + if (this.prop.type === 'ml') { + varList && varList.forEach((obj, idx) => { + varTags.appendFormatLine('
{4} | {5}
' + , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label, obj.dtype); + }); + } else { + varList && varList.forEach((obj, idx) => { + varTags.appendFormatLine('
{4}
' + , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label); + }); + } $(this.wrapSelector('.vp-ds-variable-box')).html(varTags.toString()); // focus on selected item From bc90d2b3543e95c065d0bfb4ad9025951c0c51a5 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:17:45 +0900 Subject: [PATCH 15/26] Fix chart not to add to block by default --- visualpython/js/m_visualize/Chart.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/visualpython/js/m_visualize/Chart.js b/visualpython/js/m_visualize/Chart.js index f5336bb9..d979a6e1 100644 --- a/visualpython/js/m_visualize/Chart.js +++ b/visualpython/js/m_visualize/Chart.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/com_generator', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_Const, com_util, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { +], function(chartHTml, chartCss, com_String, com_Const, com_util, com_interface, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { /** * Chart @@ -268,14 +269,16 @@ define([ $(this.wrapSelector('#vp_plImportRun')).click(function() { // generateImportCode var code = that.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Chart'); }); } From de1a802ece23688d4b1a0fac98e45dfab761b079 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:18:07 +0900 Subject: [PATCH 16/26] Add more options to value_counts --- visualpython/data/m_library/pandasLibrary.js | 36 ++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/visualpython/data/m_library/pandasLibrary.js b/visualpython/data/m_library/pandasLibrary.js index 315290fa..7ae4eb7e 100644 --- a/visualpython/data/m_library/pandasLibrary.js +++ b/visualpython/data/m_library/pandasLibrary.js @@ -2440,7 +2440,7 @@ define([ "name": "get data counts", "library": "pandas", "description": "get data value counts", - "code": "${o0} = ${i0}.value_counts()", + "code": "${o0} = ${i0}.value_counts(${bins}${sort}${ascending}${dropna}${etc})", "options": [ { "name": "i0", @@ -2457,7 +2457,39 @@ define([ "component": [ "input" ] - } + }, + { + "name": "bins", + "component": [ + "input_number" + ], + "help": "Only for Series object", + "usePair": true, + }, + { + "name": "sort", + "component": [ + "bool_select" + ], + "default": "True", + "usePair": true, + }, + { + "name": "ascending", + "component": [ + "bool_select" + ], + "default": "False", + "usePair": true, + }, + { + "name": "dropna", + "component": [ + "bool_select" + ], + "default": "True", + "usePair": true, + }, ] }, "pdIdt_info": { From 39004db4bfaeafb94524895480437e1282af6071 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 30 Aug 2023 13:18:30 +0900 Subject: [PATCH 17/26] Add statistics to Add Column menu on Frame app --- visualpython/js/m_apps/Frame.js | 46 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index d2e27d3f..d2f7c3c0 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -21,8 +21,9 @@ define([ 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/m_apps/Subset' -], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, Subset) { +], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, MultiSelector, Subset) { /** * Frame @@ -1427,6 +1428,7 @@ define([ content.appendLine(''); content.appendFormatLine('', 'vp-inner-popup-stats-method'); + let methodList = [ + 'sum', 'mean', 'median', 'max', 'min', 'std', 'var' + ]; + methodList.forEach(method => { + content.appendFormatLine('', + method, method, method); + }); + content.appendLine(''); + + content.appendLine('
'); + content.appendLine('
'); + + // tab 3. replace content.appendFormatLine(''); - // tab 3. condition + // tab 4. condition // replace page - 2. condition content.appendFormatLine(''); - // tab 4. apply + // tab 5. apply content.appendFormatLine('