diff --git a/visualpython/html/m_ml/gridSearch.html b/visualpython/html/m_ml/gridSearch.html new file mode 100644 index 00000000..976cd69c --- /dev/null +++ b/visualpython/html/m_ml/gridSearch.html @@ -0,0 +1,41 @@ + +

+ Model type +

+ +

+ Allocate to + +

+ Param grid + +

+ Param set 1 +

+ +

+ \ No newline at end of file diff --git a/visualpython/html/m_ml/pipeline.html b/visualpython/html/m_ml/pipeline.html new file mode 100644 index 00000000..464f27fa --- /dev/null +++ b/visualpython/html/m_ml/pipeline.html @@ -0,0 +1,25 @@ + +

+ Pipeline + +

+ +

+ \ No newline at end of file diff --git a/visualpython/img/apps/apps_gridSearch.svg b/visualpython/img/apps/apps_gridSearch.svg new file mode 100644 index 00000000..0d6ba9b5 --- /dev/null +++ b/visualpython/img/apps/apps_gridSearch.svg @@ -0,0 +1,6 @@ + diff --git a/visualpython/img/apps/apps_pipeline.svg b/visualpython/img/apps/apps_pipeline.svg new file mode 100644 index 00000000..aab67d91 --- /dev/null +++ b/visualpython/img/apps/apps_pipeline.svg @@ -0,0 +1,9 @@ + diff --git a/visualpython/img/item_disable.svg b/visualpython/img/item_disable.svg new file mode 100644 index 00000000..9f3970cc --- /dev/null +++ b/visualpython/img/item_disable.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_disable_hover.svg b/visualpython/img/item_disable_hover.svg new file mode 100644 index 00000000..6491b8c6 --- /dev/null +++ b/visualpython/img/item_disable_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_enable.svg b/visualpython/img/item_enable.svg new file mode 100644 index 00000000..ab75f8a5 --- /dev/null +++ b/visualpython/img/item_enable.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_enable_hover.svg b/visualpython/img/item_enable_hover.svg new file mode 100644 index 00000000..02c24e50 --- /dev/null +++ b/visualpython/img/item_enable_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/status_green.svg b/visualpython/img/status_green.svg new file mode 100644 index 00000000..13e43ec3 --- /dev/null +++ b/visualpython/img/status_green.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/status_orange.svg b/visualpython/img/status_orange.svg new file mode 100644 index 00000000..93b230c1 --- /dev/null +++ b/visualpython/img/status_orange.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/status_red.svg b/visualpython/img/status_red.svg new file mode 100644 index 00000000..6fe4a8bc --- /dev/null +++ b/visualpython/img/status_red.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/js/com/com_Config.js b/visualpython/js/com/com_Config.js index bbc6fb83..8d964c7d 100644 --- a/visualpython/js/com/com_Config.js +++ b/visualpython/js/com/com_Config.js @@ -1107,7 +1107,7 @@ define([ /** Scaling */ 'StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer', 'FunctionTransformer', 'PolynomialFeatures', 'KBinsDiscretizer', /** ETC */ - 'ColumnTransformer' + 'SimpleImputer', 'ColumnTransformer' ], 'Regression': [ 'LinearRegression', 'Ridge', 'Lasso', 'ElasticNet', 'SVR', 'DecisionTreeRegressor', 'RandomForestRegressor', 'GradientBoostingRegressor', 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor', @@ -1123,6 +1123,9 @@ define([ ], 'Auto ML': [ 'AutoSklearnRegressor', 'AutoSklearnClassifier', 'TPOTRegressor', 'TPOTClassifier' + ], + 'ETC': [ + 'GridSearchCV' ] }; @@ -1132,7 +1135,8 @@ define([ ...Config.ML_DATA_DICT['Classification'], ...Config.ML_DATA_DICT['Clustering'], ...Config.ML_DATA_DICT['Dimension Reduction'], - ...Config.ML_DATA_DICT['Auto ML'] + ...Config.ML_DATA_DICT['Auto ML'], + ...Config.ML_DATA_DICT['ETC'] ]; return Config; diff --git a/visualpython/js/com/com_generatorV2.js b/visualpython/js/com/com_generatorV2.js index 4157b2f1..8bd7b908 100644 --- a/visualpython/js/com/com_generatorV2.js +++ b/visualpython/js/com/com_generatorV2.js @@ -741,7 +741,9 @@ define([ package.options && package.options.forEach(function(v, i) { var val = state[v.name]; if (val == undefined || val == '' || val == v.default) { - val = vp_getTagValue(pageThis, v, parent=parent); + if (pageThis) { + val = vp_getTagValue(pageThis, v, parent=parent); + } } var id = '${' + v.name + '}'; if (val == undefined || val.trim() == '') { @@ -802,9 +804,11 @@ define([ if (_VP_SHOW_RESULT && package.options) { var outputOptList = package.options.filter(x => x.output === true); var outputStr = ''; - outputOptList.forEach(opt => { - outputStr += (outputStr !== ''?', ':'') + vp_getTagValue(pageThis, opt); - }) + if (pageThis) { + outputOptList.forEach(opt => { + outputStr += (outputStr !== ''?', ':'') + vp_getTagValue(pageThis, opt); + }) + } if (outputStr != '') { code += '\n'+ outputStr; } diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index d847043b..a31fe8af 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -80,8 +80,8 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Perform modeling from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } ] }, 'predict': { @@ -90,7 +90,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -100,7 +100,7 @@ define([ code: '${pred_prob_allocate} = ${model}.predict_proba(${pred_prob_featureData})', description: 'Predict class probabilities for X.', options: [ - { name: 'pred_prob_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_prob_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_prob_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -110,7 +110,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -125,7 +125,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_transform': { @@ -134,7 +134,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -153,7 +153,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } @@ -181,7 +181,7 @@ define([ code: '${dec_allocate} = ${model}.decision_function(${dec_featureData})', description: 'Compute the decision function of X.', options: [ - { name: 'dec_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'dec_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'dec_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable' } ] } @@ -198,7 +198,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Fit and predict.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -215,7 +215,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Perform clustering from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_predict': { @@ -224,7 +224,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute clusters from a data or distance matrix and predict labels.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -238,7 +238,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Compute clustering.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'predict': { @@ -247,7 +247,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -257,7 +257,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute cluster centers and predict cluster index for each sample.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -271,7 +271,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Compute clustering and transform X to cluster-distance space.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -281,7 +281,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Transform X to a cluster-distance space.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -297,7 +297,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_transform': { @@ -306,7 +306,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -321,8 +321,8 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, 'fit_transform': { @@ -331,8 +331,8 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_trans_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -342,7 +342,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -352,7 +352,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -366,7 +366,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_transform': { @@ -375,7 +375,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -385,7 +385,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -395,12 +395,57 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } @@ -415,8 +460,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: '', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, @@ -437,12 +482,34 @@ define([ code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', description: 'Permutation importance for feature evaluation.', options: [ - { name: 'importance_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'importance_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' }, + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'scoring', component: ['input'], usePair: true }, { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] + }, + 'feature_importances': { + name: 'feature_importances', + label: 'Feature importances', + code: "${fi_allocate} = vp_create_feature_importances(${model}, ${fi_featureData}${sort})", + description: 'Allocate feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fi_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'df_i' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true } + ] + }, + 'plot_feature_importances': { + name: 'plot_feature_importances', + label: 'Plot feature importances', + code: "vp_plot_feature_importances(${model}, ${fi_featureData}${sort}${top_count})", + description: 'Draw feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true }, + ] } } switch (category) { @@ -535,8 +602,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -568,6 +635,19 @@ define([ ] } } + let svcList = [ + 'DecisionTreeRegressor', + 'RandomForestRegressor', + 'GradientBoostingRegressor', + 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor' + ]; + if (svcList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } break; case 'Classification': infos = { @@ -582,8 +662,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -595,8 +675,73 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, + 'roc_curve': { + name: 'roc_curve', + label: 'ROC Curve', + import: 'from sklearn import metrics', + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.predict_proba(${roc_featureData})[:, 1])\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()", + description: '', + options: [ + { name: 'roc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'roc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'auc': { + name: 'auc', + label: 'AUC', + import: 'from sklearn import metrics', + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.predict_proba(${auc_featureData})[:, 1])', + description: '', + options: [ + { name: 'auc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, 'permutation_importance': defaultInfos['permutation_importance'] } + + // feature importances + let clfList = [ + 'DecisionTreeClassifier', + 'RandomForestClassifier', + 'GradientBoostingClassifier', + 'XGBClassifier', + 'LGBMClassifier', + 'CatBoostClassifier', + ] + if (clfList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } + + // use decision_function on ROC, AUC + let decisionFunctionTypes = [ + 'LogisticRegression', 'SVC', 'GradientBoostingClassifier' + ]; + if (decisionFunctionTypes.includes(modelType)) { + infos = { + ...infos, + 'roc_curve': { + ...infos['roc_curve'], + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.decision_function(${roc_featureData}))\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()" + }, + 'auc': { + ...infos['auc'], + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.decision_function(${auc_featureData}))', + } + } + } break; case 'Auto ML': infos = { @@ -611,16 +756,18 @@ define([ break; case 'Clustering': infos = { - // 'Size of clusters': { - // name: 'Size of clusters', - // code: "print(f'Size of clusters: {np.bincount(pred)}')", // FIXME: model.cluster_centers_ / use model info or hide it - // options: [] - // } + 'get_params': { + ...defaultInfos['get_params'] + } } if (modelType == 'KMeans') { infos = { ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Return the mean accuracy on the given test data and labels.' + }, 'cluster_centers_': { name: 'cluster_centers', label: 'Cluster centers', @@ -642,11 +789,21 @@ define([ code: "# import\nfrom scipy.cluster.hierarchy import dendrogram, ward\n\nlinkage_array = ward(${dendro_data})\ndendrogram(linkage_array, p=3, truncate_mode='level', no_labels=True)\nplt.show()", description: 'Draw a dendrogram', options: [ - { name: 'dendro_data', label: 'Data', component: ['var_select'], var_type: ['DataFrame'] } + { name: 'dendro_data', label: 'Data', component: ['data_select'], var_type: ['DataFrame'] } ] } } } + + if (modelType == 'GaussianMixture') { + infos = { + ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Compute the per-sample average log-likelihood of the given data X.' + } + } + } break; case 'Dimension Reduction': if (modelType == 'LDA') { @@ -657,8 +814,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } @@ -686,12 +843,45 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js index 350e9093..884b76a6 100644 --- a/visualpython/js/com/component/PackageManager.js +++ b/visualpython/js/com/component/PackageManager.js @@ -17,10 +17,11 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(ifHtml, ifCss, com_util, com_Const, com_String, PopupComponent, FileNavigation, LoadingSpinner) { +], function(ifHtml, ifCss, com_util, com_Const, com_String, SuggestInput, PopupComponent, FileNavigation, LoadingSpinner) { /** * PackageManager @@ -89,18 +90,24 @@ define([ } } + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('.vp-pm-search')); + } + _bindEvent() { super._bindEvent(); /** Implement binding events */ let that = this; // search item - $(this.wrapSelector('.vp-pm-search')).on('change', function(evt) { + $(document).on('change', this.wrapSelector('.vp-pm-search'), function(evt) { var value = $(this).val(); if (value != '') { $(that.wrapSelector('.vp-pm-item')).hide(); $(that.wrapSelector('.vp-pm-item')).filter(function() { - return $(this).data('key').search(value) >= 0; + let key = $(this).data('key'); + return key.search(value.toLowerCase()) >= 0; }).show(); } else { $(that.wrapSelector('.vp-pm-item')).show(); @@ -116,7 +123,14 @@ define([ // sort item $(this.wrapSelector('.vp-pm-sort-menu-item')).on('click', function() { var menu = $(this).data('menu'); - if (menu === 'name') { + if (menu === 'registered') { + // sort by name + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var keyA = parseInt($(a).data('seq')); + var keyB = parseInt($(b).data('seq')); + return keyA > keyB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'name') { // sort by name $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { var keyA = $(a).data('key'); @@ -147,6 +161,16 @@ define([ return insA > insB ? 1 : -1 }).appendTo($(that.wrapSelector('.vp-pm-table'))) } + $(that.wrapSelector('.vp-pm-sort-menu-box')).hide(); + }); + + // reload package list + $(this.wrapSelector('.vp-pm-func-reload')).on('click', function() { + // reset search keyword + $(that.wrapSelector('.vp-pm-search')).val(''); + + // load package list + that.loadPackageList(); }); // add package @@ -355,11 +379,12 @@ define([ * * @param {String} key * @param {Object} info installed, version, path + * @param {number} index sequence of initial package list * @returns */ - renderPackageItem(key, info) { + renderPackageItem(key, info, index) { var item = new com_String(); - item.appendFormatLine('

', 'vp-pm-item', key, info.installed===true?'1':'0'); + item.appendFormatLine('

', 'vp-pm-item', key, info.installed===true?'1':'0', index); item.appendFormatLine('

', 'vp-pm-item-header', (info.path?info.path:'')); item.appendFormatLine('{0}', key); if (info.installed === true) { @@ -401,6 +426,14 @@ define([ $(this.wrapSelector('.vp-pm-table')).html(''); let packageList = Object.keys(this.packageLib); + + // set auto search + let searchInput = new SuggestInput(); + searchInput.addClass('vp-pm-search vp-input'); + searchInput.setPlaceholder("Search"); + searchInput.setSuggestList(function () { return packageList; }); + $(this.wrapSelector('.vp-pm-search')).replaceWith(searchInput.toTagString()); + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); vpKernel.getPackageList(packageList).then(function(resultObj) { let { result } = resultObj; @@ -408,10 +441,10 @@ define([ // load code list var innerFuncCode = new com_String(); - Object.keys(packageInfo).forEach(key => { + Object.keys(packageInfo).forEach((key, idx) => { let info = packageInfo[key]; // installed, version, path if (info) { - var item = that.renderPackageItem(key, info); + var item = that.renderPackageItem(key, info, idx); innerFuncCode.append(item); } }); diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 90d79712..0b0bf5fb 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -112,9 +112,13 @@ define([ // show footer runButton: true, footer: true, + // position and size position: { right: 10, top: 120 }, size: { width: 400, height: 550 }, + // additional modes and infos saveOnly: false, // apply mode + resizable: true, + autoScroll: true, checkModules: [], // module aliases or function names docs: 'https://visual-python.gitbook.io/docs/getting-started/welcome-to-visual-python', helpInfo: { @@ -456,9 +460,10 @@ define([ } break; case 'run': - let result = that.run(); + var result = that.run(); if (result) { - that.save(); + // that.save(); + that.close(); } break; case 'show-detail': @@ -474,13 +479,20 @@ define([ $(this.wrapSelector('.vp-popup-detail-button')).on('click', function(evt) { var btnType = $(this).data('type'); switch(btnType) { - case 'apply': - that.save(); + // case 'apply': + // that.save(); + // break; + case 'run-save': + var result = that.run(); + if (result) { + that.save(); + } break; case 'add': - let result = that.run(false); + var result = that.run(false); if (result) { - that.save(); + // that.save(); + that.close(); } break; } @@ -599,7 +611,7 @@ define([ let { installButton, importButton, packageButton, codeview, dataview, helpview, runButton, footer, - sizeLevel, position, docs + sizeLevel, position, autoScroll, docs } = this.config; // apply link to docs @@ -668,6 +680,10 @@ define([ break; } + if (autoScroll) { + $(this.wrapSelector('.vp-popup-body')).addClass('vp-scrollbar'); + } + // set detailed size $(this.wrapSelector()).css({ width: this.config.size.width + 'px', @@ -683,7 +699,9 @@ define([ } this._bindDraggable(); - this._bindResizable(); + if (this.config.resizable) { + this._bindResizable(); + } } templateForInnerPopup() { @@ -785,9 +803,8 @@ define([ vpLog.display(VP_LOG_TYPE.DEVELOP, 'savedState', that.state); } - _saveSingleState(tag) { + _getTagValue(tag) { let id = tag.id; - let customKey = $(tag).data('key'); let tagName = $(tag).prop('tagName'); // returns with UpperCase let newValue = ''; switch(tagName) { @@ -818,7 +835,14 @@ define([ } break; } + return newValue; + } + + _saveSingleState(tag) { + let id = tag.id; + let customKey = $(tag).data('key'); + let newValue = this._getTagValue(tag); // if custom key is available, use it if (customKey && customKey != '') { // allow custom key until level 2 @@ -947,30 +971,52 @@ define([ * - focus popup * - bind codemirror */ - open() { + open(targetFrame=undefined) { vpLog.display(VP_LOG_TYPE.DEVELOP, 'open popup', this); this.loadState(); this._beforeOpen(); - this.show(); - - // set popup position if its top-left side is outside of view - let pos = $(this.wrapSelector()).position(); - if (pos) { - if (pos.top < 0) { - $(this.wrapSelector()).css({ top: 0 }); - } - if (pos.left < 0) { - $(this.wrapSelector()).css({ left: 0 }); + if (targetFrame !== undefined) { + // hide popup frame + $(this.wrapSelector('.vp-popup-header')).hide(); + $(this.wrapSelector('.vp-popup-footer')).hide(); + // set width and height to 100% + $(this.wrapSelector()).css({ + width: '100%', + height: '100%', + 'min-height': 'unset', + display: 'block', + position: 'initial', + border: '0px' + }); + $(this.wrapSelector('.vp-popup-body')).css({ + padding: '0px' + }); + // show on targetFrame + $(this.wrapSelector()).appendTo($(targetFrame)); + } else { + this.show(); + + // set popup position if its top-left side is outside of view + let pos = $(this.wrapSelector()).position(); + if (pos) { + if (pos.top < 0) { + $(this.wrapSelector()).css({ top: 0 }); + } + if (pos.left < 0) { + $(this.wrapSelector()).css({ left: 0 }); + } } } this._bindCodemirror(); - $(this.eventTarget).trigger({ - type: 'focus_option_page', - component: this - }); + if (targetFrame !== undefined) { + $(this.eventTarget).trigger({ + type: 'focus_option_page', + component: this + }); + } } setSaveOnlyMode() { @@ -1004,6 +1050,7 @@ define([ remove() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'remove popup', this); + this._unbindResizable(); this._unbindEvent(); this.closeHelpView(); $(this.wrapSelector()).remove(); diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index e8b98611..d2e27d3f 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -710,10 +710,7 @@ define([ return; } } else if (type === FRAME_EDIT_TYPE.REPLACE) { - if (content.replacetype === 'condition' && content.value === '') { - $(this.wrapSelector('.vp-inner-popup-input3')).focus(); - return; - } + ; } else if (type === FRAME_EDIT_TYPE.FILL_NA) { if (content.method === 'value' && content.value === '') { $(this.wrapSelector('.vp-inner-popup-value')).focus(); @@ -3101,7 +3098,7 @@ define([ }); let valueStr = values.join(' '); if (valueStr === "" || valueStr === "''") { - code.appendFormat("{0}[{1}] = np.NaN", tempObj, content.name); + code.appendFormat("{0}[{1}] = np.nan", tempObj, content.name); } else { code.appendFormat("{0}[{1}] = {2}", tempObj, content.name, valueStr); } @@ -3178,6 +3175,9 @@ define([ } }); var value = com_util.convertToStr(content.value, content.valueastext); + if (value === '') { + value = 'np.nan'; + } code.appendFormat(", {0}] = {1}", content.name, value); } break; @@ -3223,9 +3223,10 @@ define([ code.append(')'); } else if (tab === 'condition') { code.appendFormat("{0}.loc[", tempObj); + var condCode = new com_String(); content['list'].forEach((obj, idx) => { let { colName, oper, cond, condAsText, connector } = obj; - code.append('('); + condCode.append('('); let colValue = tempObj; if (colName && colName != '') { @@ -3237,25 +3238,31 @@ define([ } let condValue = com_util.convertToStr(cond, condAsText); if (oper == 'contains') { - code.appendFormat('{0}.str.contains({1})', colValue, condValue); + condCode.appendFormat('{0}.str.contains({1})', colValue, condValue); } else if (oper == 'not contains') { - code.appendFormat('~{0}.str.contains({1})', colValue, condValue); + condCode.appendFormat('~{0}.str.contains({1})', colValue, condValue); } else if (oper == 'starts with') { - code.appendFormat('{0}.str.startswith({1})', colValue, condValue); + condCode.appendFormat('{0}.str.startswith({1})', colValue, condValue); } else if (oper == 'ends with') { - code.appendFormat('{0}.str.endswith({1})', colValue, condValue); + condCode.appendFormat('{0}.str.endswith({1})', colValue, condValue); } else if (oper == 'isnull()' || oper == 'notnull()') { - code.appendFormat('{0}.{1}', colValue, oper); + condCode.appendFormat('{0}.{1}', colValue, oper); } else { - code.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); + condCode.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); } - code.append(')'); + condCode.append(')'); if (idx < (content['list'].length - 1)) { - code.append(connector); + condCode.append(connector); } }); + if (condCode.toString() === '') { + condCode.append(':'); + } var value = com_util.convertToStr(content.value, content.valueastext); - code.appendFormat(", {0}] = {1}", content.name, value); + if (value === '') { + value = 'np.nan'; + } + code.appendFormat("{0}, {1}] = {2}", condCode.toString(), content.name, value); } else if (tab == 'apply') { // code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); let lambdaCode = 'lambda x: '; diff --git a/visualpython/js/m_apps/Information.js b/visualpython/js/m_apps/Information.js index 025caf32..ff310874 100644 --- a/visualpython/js/m_apps/Information.js +++ b/visualpython/js/m_apps/Information.js @@ -153,7 +153,7 @@ define([ , code: "pd.plotting.scatter_matrix(${data}, marker='o', hist_kwds={'bins': 30}, s=30, alpha=.8)\nplt.show()" , dtype: ['DataFrame'] }, { id: 'boxplot', label: 'Box plot', code: "${data}.plot(kind='box')\nplt.show()", dtype: ['DataFrame', 'Series'] }, - { id: 'count_plot', label: 'Count plot', code: "${data}.value_counts().plot(kind='bar')\nplt.show()", dtype: ['DataFrame', 'Series'] } + { id: 'count_plot', label: 'Count plot', code: "${data}.value_counts().plot(kind='bar', rot=30)\nplt.show()", dtype: ['DataFrame', 'Series'] } ] } diff --git a/visualpython/js/m_apps/PandasOption.js b/visualpython/js/m_apps/PandasOption.js index cb96e92c..359de409 100644 --- a/visualpython/js/m_apps/PandasOption.js +++ b/visualpython/js/m_apps/PandasOption.js @@ -30,6 +30,7 @@ define([ this.config.sizeLevel = 2; this.config.dataview = false; this.config.checkModules = ['pd']; + this.config.docs = 'https://pandas.pydata.org/docs/user_guide/options.html'; this.state = { filter_warning: '', diff --git a/visualpython/js/m_apps/Subset.js b/visualpython/js/m_apps/Subset.js index 5c923a62..72206ba2 100644 --- a/visualpython/js/m_apps/Subset.js +++ b/visualpython/js/m_apps/Subset.js @@ -1742,7 +1742,7 @@ define([ var connector = i > 0 ? $(condList[i - 1]).find('.vp-oper-connect').val() : undefined; // if no variable selected, pass - if (varName == "") + if (varName === "" || oper === "") continue; if (useCondition) { rowSelection.append(connector); diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index 983a8168..5324f8b9 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -59,7 +59,7 @@ define([ this.modelTypeList = { 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], - 'ETC': ['make-column-transformer'] + 'ETC': ['prep-simple-imputer', 'make-column-transformer'] } this.mctEstimator = { @@ -94,10 +94,19 @@ define([ $(that.wrapSelector('#vp_installLibrary')).hide(); } - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { // load mct-targetData that.loadVariableList(); that.bindMCT(); + } else if (modelType === 'prep-simple-imputer') { + $(that.wrapSelector('#missing_values')).replaceWith(`

+ ${$(that.wrapSelector('#missing_values'))[0].outerHTML} + Text +

`); + $(that.wrapSelector('#fill_value')).replaceWith(`

+ ${$(that.wrapSelector('#fill_value'))[0].outerHTML} + Text +

`); } }); @@ -411,6 +420,7 @@ define([ generateCode() { let { modelControlType, modelType, userOption, allocateToCreation, model } = this.state; + let state = JSON.parse(JSON.stringify(this.state)); let code = new com_String(); if (modelControlType == 'creation') { /** @@ -422,22 +432,33 @@ define([ let config = this.modelConfig[modelType]; code.appendLine(config.import); + if (modelType === 'prep-simple-imputer') { + let checkList = ['missing_values', 'fill_value']; + checkList.forEach(checkKey => { + try { + state[checkKey] = com_util.convertToStr(state[checkKey], state[checkKey + '_istext']); + } catch(e) { + ; + } + }); + } + // model code let modelCode = config.code; - modelCode = com_generator.vp_codeGenerator(this, config, this.state, (userOption != ''? ', ' + userOption : '')); + modelCode = com_generator.vp_codeGenerator(this, config, state, (userOption != ''? ', ' + userOption : '')); // generate mct code - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { let mctCodes = []; - let { mct_estimator1, mct_columns1, mct_estimator2, mct_columns2 } = this.state; + let { mct_estimator1, mct_columns1, mct_estimator2, mct_columns2 } = state; if (mct_estimator1 != undefined && mct_estimator1 != '') { code.appendLine(this.modelConfig[mct_estimator1].import); - let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator1code, mct_columns1)); } if (mct_estimator2 != undefined && mct_estimator2 != '') { code.appendLine(this.modelConfig[mct_estimator2].import); - let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator2code, mct_columns2)); } modelCode = modelCode.replace('${mct_code}', mctCodes.join(', ')); diff --git a/visualpython/js/m_ml/FitPredict.js b/visualpython/js/m_ml/FitPredict.js index 8cf91de5..3dfd672f 100644 --- a/visualpython/js/m_ml/FitPredict.js +++ b/visualpython/js/m_ml/FitPredict.js @@ -685,6 +685,51 @@ define([ } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } diff --git a/visualpython/js/m_ml/GridSearch.js b/visualpython/js/m_ml/GridSearch.js new file mode 100644 index 00000000..a2006b26 --- /dev/null +++ b/visualpython/js/m_ml/GridSearch.js @@ -0,0 +1,494 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : GridSearch.js + * Author : Black Logic + * Note : GridSearch + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] GridSearch +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/gridSearch.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/gridSearch'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, com_util, com_interface, com_String, com_generator, ML_LIBRARIES, PopupComponent, MultiSelector, SuggestInput, ModelEditor) { + + /** + * GridSearch + */ + class GridSearch extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 3; + this.config.dataview = false; + this.config.docs = 'https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html'; + + this.state = { + modelType: 'ln-rgs', + userOption: '', + allocateToCreation: 'model', + ...this.state + } + + this.targetSetTag = null; + + this.modelConfig = ML_LIBRARIES; + + this.modelTypeList = { + 'Regression': ['ln-rgs', 'ridge', 'lasso', 'elasticnet', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], + 'Classfication': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'] + } + + + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('click', this.wrapSelector('.vp-param-set-del')); + $(document).off('click', this.wrapSelector('.vp-param-item-add')); + $(document).off('click', this.wrapSelector('.vp-param-item-del')); + $(document).off('keyup', this.wrapSelector('.vp-param-val')); + $(document).off('click', this.wrapSelector('.vp-param-result-item-del')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select model type + $(this.wrapSelector('#modelType')).on('change', function() { + let modelType = $(this).val(); + that.state.modelType = modelType; + + // show install button + if (that.modelConfig[modelType].install != undefined) { + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + + // reset model param set + $(that.wrapSelector('.vp-param-grid-box')).html(''); + $(that.wrapSelector('.vp-param-grid-box')).html(that.templateForParamSet()); + }); + + // Add param set + $(this.wrapSelector('#vp_addParamSet')).on('click', function() { + let newSet = $(that.templateForParamSet()); + $(that.wrapSelector('.vp-param-grid-box')).append(newSet); + // focus + $(newSet)[0].scrollIntoView(); + }); + + // delete param set + $(document).on('click', this.wrapSelector('.vp-param-set-del'), function() { + $(this).closest('.vp-param-set-box').remove(); + + // rename param set + $(that.wrapSelector('.vp-param-set-name')).each((i, tag) => { + $(tag).text('Param set ' + (i + 1)); + }); + }); + + // Add param item + $(document).on('click', this.wrapSelector('.vp-param-item-add'), function() { + that.targetSetTag = $(this).parent(); // target param-set-box + that.openParamPopup(); + }); + + // Delete param item + $(document).on('click', this.wrapSelector('.vp-param-item-del'), function() { + $(this).closest('.vp-param-item').remove(); + }); + + // add param value - using enter + $(document).on('keyup', this.wrapSelector('.vp-param-val'), function(event) { + var keycode = event.keyCode + ? event.keyCode + : event.which; + if (keycode == vpEvent.keyManager.keyCode.enter) { // enter + that.handleAddParamValue($(this)); + } + if (keycode === 188) {// ,< + let val = $(this).val(); + $(this).val(val.split(',')[0]); // remove , and add param + that.handleAddParamValue($(this)); + } + }); + + // delete param set item + $(document).on('click', this.wrapSelector('.vp-param-result-item-del'), function() { + $(this).closest('.vp-param-result-item').remove(); + }); + } + + handleAddParamValue(thisTag) { + let parentTag = $(thisTag).parent(); + let paramIsText = $(parentTag).find('.vp-param-val').data('type') === 'text'; // text / var + let paramVal = $(parentTag).find('.vp-param-val').val(); + // check , and split it + let paramSplit = paramVal.split(','); + paramSplit && paramSplit.forEach(val => { + // add only if it is not empty value + if (val.trim() !== '') { + val = com_util.convertToStr(val.trim(), paramIsText); + $(parentTag).find('.vp-param-result-box').append(` + ${val} + + + `); + // clear param val + $(parentTag).find('.vp-param-val').val(''); + } + }); + } + + bindEventForInnerPopup() { + let that = this; + // Inner popup: Select param + $(document).off('click', this.wrapSelector('.vp-inner-param-list-item')); + $(document).on('click', this.wrapSelector('.vp-inner-param-list-item'), function() { + if ($(this).hasClass('selected')) { + $(this).removeClass('selected'); + } else { + $(this).addClass('selected'); + } + }); + + // Inner popup: Add param + $(this.wrapSelector('.vp-add-param-btn')).on('click', function() { + let newParam = $(that.wrapSelector('.vp-add-param-name')).val(); + if (newParam != undefined && newParam.trim() != '') { + // check if exist + let checkTag = $(that.wrapSelector(`.vp-inner-param-list-item[data-name="${newParam}"]`)); + if (checkTag.length > 0) { + if (!checkTag.hasClass('selected')) { + checkTag.addClass('selected'); + } + checkTag[0].scrollIntoView(); + } else { + // add to param list + $(that.wrapSelector('.vp-inner-param-list-box')).append( + `

${newParam}

` + ); + // scroll to added item + $(that.wrapSelector('.vp-inner-param-list-item.selected:last'))[0].scrollIntoView(); + } + + // clear input value + $(that.wrapSelector('.vp-add-param-name')).val(''); + } + }); + } + + templateForParamSet() { + let paramSetNo = 1; + // set param set number + paramSetNo += $(this.wrapSelector('.vp-param-set-box')).length; + return `

+ Param set ${paramSetNo} +

+ +

`; + } + + templateForParamPopup() { + let config = this.modelConfig[this.state.modelType]; + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('

{2}

' + , opt.name, opt.name, opt.name); + }); + + return ` +

NOTE: Select parameters to add.

+ ${optBox.toString()} +

+ + +

+ `; + } + + openParamPopup() { + let size = { width: 400, height: 300 }; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); + + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); + + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForParamPopup()); + + this.bindEventForInnerPopup(); + + // show popup box + this.openInnerPopup('Add parameter'); + } + + handleInnerOk() { + // get selected param list + let paramList = []; + $(this.wrapSelector('.vp-inner-param-list-item.selected')).each((i, tag) => { + paramList.push($(tag).data('name')); + }); + + if (paramList.length > 0) { + // add param box + $(this.targetSetTag).find('.vp-param-item-box').append(this.templateForParamItemList(paramList)); + + this.closeInnerPopup(); + } else { + com_util.renderAlertModal('No params selected. Please select more than one param.'); + } + + this.targetSetTag = null; + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Model creation + //================================================================ + // model types + let modelTypeTag = new com_String(); + Object.keys(this.modelTypeList).forEach(modelCategory => { + let modelOptionTag = new com_String(); + that.modelTypeList[modelCategory].forEach(opt => { + let optConfig = that.modelConfig[opt]; + let selectedFlag = ''; + if (opt == that.state.modelType) { + selectedFlag = 'selected'; + } + modelOptionTag.appendFormatLine('', + opt, selectedFlag, optConfig.name); + }) + modelTypeTag.appendFormatLine('{1}', + modelCategory, modelOptionTag.toString()); + }); + $(page).find('#modelType').html(modelTypeTag.toString()); + + //================================================================ + // GridSearch option + //================================================================ + $(page).find('.vp-model-option-box').html(this.templateForOption('grid-search', ['estimator', 'param_grid'])); + + // show install button + if (this.modelConfig[this.state.modelType].install != undefined) { + this.showInstallButton(); + } else { + this.hideInstallButton(); + } + + // render option page + // $(page).find('.vp-model-param-box').html(this.templateForOption(this.state.modelType)); + + return page; + } + + templateForOption(modelType, excludeOptList=[]) { + let config = this.modelConfig[modelType]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + if (!excludeOptList.includes(opt.name)) { + optBox.appendFormatLine('{2}' + , opt.name, opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + } + }); + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + return optBox.toString(); + } + + /** + * template for param box + * @param {*} paramList ['param-name1', 'param-name2', ...] + */ + templateForParamItemList(paramList) { + let that = this; + let config = this.modelConfig[this.state.modelType]; + let paramSet = new com_String(); + paramList && paramList.forEach(name => { + // get option component + let componentType = 'input'; // default + let paramObj = config.options.filter(x => x.name === name).at(0); + if ((paramObj !== undefined) && (['option_select', 'bool_select'].includes(paramObj.component[0]))) { + componentType = 'option'; + } + paramSet.appendLine('

'); + paramSet.appendFormatLine('{1}', name, com_util.optionToLabel(name)); + paramSet.appendLine('

'); + paramSet.appendFormatLine('

', name); + paramSet.appendFormatLine('', name); + if (componentType === 'input') { + paramSet.appendFormatLine('', name); + + } else { + // paramSet.appendFormatLine('', name); + var suggestInput = new SuggestInput(); + suggestInput.setPlaceholder('Type here'); + suggestInput.addClass('vp-param-val'); + suggestInput.addAttribute('data-name', name); + suggestInput.addAttribute('data-type', (paramObj.type === 'text' ? 'text' : 'var')); + let optionList = paramObj.options; + if (paramObj.component[0] === 'bool_select') { + optionList = ['True', 'False']; + } + suggestInput.setSuggestList(function() { return optionList; }); + suggestInput.setNormalFilter(false); + suggestInput.setSelectEvent(function(selectedValue) { + // trigger change + let thisTag = $(that.wrapSelector('.' + suggestInput.uuid)); + that.handleAddParamValue($(thisTag)); + $(thisTag).val(''); + }); + paramSet.appendLine(suggestInput.toTagString()); + } + paramSet.appendLine('

'); + paramSet.appendLine('

'); // vp-param-item + }); + return paramSet.toString(); + } + + render() { + super.render(); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + } + + generateInstallCode() { + let installCode = this.modelConfig[this.state.modelType].install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + return [ installCode ]; + } + + generateCode() { + let { modelType, userOption, allocateToCreation, model } = this.state; + let code = new com_String(); + /** + * Model Creation + * --- + * from module import model_function + * model = Model(key=value, ...) + */ + let gsConfig = this.modelConfig['grid-search']; + + + let state = JSON.parse(JSON.stringify(this.state)); + let estConfig = this.modelConfig[this.state.modelType]; + let estimator = com_generator.vp_codeGenerator(null, estConfig, {}, ''); + state['estimator'] = estimator; + state['param_grid'] = '{}'; + + let reservedKeywordList = ['None', 'True', 'False', 'np.nan', 'np.NaN']; + let paramGrid = []; + // generate param_grid + $(this.wrapSelector('.vp-param-set-box')).each((i, tag) => { + let paramSet = {}; + $(tag).find('.vp-param-result-box').each((j, resTag) => { + let paramName = $(resTag).data('name'); + let paramValList = []; + $(resTag).find('.vp-param-result-item').each((k, itemTag) => { + let val = $(itemTag).data('value'); + if (!isNaN(val)) { + // numeric string -> numeric + val = parseFloat(val); + } + paramValList.push(val); + }); + if (paramValList.length > 0) { + // Add only its result exists + paramSet[paramName] = paramValList; + } + }); + if (Object.keys(paramSet).length > 0) { + paramGrid.push(paramSet); + } + }); + + state['param_grid'] = this.paramStringify(paramGrid); + + // import code + code.appendLine(gsConfig.import); + code.appendLine(estConfig.import); + + // model code + let gsCode = gsConfig.code; + gsCode = com_generator.vp_codeGenerator(this, gsConfig, state, (userOption != ''? ', ' + userOption : '')); + code.appendLine(); + code.appendFormat('{0} = {1}', allocateToCreation, gsCode); + + return code.toString(); + } + + paramStringify(paramGridList=[]) { + let paramGridCode = new com_String(); + if (paramGridList.length > 1) { + paramGridCode.append('['); + } + paramGridList.forEach((paramSet, i) => { + if (i > 0) { + paramGridCode.append(', \n '); + } + paramGridCode.append('{'); + Object.keys(paramSet).forEach((paramName, j) => { + if (j > 0) { + paramGridCode.append(', '); + } + paramGridCode.appendFormat("'{0}': [{1}]", paramName, paramSet[paramName].toString()); + }); + paramGridCode.append('}'); + }) + + if (paramGridList.length > 1) { + paramGridCode.append(']'); + } + return paramGridCode.toString(); + } + + } + + return GridSearch; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 00cdcf4a..1d450a0a 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -779,6 +779,39 @@ define([ } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js new file mode 100644 index 00000000..1c2e2bcc --- /dev/null +++ b/visualpython/js/m_ml/Pipeline.js @@ -0,0 +1,544 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Pipeline.js + * Author : Black Logic + * Note : Pipeline + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] Pipeline +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/pipeline.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/pipeline'), + __VP_RAW_LOADER__('vp_base/data/libraries.json'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, librariesJson, com_util, com_String, com_generator, ML_LIBRARIES, PopupComponent, SuggestInput, ModelEditor) { + + /** + * Pipeline + */ + class Pipeline extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 4; + this.config.dataview = false; + this.config.autoScroll = false; + + this.state = { + templateType: '', + modelStep: -1, // get modelStep sequence + modelType: '', // get modelType + modelTypeName: '', // get modelTypeName + model: 'model', + // now selected pipeline info + pipeline: [ + // copy of step + info + // { name, label, useApp, state, app } + ], + ...this.state + } + + /** + * useApp + - Data Split + - Data Prep + - Regressor + - Classifier + - Clustering + - Dimension + - GridSearch + - Evaluation + + - Fit + - Transform + - Predict + */ + this.templateList = { + 'data-prep': { + label: 'Data Preparation', + modelStep: 0, + step: [ + /** + * ml_* is pre-defined app + * pp_* is defined only for Pipeline + */ + { name: 'ml_dataPrep', label: 'Data Prep', useApp: true }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' } + ] + }, + 'regression': { + label: 'Regression', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_regression', label: 'Regressor', useApp: true }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'rgs' } }, + ] + }, + 'classification': { + label: 'Classification', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_classification', label: 'Classifier', useApp: true }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'clf' } }, + ] + }, + 'clustering': { + label: 'Clustering', + modelStep: 0, + step: [ + { name: 'ml_clustering', label: 'Clustering', useApp: true }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'cls' } }, + ] + }, + 'dimension': { + label: 'Dimension Reduction', + modelStep: 0, + step: [ + { name: 'ml_dimensionReduction', label: 'Dimension Reduction', useApp: true }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' } + ] + }, + 'gridSearch': { + label: 'GridSearch', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_gridSearch', label: 'GridSearch', useApp: true }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true }, + ] + } + } + + // menu libraries for ml + let libObj = JSON.parse(librariesJson); + this.mlAppList = libObj.library.item.filter(x => x.id === 'pkg_ml')[0].item; + + this.modelConfig = ML_LIBRARIES; + } + + _unbindEvent() { + super._unbindEvent(); + + $(document).off('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]')); + $(document).off('click', this.wrapSelector('.vp-pp-item-toggle')); + $(document).off('change', this.wrapSelector(`#modelType`)); + $(document).off('change', this.wrapSelector('#allocateToCreation')); + + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select template + $(this.wrapSelector('#templateType')).on('change', function() { + let type = $(this).val(); + that.state.templateType = type; + that.state.model = 'model'; + that.state.modelType = ''; + that.state.modelTypeName = ''; + that.state.modelStep = -1; + + // reset pp-page + $(that.wrapSelector('.vp-pp-template')).html(''); + $(that.wrapSelector('.vp-pp-step-content')).html(''); + $(that.wrapSelector('.vp-pp-step-title')).text(''); + that.state.pipeline = []; + + that.handleChangeTemplate(type); + }); + + // select pipeline item + $(document).on('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]'), function() { + if (!$(this).hasClass('selected')) { + let title = $(this).data('label'); + let stepSeq = parseInt($(this).data('seq')); // 0 ~ n + let name = $(this).data('name'); + let ppObj = that.state.pipeline[stepSeq]; + // set title + $(that.wrapSelector('.vp-pp-step-title')).text(title); + + // show page + $(that.wrapSelector(`.vp-pp-step-page:not([data-name="${name}"])`)).hide(); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).show(); + if (ppObj.useApp === true) { + ppObj.app && ppObj.app.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`))); + } else { + that.renderApp(name); + } + + // check selected + $(that.wrapSelector('.vp-pp-item')).removeClass('selected'); + $(this).addClass('selected'); + } + }); + + // pipeline item toggle (enable/disable) + $(document).on('click', this.wrapSelector('.vp-pp-item-toggle'), function(evt) { + evt.stopPropagation(); + let itemTag = $(this).closest('.vp-pp-item'); + let name = $(itemTag).attr('data-name'); + let flag = $(itemTag).attr('data-flag'); // enabled / disabled + if (flag === 'enabled') { + $(itemTag).attr('data-flag', 'disabled'); + $(this).prop('checked', false); + } else if (flag === 'disabled') { + $(itemTag).attr('data-flag', 'enabled'); + $(this).prop('checked', true); + } + + if ($(itemTag).hasClass('selected')) { + // check if this page is this item's, then hide it. + $(that.wrapSelector('.vp-pp-step-title')).text(''); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + $(itemTag).removeClass('selected'); + } + + }); + + // model type change event + $(document).on('change', this.wrapSelector(`#modelType`), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + + let modelType = $(this).val(); + let modelObj = that.modelConfig[modelType]; + let modelTypeName = modelObj.code.split('(')[0]; + + that.state.modelType = modelType; + that.state.modelTypeName = modelTypeName; + + // show fit / predict / transform depends on model selection + let defaultActions = ['fit', 'predict', 'transform']; + let actions = that.modelEditor.getAction(modelTypeName); + defaultActions.forEach(actKey => { + if (actions[actKey] === undefined) { + // if undefined, hide step + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).show(); + } + }); + + }); + + // model allocation variable change + $(document).on('change', this.wrapSelector('#allocateToCreation'), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelAllocation = $(this).val(); + that.state.model = modelAllocation; + }); + } + + /** + * + * @param {*} type template type + */ + handleChangeTemplate(type) { + let that = this; + if (type !== '') { + let tplObj = this.templateList[type]; + this.state.modelStep = tplObj.modelStep; + + let ppTag = new com_String(); + let appFileList = []; + // load pipeline items + tplObj.step.forEach((stepObj, idx) => { + let { name, label, useApp=false, state={} } = stepObj; + ppTag.appendFormatLine(`

+ {3} + +

`, name, idx, label, label); + + // get pages + if (useApp === true) { + let mlObj = that.mlAppList.filter(x => x.id === name)[0]; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.push({ index: idx, name: name, file: './' + mlObj.file}); + } else { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } + } + let pipeObj = { + name: name, + label: label, + useApp: useApp, + state: state + }; + if (tplObj.modelStep === idx) { + pipeObj.modelStep = true; + } + that.state.pipeline.push(pipeObj); + // append pages + $(that.wrapSelector('.vp-pp-step-content')).append( + `

`); + if (useApp === false) { + that.renderApp(name); + // hide + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + } + + }); + $(that.wrapSelector('.vp-pp-template')).html(ppTag.toString()); + + // render pages + // for lite and lab + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.forEach((obj, argIdx) => { + let MlComponent = require(obj.file); + if (MlComponent) { + // DUP AREA: pp-1 + let { name, label, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-seq="0"]')).click(); + // end of DUP AREA: pp-1 + } + }); + + } else { + // for notebook and others + window.require(appFileList.map(x => x.file), function() { + appFileList.forEach((obj, argIdx) => { + let MlComponent = arguments[argIdx]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, label, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-seq="0"]')).click(); + // end of DUP AREA: pp-1 + } + }) + + }); + } + } + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Template list creation + //================================================================ + let tplTag = new com_String(); + tplTag.appendLine(''); + Object.keys(this.templateList).forEach(tplKey => { + let tplObj = that.templateList[tplKey]; + let selectedFlag = ''; + if (tplKey == that.state.templateType) { + selectedFlag = 'selected'; + } + tplTag.appendFormatLine('', + tplKey, selectedFlag, tplObj.label); + }); + $(page).find('#templateType').html(tplTag.toString()); + + return page; + } + + render() { + super.render(); + + // resize codeview + $(this.wrapSelector('.vp-popup-codeview-box')).css({'height': '300px'}); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + // load state + if (this.state.templateType !== '') { + $(this.wrapSelector('#templateType')).trigger('change'); + } + } + + /** + * Handle app view before open + * @param {string} appId + * @param {*} mlApp + */ + handleAppView(appId, mlApp) { + switch (appId) { + case 'ml_dataSplit': + $(mlApp.wrapSelector('#inputData')).parent().hide(); + break; + } + } + + /** + * Custom pages + * @param {*} appId + */ + renderApp(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let tag = ''; + switch (appId) { + case 'pp_fit': + tag = this.templateForOptionPage(actions['fit']); + break; + case 'pp_predict': + tag = this.templateForOptionPage(actions['predict']); + break; + case 'pp_transform': + tag = this.templateForOptionPage(actions['transform']); + break; + } + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(` +

${tag}

+ `); + } + + templateForOptionPage(packageObj) { + let optBox = new com_String(); + // render tag + packageObj && packageObj.options && packageObj.options.forEach(opt => { + let label = opt.name; + if (opt.label != undefined) { + label = opt.label; + } + // fix label + label = com_util.optionToLabel(label); + optBox.appendFormatLine('{2}' + , opt.name, opt.name, label); + let content = com_generator.renderContent(this, opt.component[0], opt, this.state); + optBox.appendLine(content[0].outerHTML); + }); + return optBox.toString(); + } + + generateCodeForOptionPage(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let actObj = {}; + switch (appId) { + case 'pp_fit': + actObj = actions['fit']; + break; + case 'pp_predict': + actObj = actions['predict']; + break; + case 'pp_transform': + actObj = actions['transform']; + break; + } + + let code = new com_String(); + if (actObj.import != undefined) { + code.appendLine(actObj.import); + code.appendLine(); + } + + let that = this; + let state = { + model: this.state.model + }; + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"] .vp-state`)).each((idx, tag) => { + let id = tag.id; + let value = that._getTagValue(tag); + state[id] = value; + }); + + let modelCode = com_generator.vp_codeGenerator(this, actObj, state); + modelCode = modelCode.replace('${model}', state.model); + code.append(modelCode); + return { state: state, code: code.toString() }; + } + + generateCode() { + let that = this; + let { template } = this.state; + let code = new com_String(); + + let stepNo = 1; + this.state.pipeline.forEach((ppObj, idx) => { + let { name, label, useApp, app } = ppObj; + + // check disabled + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-seq="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-seq="${idx}"]`)).data('flag') === 'enabled'; + if (isVisible && isEnabled) { + if (code.toString() !== '') { + code.appendLine(); + code.appendLine(); + } + code.appendFormatLine("# [{0}] {1}", stepNo, label); + if (useApp) { + code.append(app.generateCode()); + // save state + that.state.pipeline[idx].state = app.state; + } else { + let ppResult = that.generateCodeForOptionPage(name); + code.append(ppResult.code); + // save state + that.state.pipeline[idx].state = ppResult.state; + } + stepNo++; + } + }); + + return code.toString(); + } + + } + + return Pipeline; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/evaluation.js b/visualpython/js/m_ml/evaluation.js index 6231b4a3..6b36f6ba 100644 --- a/visualpython/js/m_ml/evaluation.js +++ b/visualpython/js/m_ml/evaluation.js @@ -33,7 +33,7 @@ define([ this.config.checkModules = ['metrics']; this.state = { - modelType: 'rgs', + modelType: 'rgs', // rgs / clf / cls predictData: 'pred', targetData: 'y_test', // regression @@ -156,35 +156,6 @@ define([ }); $(page).find('#targetData2').replaceWith(targetData2Selector.toTagString()); - // load state - let that = this; - Object.keys(this.state).forEach(key => { - let tag = $(page).find('#' + key); - let tagName = $(tag).prop('tagName'); // returns with UpperCase - let value = that.state[key]; - if (value == undefined) { - return; - } - switch(tagName) { - case 'INPUT': - let inputType = $(tag).prop('type'); - if (inputType == 'text' || inputType == 'number' || inputType == 'hidden') { - $(tag).val(value); - break; - } - if (inputType == 'checkbox') { - $(tag).prop('checked', value); - break; - } - break; - case 'TEXTAREA': - case 'SELECT': - default: - $(tag).val(value); - break; - } - }); - $(page).find('.vp-upper-box').hide(); $(page).find('.vp-upper-box.' + this.state.modelType).show(); diff --git a/visualpython/js/m_stats/FactorAnalysis.js b/visualpython/js/m_stats/FactorAnalysis.js index 6d5ca900..188efe19 100644 --- a/visualpython/js/m_stats/FactorAnalysis.js +++ b/visualpython/js/m_stats/FactorAnalysis.js @@ -160,7 +160,12 @@ define([ } generateInstallCode() { - return [ '!pip install factor-analyzer']; + let installCode = '!pip install factor-analyzer'; + // Add installation code + if (vpConfig.extensionType === 'lite') { + installCode = '%pip install factor-analyzer'; + } + return [ installCode ]; } generateCode() { @@ -168,13 +173,6 @@ define([ let codeList = []; let code = new com_String(); - // Add installation code FIXME: - if (vpConfig.extensionType === 'lite') { - codeList.push('%pip install factor-analyzer'); - } else { - codeList.push('!pip install factor-analyzer'); - } - // data declaration code.appendFormat("vp_df = {0}", data); if (this.columnSelector) { diff --git a/visualpython/js/m_stats/Regression.js b/visualpython/js/m_stats/Regression.js index 68b2c49d..709caf4e 100644 --- a/visualpython/js/m_stats/Regression.js +++ b/visualpython/js/m_stats/Regression.js @@ -508,7 +508,7 @@ define([ code.appendLine("# Mean Centering "); independentValue = com_util.formatString("{0}_MC", independentValue); moderatedValue = com_util.formatString("{0}_MC", moderatedValue); - code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", independentValue, independent, independent); + code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", independentValue, independent, independent); code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", moderatedValue, moderated, moderated); } // Model 1 to 3 diff --git a/visualpython/js/m_stats/StudentstTest.js b/visualpython/js/m_stats/StudentstTest.js index fc0c7341..8947c5b2 100644 --- a/visualpython/js/m_stats/StudentstTest.js +++ b/visualpython/js/m_stats/StudentstTest.js @@ -123,18 +123,18 @@ define([ var colName = $(this).find('option:selected').text(); var colDtype = $(this).find('option:selected').attr('data-type'); that.state.groupingVariable = colCode; - $(that.wrapSelector('#group1')).html(''); - $(that.wrapSelector('#group2')).html(''); // get result and load column list vpKernel.getColumnCategory(that.state.data, colCode).then(function(resultObj) { let { result } = resultObj; + $(that.wrapSelector('#group1')).html(''); + $(that.wrapSelector('#group2')).html(''); try { var category = JSON.parse(result); if (category && category.length > 0 && colDtype == 'object') { // if it's categorical column and its dtype is object, check 'Text' as default category.forEach(obj => { let selected1 = obj.value === that.state.group1; - let selected2 = obj.value === that.state.group1; + let selected2 = obj.value === that.state.group2; $(that.wrapSelector('#group1')).append(``); $(that.wrapSelector('#group2')).append(``); }); diff --git a/visualpython/js/m_visualize/Seaborn.js b/visualpython/js/m_visualize/Seaborn.js index b153c3b0..7bb2759d 100644 --- a/visualpython/js/m_visualize/Seaborn.js +++ b/visualpython/js/m_visualize/Seaborn.js @@ -361,29 +361,33 @@ define([ let val = $(this).val(); if (val !== '') { // enable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', false); + $(that.wrapSelector('#xticks_label')).attr('readonly', false); } else { // disable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', true); + $(that.wrapSelector('#xticks_label')).attr('readonly', true); } }); $(this.wrapSelector('#yticks')).on('change', function() { let val = $(this).val(); if (val !== '') { // enable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', false); + $(that.wrapSelector('#yticks_label')).attr('readonly', false); } else { // disable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', true); + $(that.wrapSelector('#yticks_label')).attr('readonly', true); } }); // axes - ticks label: inform user to type location option to use label - $(this.wrapSelector('#xticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#xticks')).focus(); + $(this.wrapSelector('#xticks_label')).on('click', function() { + if ($(that.wrapSelector('#xticks')).val() === '') { + $(that.wrapSelector('#xticks')).focus(); + } }); - $(this.wrapSelector('#yticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#yticks')).focus(); + $(this.wrapSelector('#yticks_label')).on('click', function() { + if ($(that.wrapSelector('#yticks')).val() === '') { + $(that.wrapSelector('#yticks')).focus(); + } }); // preview refresh diff --git a/visualpython/visualpython.yaml b/visualpython/visualpython.yaml index 11ae71f3..a4e3d5bb 100644 --- a/visualpython/visualpython.yaml +++ b/visualpython/visualpython.yaml @@ -5,28 +5,4 @@ Link: README.md Icon: img/logo.png Main: visualpython.js Compatibility: 4.x, 5.x, 6.x -Parameters: -- name: vpcfg.run_code_without_asking - description: Run code without asking - input_type: checkbox - default: false -- name: vpcfg.change_task_without_asking - description: Change task without asking - input_type: checkbox - default: false -- name: vpcfg.code_insert_position - description: Code insert position - input_type: text - default: below -- name: vpcfg.api_list_sort_by - description: Api list sorting by... - input_type: text - default: frequency -- name: vpcfg.default_variable_for_required - description: Default variable for required input - input_type: checkbox - default: false -- name: vpcfg.auto_import_package - description: Automatically import packages - input_type: checkbox - default: false +Parameters: \ No newline at end of file