	sepal_length	sepal_width	petal_length	petal_width	variety
0	5.1	3.5	1.4	0.2	Setosa
1	4.9	3.0	1.4	0.2	Setosa
2	4.7	3.2	1.3	0.2	Setosa
3	4.6	3.1	1.5	0.2	Setosa
4	5.0	3.6	1.4	0.2	Setosa
...	...	...	...	...	...
145	6.7	3.0	5.2	2.3	Virginica
146	6.3	2.5	5.0	1.9	Virginica
147	6.5	3.0	5.2	2.0	Virginica
148	6.2	3.4	5.4	2.3	Virginica
149	5.9	3.0	5.1	1.8	Virginica

sepal_length

sepal_width

petal_length

petal_width

variety

5.1

3.5

1.4

0.2

Setosa

4.9

3.0

1.4

0.2

Setosa

4.7

3.2

1.3

0.2

Setosa

4.6

3.1

1.5

0.2

Setosa

5.0

3.6

1.4

0.2

Setosa

...

145

6.7

3.0

5.2

2.3

Virginica

146

6.3

2.5

5.0

1.9

Virginica

147

6.5

3.0

5.2

2.0

Virginica

148

6.2

3.4

5.4

2.3

Virginica

149

5.9

3.0

5.1

1.8

Virginica

	variety	sepal_length	sepal_width	petal_length	petal_width
0	Setosa	5.1	3.5	1.4	0.2
1	Setosa	4.9	3.0	1.4	0.2
2	Setosa	4.7	3.2	1.3	0.2
3	Setosa	4.6	3.1	1.5	0.2
4	Setosa	5.0	3.6	1.4	0.2
...	...	...	...	...	...
145	Virginica	6.7	3.0	5.2	2.3
146	Virginica	6.3	2.5	5.0	1.9
147	Virginica	6.5	3.0	5.2	2.0
148	Virginica	6.2	3.4	5.4	2.3
149	Virginica	5.9	3.0	5.1	1.8

variety

sepal_length

sepal_width

petal_length

petal_width

Setosa

5.1

3.5

1.4

0.2

Setosa

4.9

3.0

1.4

0.2

Setosa

4.7

3.2

1.3

0.2

Setosa

4.6

3.1

1.5

0.2

Setosa

5.0

3.6

1.4

0.2

...

145

Virginica

6.7

3.0

5.2

2.3

146

Virginica

6.3

2.5

5.0

1.9

147

Virginica

6.5

3.0

5.2

2.0

148

Virginica

6.2

3.4

5.4

2.3

149

Virginica

5.9

3.0

5.1

1.8

Type Data

@@ -22,7 +22,7 @@

Option Page -

diff --git a/visualpython/html/component/fileNavigation.html b/visualpython/html/component/fileNavigation.html index 187b39c7..e02507b4 100644 --- a/visualpython/html/component/fileNavigation.html +++ b/visualpython/html/component/fileNavigation.html @@ -10,7 +10,7 @@ --> -

diff --git a/visualpython/html/component/packageManager.html b/visualpython/html/component/packageManager.html index 956fd6e6..1e8ca640 100644 --- a/visualpython/html/component/packageManager.html +++ b/visualpython/html/component/packageManager.html @@ -12,12 +12,16 @@

+ +

diff --git a/visualpython/html/component/popupComponent.html b/visualpython/html/component/popupComponent.html index 64b0f58d..961f83e8 100644 --- a/visualpython/html/component/popupComponent.html +++ b/visualpython/html/component/popupComponent.html @@ -62,7 +62,7 @@

Install Package @@ -91,11 +91,11 @@

+ -

Options

diff --git a/visualpython/html/m_ml/gridSearch.html b/visualpython/html/m_ml/gridSearch.html new file mode 100644 index 00000000..ee276db9 --- /dev/null +++ b/visualpython/html/m_ml/gridSearch.html @@ -0,0 +1,41 @@ + +

+ Model type +

+ +

+ Allocate to + +

+ Param grid +

+ Param set 1 +

+ +

+ \ No newline at end of file diff --git a/visualpython/html/m_ml/modelInfo.html b/visualpython/html/m_ml/modelInfo.html index 690efb8b..02badc27 100644 --- a/visualpython/html/m_ml/modelInfo.html +++ b/visualpython/html/m_ml/modelInfo.html @@ -38,7 +38,7 @@

Options

diff --git a/visualpython/html/m_ml/pipeline.html b/visualpython/html/m_ml/pipeline.html new file mode 100644 index 00000000..f9d5570e --- /dev/null +++ b/visualpython/html/m_ml/pipeline.html @@ -0,0 +1,29 @@ + +

+ Pipeline + +

+ +

+ \ No newline at end of file diff --git a/visualpython/html/m_stats/anova.html b/visualpython/html/m_stats/anova.html index 17b356bd..582cd58c 100644 --- a/visualpython/html/m_stats/anova.html +++ b/visualpython/html/m_stats/anova.html @@ -50,8 +50,8 @@

Significance level - Post hoc analysis -

+ Post hoc analysis +

Tukey HSD Tukey diff --git a/visualpython/html/m_stats/probDist.html b/visualpython/html/m_stats/probDist.html index 0bab3ef5..e1a54fa7 100644 --- a/visualpython/html/m_stats/probDist.html +++ b/visualpython/html/m_stats/probDist.html @@ -20,8 +20,8 @@

Action diff --git a/visualpython/html/m_stats/regression.html b/visualpython/html/m_stats/regression.html index 124f7066..93ac63fd 100644 --- a/visualpython/html/m_stats/regression.html +++ b/visualpython/html/m_stats/regression.html @@ -76,6 +76,7 @@ Residual Normality test Residual histogram Residual scatterplot + RMSE

diff --git a/visualpython/html/m_stats/studentstTest.html b/visualpython/html/m_stats/studentstTest.html index c250b06d..d1244cb5 100644 --- a/visualpython/html/m_stats/studentstTest.html +++ b/visualpython/html/m_stats/studentstTest.html @@ -40,9 +40,15 @@

Group 1 - +

+ + Text +

Group 2 - +

+ + Text +

diff --git a/visualpython/html/m_visualize/seaborn.html b/visualpython/html/m_visualize/seaborn.html index 3d560f7f..08e99bd7 100644 --- a/visualpython/html/m_visualize/seaborn.html +++ b/visualpython/html/m_visualize/seaborn.html @@ -100,6 +100,14 @@ Bins

+ Orient + +

Kde

+ Errorbar +

Legend

diff --git a/visualpython/img/apps/apps_gridSearch.svg b/visualpython/img/apps/apps_gridSearch.svg new file mode 100644 index 00000000..0d6ba9b5 --- /dev/null +++ b/visualpython/img/apps/apps_gridSearch.svg @@ -0,0 +1,6 @@ + diff --git a/visualpython/img/apps/apps_pipeline.svg b/visualpython/img/apps/apps_pipeline.svg new file mode 100644 index 00000000..aab67d91 --- /dev/null +++ b/visualpython/img/apps/apps_pipeline.svg @@ -0,0 +1,9 @@ + diff --git a/visualpython/img/item_disable.svg b/visualpython/img/item_disable.svg new file mode 100644 index 00000000..9f3970cc --- /dev/null +++ b/visualpython/img/item_disable.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_disable_hover.svg b/visualpython/img/item_disable_hover.svg new file mode 100644 index 00000000..6491b8c6 --- /dev/null +++ b/visualpython/img/item_disable_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_enable.svg b/visualpython/img/item_enable.svg new file mode 100644 index 00000000..ab75f8a5 --- /dev/null +++ b/visualpython/img/item_enable.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_enable_hover.svg b/visualpython/img/item_enable_hover.svg new file mode 100644 index 00000000..02c24e50 --- /dev/null +++ b/visualpython/img/item_enable_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/status_green.svg b/visualpython/img/status_green.svg new file mode 100644 index 00000000..13e43ec3 --- /dev/null +++ b/visualpython/img/status_green.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/status_orange.svg b/visualpython/img/status_orange.svg new file mode 100644 index 00000000..93b230c1 --- /dev/null +++ b/visualpython/img/status_orange.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/status_red.svg b/visualpython/img/status_red.svg new file mode 100644 index 00000000..6fe4a8bc --- /dev/null +++ b/visualpython/img/status_red.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/js/MainFrame.js b/visualpython/js/MainFrame.js index 1b9d0386..b16149b1 100644 --- a/visualpython/js/MainFrame.js +++ b/visualpython/js/MainFrame.js @@ -354,7 +354,7 @@ define([ $('#vp_wrapper').width(newVpWidth); if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: set parent width and position, min-width - let target = $('#vp_wrapper').parent(); + let target = $('#vp_wrapper').closest('#jp-right-stack'); // compatible for notebook 7.x let prevWidth = target[0].getBoundingClientRect().width; let prevLeft = $(target).position().left; let widthDiff = newVpWidth - vpWidth; @@ -367,11 +367,23 @@ define([ vpLab.shell.collapseRight(); vpLab.shell.expandRight(); } - let relativeSizes = vpLab.shell._hsplitPanel.layout.relativeSizes(); - let absoluteSizes = vpLab.shell._hsplitPanel.layout.absoluteSizes(); - let newSize = absoluteSizes[1] - widthDiff; - let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); - vpLab.shell._hsplitPanel.layout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + if (vpLab.name === 'JupyterLab') { + let splitLayout = vpLab.shell._hsplitPanel.layout; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.absoluteSizes(); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } else { + // compatible for notebook 7.x + // vpLab.shell.layout.widgets[0] + let splitLayout = vpLab.shell.layout.widgets[0]; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.widgets.map(x => x.node.getBoundingClientRect().width); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } } else { $('#vp_wrapper').resizable({ minWidth: MENU_MIN_WIDTH + MENU_BOARD_SPACING }); } @@ -384,7 +396,7 @@ define([ $('#vp_wrapper').width(newVpWidth); if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: set parent width and position, min-width - let target = $('#vp_wrapper').parent(); + let target = $('#vp_wrapper').closest('#jp-right-stack'); // compatible for notebook 7.x let prevWidth = target[0].getBoundingClientRect().width; let prevLeft = $(target).position().left; let widthDiff = newVpWidth - vpWidth; @@ -397,11 +409,23 @@ define([ vpLab.shell.collapseRight(); vpLab.shell.expandRight(); } - let relativeSizes = vpLab.shell._hsplitPanel.layout.relativeSizes(); - let absoluteSizes = vpLab.shell._hsplitPanel.layout.absoluteSizes(); - let newSize = absoluteSizes[1] - widthDiff; - let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); - vpLab.shell._hsplitPanel.layout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + if (vpLab.name === 'JupyterLab') { + let splitLayout = vpLab.shell._hsplitPanel.layout; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.absoluteSizes(); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } else { + // compatible for notebook 7.x + // vpLab.shell.layout.widgets[0] + let splitLayout = vpLab.shell.layout.widgets[0]; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.widgets.map(x => x.node.getBoundingClientRect().width); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } } else { $('#vp_wrapper').resizable({ minWidth: VP_MIN_WIDTH }); } @@ -496,10 +520,10 @@ define([ let parentBlock = null; let prevBlock = null; loadStateList.forEach(obj => { - let { blockType, menuId, menuState, menuConfig, argIdx, position, afterAction } = obj; + let { file, blockType, menuId, menuState, menuConfig, argIdx, position, afterAction } = obj; // get OptionComponent Object // LAB: relative path needed - let OptionComponent = require('./' + menuConfig.file); + let OptionComponent = require('./' + file); if (OptionComponent) { let taskState = menuState.taskState; let blockState = menuState.blockState; diff --git a/visualpython/js/com/com_Config.js b/visualpython/js/com/com_Config.js index e6bc6379..9062699a 100644 --- a/visualpython/js/com/com_Config.js +++ b/visualpython/js/com/com_Config.js @@ -65,7 +65,7 @@ define([ this.data = { // Configuration 'vpcfg': { - + 'runType': 'run' }, // User defined code for Snippets 'vpudf': { @@ -75,7 +75,8 @@ define([ 'import matplotlib.pyplot as plt', '%matplotlib inline', 'import seaborn as sns', - 'import plotly.express as px' + 'import plotly.express as px', + 'import pyarrow as pa' ], 'matplotlib customizing': [ 'import matplotlib.pyplot as plt', @@ -132,7 +133,8 @@ define([ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' ] - } + }, + { library: 'pyarrow', alias:'pa' }, ] } @@ -197,7 +199,7 @@ define([ code: 'import joblib', type: 'package' }, - 'scipy.stats': { + 'stats': { code: 'from scipy import stats', type: 'package' }, @@ -205,9 +207,13 @@ define([ code: 'import scipy', type: 'package' }, - 'statsmodels.api': { + 'sm': { code: 'import statsmodels.api as sm', type: 'package' + }, + 'pyarrow': { + code: 'import pyarrow as pa', + type: 'package' } } @@ -264,8 +270,7 @@ define([ vp_config_version: '1.0.0', vp_signature: 'VisualPython', vp_position: {}, - // CHROME: default to display vp - vp_section_display: true, + vp_section_display: false, vp_note_display: false, vp_menu_width: Config.MENU_MIN_WIDTH, vp_note_width: Config.BOARD_MIN_WIDTH @@ -569,7 +574,7 @@ define([ that._checkMounted().then(function() { that._readFromColab(configKey).then(function(result) { let data = result; - if (data == undefined || data == {}) { + if (data == undefined || (data instanceof Object && Object.keys(data).length === 0)) { resolve(data); return; } @@ -577,7 +582,7 @@ define([ resolve(data); return; } - if (Object.keys(data).length > 0) { + if (data instanceof Object && Object.keys(data).length > 0) { resolve(data[dataKey]); return; } @@ -593,7 +598,7 @@ define([ // LAB: use local .visualpython files that._readFromLab(configKey).then(function(result) { let data = result; - if (data == undefined || data == {}) { + if (data == undefined || (data instanceof Object && Object.keys(data).length === 0)) { resolve(data); return; } @@ -601,7 +606,7 @@ define([ resolve(data); return; } - if (Object.keys(data).length > 0) { + if (data instanceof Object && Object.keys(data).length > 0) { resolve(data[dataKey]); return; } @@ -637,7 +642,7 @@ define([ /** * Set configuration data (on server) * @param {Object} dataObj - * @param {String} configKey + * @param {String} configKey vpcfg / vpudf / vpimport / vppackman */ setData(dataObj, configKey='vpudf') { let that = this; @@ -845,7 +850,7 @@ define([ throw new Error('Error', response); } }).then(function (data) { - resolve(data.info.version); + resolve(data.info.version, data.releases); }).catch(function(err) { let errMsg = err.message; if (errMsg.includes('Failed to fetch')) { @@ -863,6 +868,40 @@ define([ return Config.version; } + checkVersionTimestamp = function() { + let that = this; + // check version timestamp + let nowDate = new Date(); + this.getData('version_timestamp', 'vpcfg').then(function(data) { + let doCheckVersion = false; + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Checking its version timestamp... : ' + data); + if (data == undefined || (data instanceof Object && Object.keys(data).length === 0)) { + // no timestamp, check version + doCheckVersion = true; + } else if (data != '') { + let lastCheck = new Date(parseInt(data)); + let diffCheck_now = new Date(nowDate.getFullYear(), nowDate.getMonth() + 1, nowDate.getDate()); + let diffCheck_last = new Date(lastCheck.getFullYear(), lastCheck.getMonth() + 1, lastCheck.getDate()); + + let diff = Math.abs(diffCheck_now.getTime() - diffCheck_last.getTime()); + diff = Math.ceil(diff / (1000 * 3600 * 24)); + + if (diff >= 1) { + // if More than 1 day passed, check version + doCheckVersion = true; + } + } + + // check version and update version_timestamp + if (doCheckVersion == true) { + that.checkVpVersion(true); + } + + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + }) + } + checkVpVersion(background=false) { let that = this; let nowVersion = this.getVpInstalledVersion(); @@ -871,14 +910,29 @@ define([ packageName = 'jupyterlab-visualpython'; } this.getPackageVersion(packageName).then(function(latestVersion) { - if (nowVersion === latestVersion) { + let showUpdater = false; + if (nowVersion !== latestVersion) { + let nowVerParts = nowVersion.split('.').map(x => ~~x); + let latVerParts = latestVersion.split('.').map(x => ~~x); + for (var i = 0; i < nowVerParts.length; i++) { + const a = nowVerParts[i]; + const b = latVerParts[i]; + if (a < b) { + showUpdater = true; + break; + } else if (a > b) { + break; + } + } + } + if (showUpdater === false) { // if it's already up to date // hide version update icon $('#vp_versionUpdater').hide(); - if (background) { + if (background === true) { ; } else { - let msg = com_util.formatString('Visual Python is up to date. ({0})', latestVersion); + let msg = com_util.formatString('Visual Python is up to date. ({0})', nowVersion); com_util.renderInfoModal(msg); } // update version_timestamp @@ -902,6 +956,8 @@ define([ switch (clickedBtnIdx) { case 0: // cancel + // update version_timestamp + that.setData({ 'version_timestamp': new Date().getTime() }, 'vpcfg'); break; case 1: // update @@ -1017,7 +1073,7 @@ define([ /** * Version */ - Config.version = "2.4.5"; + Config.version = "3.0.2"; /** * Type of mode @@ -1068,11 +1124,11 @@ define([ Config.ML_DATA_DICT = { 'Data Preparation': [ /** Encoding */ - 'OneHotEncoder', 'LabelEncoder', 'OrdinalEncoder', 'TargetEncoder', 'SMOTE', + 'OneHotEncoder', 'LabelEncoder', 'OrdinalEncoder', 'TargetEncoder', /** Scaling */ 'StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer', 'FunctionTransformer', 'PolynomialFeatures', 'KBinsDiscretizer', /** ETC */ - 'ColumnTransformer' + 'SimpleImputer', 'SMOTE', 'ColumnTransformer' ], 'Regression': [ 'LinearRegression', 'Ridge', 'Lasso', 'ElasticNet', 'SVR', 'DecisionTreeRegressor', 'RandomForestRegressor', 'GradientBoostingRegressor', 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor', @@ -1088,6 +1144,9 @@ define([ ], 'Auto ML': [ 'AutoSklearnRegressor', 'AutoSklearnClassifier', 'TPOTRegressor', 'TPOTClassifier' + ], + 'ETC': [ + 'GridSearchCV' ] }; @@ -1097,7 +1156,8 @@ define([ ...Config.ML_DATA_DICT['Classification'], ...Config.ML_DATA_DICT['Clustering'], ...Config.ML_DATA_DICT['Dimension Reduction'], - ...Config.ML_DATA_DICT['Auto ML'] + ...Config.ML_DATA_DICT['Auto ML'], + ...Config.ML_DATA_DICT['ETC'] ]; return Config; diff --git a/visualpython/js/com/com_Const.js b/visualpython/js/com/com_Const.js index b7e9d7cd..fbd18788 100644 --- a/visualpython/js/com/com_Const.js +++ b/visualpython/js/com/com_Const.js @@ -19,7 +19,7 @@ define ([ class Constants { } Constants.TOOLBAR_BTN_INFO = { - HELP: "Visual Python 2.4.5" + HELP: "Visual Python 3.0.2" , ICON: "vp-main-icon" , ID: "vpBtnToggle" , NAME: "toggle-vp" diff --git a/visualpython/js/com/com_Event.js b/visualpython/js/com/com_Event.js index 3cc1aece..10b3e09b 100644 --- a/visualpython/js/com/com_Event.js +++ b/visualpython/js/com/com_Event.js @@ -207,8 +207,13 @@ define([], function() { that.keyManager.keyCheck.shiftKey = false; } if (evt.keyCode == that.keyManager.keyCode.escKey) { + // check if there is visible file navigation : FileNavigation + if ($('.vp-filenavigation-base:visible').length > 0) { + // close file navigation + $('.vp-filenavigation-base:visible').remove(); + } // check if there is visible data selector : DataSelector - if ($('.vp-dataselector-base:visible').length > 0) { + else if ($('.vp-dataselector-base:visible').length > 0) { // close data selector $('.vp-dataselector-base:visible').remove(); } diff --git a/visualpython/js/com/com_Kernel.js b/visualpython/js/com/com_Kernel.js index 4282c1ad..addb7093 100644 --- a/visualpython/js/com/com_Kernel.js +++ b/visualpython/js/com/com_Kernel.js @@ -473,14 +473,27 @@ define([ getLabNotebookPanel(){ var mainWidgets = this.app.shell.widgets('main'); var widget = mainWidgets.next(); - while(widget){ - if(widget.sessionContext){ + while (widget) { + if (widget.sessionContext) { var type = widget.sessionContext.type; if(type == 'notebook' || type == 'console'){ //other wigets might be of type DocumentWidget if (widget.isVisible){ return widget; } } + } else if (widget.value !== undefined && widget.done === false) { + // for upper lab 4 + let widgetObj = widget.value; + if (widgetObj.sessionContext) { + var type = widgetObj.sessionContext.type; + if(type == 'notebook' || type == 'console'){ //other wigets might be of type DocumentWidget + if (widgetObj.isVisible){ + return widgetObj; + } + } + } + } else if (widget.done === true) { + break; } widget = mainWidgets.next(); } @@ -494,6 +507,14 @@ define([ while (widget) { if (widget.sessionContext) { widgetList.push(widget); + } else if (widget.value !== undefined && widget.done === false) { + // for upper lab 4 + let widgetObj = widget.value; + if (widgetObj.sessionContext) { + widgetList.push(widgetObj); + } + } else if (widget.done === true) { + break; } widget = mainWidgets.next(); } @@ -589,10 +610,14 @@ define([ * @param {*} dataframe * @returns */ - getRowList(dataframe) { + getRowList(dataframe, start_idx) { var that = this; + if (typeof start_idx === 'undefined') { + start_idx = 0; + } + return new Promise(function(resolve, reject) { - that.execute(com_util.formatString('_vp_print(_vp_get_rows_list({0}))', dataframe)) + that.execute(com_util.formatString('_vp_print(_vp_get_rows_list({0}, {1}))', dataframe, start_idx)) .then(function(resultObj) { resolve(resultObj); }).catch(function(err) { diff --git a/visualpython/js/com/com_generatorV2.js b/visualpython/js/com/com_generatorV2.js index 3cff1f0e..42e6c760 100644 --- a/visualpython/js/com/com_generatorV2.js +++ b/visualpython/js/com/com_generatorV2.js @@ -14,8 +14,9 @@ define([ 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/com/component/FileNavigation' -], function (com_util, com_makeDom, SuggestInput, VarSelector2, DataSelector, FileNavigation) { +], function (com_util, com_makeDom, SuggestInput, VarSelector2, DataSelector, MultiSelector, FileNavigation) { /** * show result after code executed */ @@ -39,6 +40,7 @@ define([ 'var_select': 'Select Variable', 'var_multi': 'Select N-Variables', 'col_select': 'Select Column', + 'col_multi': 'Select Columns', 'textarea': 'Input textarea', 'input_number': 'Input number', 'input_text': 'Input text', @@ -137,7 +139,7 @@ define([ package.options && package.options.forEach(function(o, i) { var obj = JSON.parse(JSON.stringify(o)); let newTag = vp_createTag(pageThis, obj, state); - if (obj.required) { + if (obj.required === true || obj.output === true) { tblInput.append(newTag); } else { tblOption.append(newTag); @@ -145,7 +147,11 @@ define([ }); // TODO: userOption + if (package.code.includes('${etc}')) { + + } + bindMultiSelector(pageThis); bindAutoComponentEvent(pageThis); } @@ -171,7 +177,7 @@ define([ } else if (output === true) { requiredFontStyle = 'vp-bold'; } - var lblTag = $(`${label}`).attr({ + var lblTag = $(`${label || com_util.optionToLabel(name)}`).attr({ 'for': name, 'class': requiredFontStyle, 'title': '(' + name + ')' @@ -312,7 +318,8 @@ define([ case 'option_select': var optSlct = $('').attr({ 'class':'vp-select option-select vp-state', - 'id':obj.name + 'id':obj.name, + 'title': (obj.help==undefined?'':obj.help), }); obj.options.forEach((opt, idx, arr) => { var label = (obj.options_label != undefined? obj.options_label[idx]:opt); @@ -365,9 +372,12 @@ define([ let dataSelector = new DataSelector({ pageThis: pageThis, id: obj.name, + type: obj.comp_type || 'data', allowDataType: obj.var_type, placeholder: obj.placeholder || 'Select data', value: value, + columnSelection: obj.columnSelection || 'multiple', // single / multiple + returnFrameType: obj.returnFrameType || '', // '' / DataFrame / Series required: obj.required === true }); content = $(dataSelector.toTagString()); @@ -379,7 +389,8 @@ define([ id: obj.name, class: 'vp-input vp-state', placeholder: obj.placeholder || 'Select data', - required: obj.required === true + required: obj.required === true, + title: (obj.help==undefined?'':obj.help), }); vp_generateVarSuggestInput(pageThis.wrapSelector(), obj); content = tag; @@ -390,7 +401,8 @@ define([ 'id': obj.name, 'class': 'vp-select var-multi vp-state', // multiple selection true - 'multiple': true + 'multiple': true, + 'title': (obj.help==undefined?'':obj.help), }); vp_generateVarSelect(tag, obj.var_type, obj.value); content = tag; @@ -399,10 +411,28 @@ define([ var tag = $('').attr({ 'type': 'text', 'id': obj.name, - 'class': 'vp-input vp-state' + 'class': 'vp-input vp-state', + 'title': (obj.help==undefined?'':obj.help), }); content = tag; break; + case 'col_multi': + var tag1 = $('

').attr({ + 'id': obj.name + '_auto_comp', + 'class': 'vp-auto-multi-column', + 'data-id': obj.name, + 'data-target': obj.comp_target || 'i0', + 'style': 'height: 150px;', + 'title': (obj.help==undefined?'':obj.help), + }); + var tag2 = $('').attr({ + type: 'hidden', + id: obj.name, + class: 'vp-input vp-state', + }) + var tag = $('

').append(tag1).append(tag2); + content = tag; + break; case 'textarea': var textarea = $(``); // cell metadata test @@ -707,6 +737,17 @@ define([ case 'dtype': value = $(pageThis.wrapSelector(parent + ' #'+obj.name)).val(); break; + case 'col_multi': + let colList = pageThis.autoGen[obj.name].getDataList(); + pageThis.state[obj.name] = colList.map(data => { return data.code }); + value = colList.map(data => { return data.code }).join(','); + if (colList.length == 0) { + value = ''; + } else if (colList.length > 0) { + value = '[' + value + ']'; + } + $(pageThis.wrapSelector('#'+obj.name)).val(value); + break; case 'file-open': case 'file-save': case 'table': @@ -738,7 +779,9 @@ define([ package.options && package.options.forEach(function(v, i) { var val = state[v.name]; if (val == undefined || val == '' || val == v.default) { - val = vp_getTagValue(pageThis, v, parent=parent); + if (pageThis) { + val = vp_getTagValue(pageThis, v, parent=parent); + } } var id = '${' + v.name + '}'; if (val == undefined || val.trim() == '') { @@ -759,7 +802,11 @@ define([ } // code completion 2 if (v.usePair == true) { - val = ', ' + v.name + '=' + val; + if (v.pairKey) { + val = ', ' + v.pairKey + '=' + val; + } else { + val = ', ' + v.name + '=' + val; + } } // code completion 3 if (v.component != undefined && v.componentCode != undefined) { @@ -799,9 +846,11 @@ define([ if (_VP_SHOW_RESULT && package.options) { var outputOptList = package.options.filter(x => x.output === true); var outputStr = ''; - outputOptList.forEach(opt => { - outputStr += (outputStr !== ''?', ':'') + vp_getTagValue(pageThis, opt); - }) + if (pageThis) { + outputOptList.forEach(opt => { + outputStr += (outputStr !== ''?', ':'') + vp_getTagValue(pageThis, opt); + }) + } if (outputStr != '') { code += '\n'+ outputStr; } @@ -843,7 +892,7 @@ define([ suggestInputX.addClass('vp-input vp-state'); suggestInputX.setNormalFilter(false); suggestInputX.setValue(defaultValue); - $(selector + ' #' + columnInputId).replaceWith(function() { + $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return suggestInputX.toTagString(); }); } else { @@ -852,7 +901,7 @@ define([ 'id': columnInputId, 'class': 'vp-select vp-state' }); - $(selector + ' #' + columnInputId).replaceWith(function() { + $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return $(tag); }); } @@ -908,7 +957,7 @@ define([ suggestInputX.setSuggestList(function() { return list; }); //FIXME: suggestInputX.setNormalFilter(false); suggestInputX.setValue(defaultValue); - $(selector + ' #' + columnInputId).replaceWith(function() { + $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return suggestInputX.toTagString(); }); } else { @@ -1225,6 +1274,63 @@ define([ return $(''); } + var bindMultiSelector = function(pageThis) { + //==================================================================== + // for column multi selector + //==================================================================== + $(pageThis.wrapSelector('.vp-auto-multi-column')).each((idx, tag) => { + let compId = tag.id; + let id = $(tag).data('id'); + let targetId = $(tag).data('target'); + let colSelector = new MultiSelector( + pageThis.wrapSelector('#' + compId), + { + mode: 'columns', parent: (pageThis.state[targetId] || ''), selectedList: pageThis.state[compId], + change: function(type, list) { + let value = list.map(data => { return data.code }).join(','); + if (list.length == 0) { + value = ''; + } else if (list.length > 0) { + value = '[' + value + ']'; + } + pageThis.state[compId] = list.map(data => { return data.code }); + pageThis.state[id] = value; + $(pageThis.wrapSelector('#'+id)).val(value); + } + } + ); + pageThis.autoGen = { + [id]: colSelector, + ...pageThis.autoGen + }; + $(pageThis.wrapSelector('#' + targetId)).on('change', function() { + let targetVariable = $(this).val(); + let colSelector = new MultiSelector( + pageThis.wrapSelector('#' + compId), + { + mode: 'columns', parent: targetVariable, selectedList: pageThis.state[compId], + change: function(type, list) { + let value = list.map(data => { return data.code }).join(','); + if (list.length == 0) { + value = ''; + } else if (list.length > 0) { + value = '[' + value + ']'; + } + pageThis.state[compId] = list.map(data => { return data.code }); + pageThis.state[id] = value; + $(pageThis.wrapSelector('#'+id)).val(value); + } + }, + ); + pageThis.autoGen = { + [id]: colSelector, + ...pageThis.autoGen + }; + }); + + }); + } + var bindAutoComponentEvent = function(pageThis) { let selector = pageThis.wrapSelector(); // Auto-component selector @@ -1488,7 +1594,6 @@ define([ $(pageThis.wrapSelector('#'+id)).val(code); }); - //==================================================================== // Event for tabBlock //==================================================================== diff --git a/visualpython/js/com/com_interface.js b/visualpython/js/com/com_interface.js index 4c12f1e5..ae7800a4 100644 --- a/visualpython/js/com/com_interface.js +++ b/visualpython/js/com/com_interface.js @@ -122,13 +122,23 @@ define([ var notebook = notebookPanel.content; var notebookModel = notebook.model; - var cellModel = notebookModel.contentFactory.createCell(type, {}); - cellModel.value.text = command; - - const newCellIndex = notebook.activeCellIndex + 1; + // LAB 4.x: notebookPanel.content.contentFactory.createCodeCell / createMarkdownCell + // var cellModel = notebookModel.contentFactory.createCell(type, {}); // 셀 추가 - notebookModel.cells.insert(newCellIndex, cellModel); - notebook.activeCellIndex = newCellIndex; + // LAB 4.x: insert cell api 변경 + // notebookModel.cells.insert(newCellIndex, cellModel); + // notebook.activeCellIndex = newCellIndex; + NotebookActions.insertBelow(notebook); + try { + NotebookActions.changeCellType(notebook, type); + } catch(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + } + // let activeCellIndex = notebook.activeCellIndex; + // let cell = notebookModel.cells.model.getCell(activeCellIndex); + // cell.setSource(command); + let activeCell = notebook.activeCell; + activeCell.model.sharedModel.setSource(command); if (exec == true) { try{ @@ -145,18 +155,19 @@ define([ var widget = labConsole.widgets[0]; if (type === 'markdown') { // add # before the lines - command = '#' + command.split('\n').join('#'); + command = '#' + command.split('\n').join('\n#'); } - + + // LAB 4.x: console insert and run + widget.promptCell.model.sharedModel.setSource(command) // execute or not if (exec == true) { // console allow only code cell - var cellModel = widget.createCodeCell(); - cellModel.model.value.text = command; - widget.addCell(cellModel); - widget._execute(cellModel); - } else { - widget.promptCell.model.value.text = command; + // var cellModel = widget.createCodeCell(); + // cellModel.model.value.text = command; + // widget.addCell(cellModel); + // widget._execute(cellModel); + widget.execute() } widget.promptCell.inputArea.editor.focus(); } @@ -263,15 +274,24 @@ define([ var notebookModel = notebook.model; var options = { }; - var cellModel = notebookModel.contentFactory.createCell(type, options); - cellModel.value.text = command; - - const newCellIndex = notebook.activeCellIndex + 1; + // LAB 4.x: notebookPanel.content.contentFactory.createCodeCell / createMarkdownCell + // var cellModel = notebookModel.contentFactory.createCell(type, options); // 셀 추가 - notebookModel.cells.insert(newCellIndex, cellModel); - notebook.activeCellIndex = newCellIndex; - - var cell = notebook.activeCell; + // LAB 4.x: insert cell api 변경 + // notebookModel.cells.insert(newCellIndex, cellModel); + // notebook.activeCellIndex = newCellIndex; + NotebookActions.insertBelow(notebook); + try { + NotebookActions.changeCellType(notebook, type); + } catch(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + } + // let activeCellIndex = notebook.activeCellIndex; + // let cell = notebookModel.cells.model.getCell(activeCellIndex); + // cell.setSource(command); + let activeCell = notebook.activeCell; + activeCell.model.sharedModel.setSource(command); + if (exec == true) { try{ NotebookActions.run(notebook, sessionContext); @@ -280,9 +300,27 @@ define([ } } } else if (sessionType === 'console') { - var console = notebookPanel.content; - var cellModel = console.contentFactory.createCell(type, {}); - cellModel.value.text = command; + // LAB 4.x: notebookPanel.console.contentFactory.createCodeCell + // var console = notebookPanel.content; + // var cellModel = console.contentFactory.createCell(type, {}); + var labConsole = notebookPanel.content; + var widget = labConsole.widgets[0]; + if (type === 'markdown') { + // add # before the lines + command = '#' + command.split('\n').join('\n#'); + } + + // LAB 4.x: console insert and run + widget.promptCell.model.sharedModel.setSource(command) + // execute or not + if (exec == true) { + // console allow only code cell + // var cellModel = widget.createCodeCell(); + // cellModel.model.value.text = command; + // widget.addCell(cellModel); + // widget._execute(cellModel); + widget.execute() + } } } else { // No session found @@ -299,10 +337,13 @@ define([ } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: TODO: - let activeCell = notebookPanel.content.activeCell; - let activeCellTop = $(activeCell.node).position().top; - // scroll to current cell - $(vpKernel.getLabNotebookPanel().content.activeCell.node)[0].scrollIntoView(true); + let sessionType = notebookPanel.sessionContext.type; + if (sessionType === 'notebook') { + let activeCell = notebookPanel.content.activeCell; + let activeCellTop = $(activeCell.node).position().top; + // scroll to current cell + $(notebookPanel.content.activeCell.node)[0].scrollIntoView(true); + } } com_util.renderSuccessMessage('Your code is successfully generated.'); diff --git a/visualpython/js/com/com_util.js b/visualpython/js/com/com_util.js index 2afb8190..5f63ecf7 100644 --- a/visualpython/js/com/com_util.js +++ b/visualpython/js/com/com_util.js @@ -245,6 +245,31 @@ define([ return label; } + /** + * Pack the code with ignoring the warnings + * @param {*} code + */ + var ignoreWarning = function(code) { + var convertedCode = new com_String(); + convertedCode.appendLine('import warnings'); + convertedCode.appendLine('from IPython.display import display'); + convertedCode.appendLine('with warnings.catch_warnings():'); + convertedCode.appendLine(" warnings.simplefilter('ignore')"); + let codeLines = code.split('\n'); + codeLines.forEach((line, idx) => { + if (idx > 0) { + convertedCode.appendLine(); + } + if (codeLines.length == idx + 1) { + // last line + convertedCode.appendFormat(" display({0})", line); + } else { + convertedCode.appendFormat(" {0}", line); + } + }); + return convertedCode.toString(); + } + //============================================================================ // Cross-browser RegEx Split //============================================================================ @@ -377,6 +402,7 @@ define([ setIsAPIListRunCode: setIsAPIListRunCode, getIsAPIListRunCode: getIsAPIListRunCode, safeString: safeString, + ignoreWarning: ignoreWarning, regex_split: regex_split } diff --git a/visualpython/js/com/component/AlertModal.js b/visualpython/js/com/component/AlertModal.js index 6931eef2..813b7f9c 100644 --- a/visualpython/js/com/component/AlertModal.js +++ b/visualpython/js/com/component/AlertModal.js @@ -21,9 +21,14 @@ define([ /** * AlertModal + * com_util.renderAlertModal(title, detail); + * - title: string + * - detail: object + * - content: string + * - type: text / code */ class AlertModal extends Component { - constructor(title, detail='') { + constructor(title, detail={ content:'', type:'text' }) { super($('body'), { title: title, detail: detail }); } @@ -44,9 +49,16 @@ define([ // set title $(this.wrapSelector('.vp-alertModal-titleStr')).text(this.state.title); - // set detail as tooltip - if (this.state.detail) { - $(this.wrapSelector('.vp-alertModal-titleStr')).attr({ title: this.state.detail }); + // set detail + let { content='', type='text' } = this.state.detail; + if (content !== '') { + if (type === 'code') { + $(this.wrapSelector('.vp-alertModal-detailStr')).html('

' + content + '

') + } else { + $(this.wrapSelector('.vp-alertModal-detailStr')).text(content); + } + } else { + $(this.wrapSelector('.vp-alertModal-detailStr')).hide(); } } diff --git a/visualpython/js/com/component/DataSelector.js b/visualpython/js/com/component/DataSelector.js index d6d34108..1b64b9ee 100644 --- a/visualpython/js/com/component/DataSelector.js +++ b/visualpython/js/com/component/DataSelector.js @@ -50,7 +50,7 @@ define([ super._init(); this.prop = { - type: 'data', // selector type : data / column + type: 'data', // selector type : data / column / ml pageThis: null, // target's page object id: '', // target id value: null, // pre-defined value @@ -58,6 +58,12 @@ define([ finish: null, // callback after selection (value, dtype) select: null, // callback after selection from suggestInput (value, dtype) allowDataType: null, // list of allowed data types + dataCategory: null, // list of data category (use it for ml categories) + columnSelection: 'multiple', // single/multi : allowed column selection + returnFrameType: '', // DataFrame/Series : required data type for DataFrame -> Series/DataFrame operation + // if Series, only one column selection is allowed and returns Series + // if DataFrame, always returns DataFrame + // if Empty(=== ''), if one column selected, returns Series / others, returns DataFrame // additional options boxClasses: '', classes: '', @@ -71,9 +77,23 @@ define([ // If null, define default allow data type if (!this.prop.allowDataType) { // default allow data types - this.prop.allowDataType = [ - 'DataFrame', 'Series', 'ndarray', 'list', 'dict' - ] + if (this.prop.type === 'ml') { + this.prop.allowDataType = vpConfig.getMLDataTypes(); + } else { + this.prop.allowDataType = [ + 'DataFrame', 'Series', 'ndarray', 'list', 'dict' + ] + } + } + if (!this.prop.dataCategory) { + if (this.prop.type === 'ml') { + this.prop.dataCategory = vpConfig.getMLCategories(); + } else { + this.prop.dataCategory = this.prop.allowDataType; + } + } + if (this.prop.returnFrameType === 'Series') { + this.prop.columnSelection = 'single'; // only single selection allowed } this.state = { @@ -86,6 +106,7 @@ define([ slicingEnd1: '', slicingStart2: '', slicingEnd2: '', + singleColumn: '', ndRowType: 'slicing', ndColType: 'slicing', useIndex: false, @@ -282,7 +303,11 @@ define([ that.renderVariableBox(that._varList.filter(obj => !that.prop.allowDataType.includes(obj.dtype))); } else { // filter variable list - that.renderVariableBox(that._varList.filter(obj => obj.dtype == type)); + if (that.prop.type === 'ml') { + that.renderVariableBox(that._varList.filter(obj => vpConfig.getMLDataDict(type).includes(obj.dtype))); + } else { + that.renderVariableBox(that._varList.filter(obj => obj.dtype == type)); + } } }); @@ -393,6 +418,19 @@ define([ } } + templateForSingleSelector() { + return ` +

+ Use Index +

+ Column + +

+ ` + } + templateForMultiSelector() { return `

@@ -530,7 +568,7 @@ define([ let varTags = new com_String(); let types = [ 'All', - ...this.prop.allowDataType, + ...this.prop.dataCategory, 'Others' ]; // Add Data Types to filter @@ -551,10 +589,17 @@ define([ let that = this; let varTags = new com_String(); varTags = new com_String(); - varList && varList.forEach((obj, idx) => { - varTags.appendFormatLine('

{4}

' - , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label); - }); + if (this.prop.type === 'ml') { + varList && varList.forEach((obj, idx) => { + varTags.appendFormatLine('

{4} | {5}

' + , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label, obj.dtype); + }); + } else { + varList && varList.forEach((obj, idx) => { + varTags.appendFormatLine('

{4}

' + , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label); + }); + } $(this.wrapSelector('.vp-ds-variable-box')).html(varTags.toString()); // focus on selected item @@ -576,22 +621,93 @@ define([ switch (dataType) { case 'DataFrame': - // render option page - $(this.wrapSelector('.vp-ds-option-inner-box')).html(this.templateForMultiSelector()); // column selecting - this._columnSelector = new MultiSelector(this.wrapSelector('.vp-ds-df-multiselector'), - { mode: 'columns', parent: [data], selectedList: this.state.indexing, allowAdd: true } - ); - + if (this.prop.columnSelection === 'single') { + // render option page + $(this.wrapSelector('.vp-ds-option-inner-box')).html(this.templateForSingleSelector()); + // bind column source + // com_generator.vp_bindColumnSource(this, 'data', ['singleColumn'], 'select', false, false); + // vp_bindColumnSource(pageThis, targetId, columnInputIdList, tagType="input", columnWithEmpty=false, columnWithIndex=false) + const columnInputIdList = ['singleColumn']; + if (data === '') { + // reset with no source + columnInputIdList && columnInputIdList.forEach(columnInputId => { + let defaultValue = that.state[columnInputId]; + if (defaultValue === null || defaultValue === undefined) { + defaultValue = ''; + } + // option tags + var tag = $('').attr({ + 'id': columnInputId, + 'class': 'vp-select vp-state' + }); + $(that.wrapSelector('#' + columnInputId)).replaceWith(function() { + return $(tag); + }); + }); + return ; + } + // get result and show on detail box + vpKernel.getColumnList(data).then(function(resultObj) { + try { + let { result, type, msg } = resultObj; + var { list } = JSON.parse(result); + + // columns using suggestInput + columnInputIdList && columnInputIdList.forEach((columnInputId, idx) => { + let defaultValue = that.state[columnInputId]; + if (defaultValue === null || defaultValue === undefined) { + defaultValue = ''; + } + // create tag + var tag = $('').attr({ + 'id': columnInputId, + 'class': 'vp-select vp-state w150' + }); + // make tag + list.forEach((listVar, idx) => { + var option = document.createElement('option'); + $(option).attr({ + 'value':listVar.value, + 'text':listVar.label, + 'data-type':listVar.dtype + }); + // cell metadata test : defaultValue as selected + if (listVar.value === defaultValue) { + $(option).prop('selected', true); + } + option.append(document.createTextNode(listVar.label)); + $(tag).append(option); + }); + $(that.wrapSelector('#' + columnInputId)).replaceWith(function() { + return $(tag); + }); + $(that.wrapSelector('#' + columnInputId)).trigger('change'); + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, 'com_generator - bindColumnSource error ', err) + }); + } catch (e) { + vpLog.display(VP_LOG_TYPE.ERROR, 'com_generator - bindColumnSource: not supported data type. ', e); + } + }); + } else { + // render option page + $(this.wrapSelector('.vp-ds-option-inner-box')).html(this.templateForMultiSelector()); + this._columnSelector = new MultiSelector(this.wrapSelector('.vp-ds-df-multiselector'), + { mode: 'columns', parent: [data], selectedList: this.state.indexing, allowAdd: true } + ); + } // bind event $(this.wrapSelector('#useIndex')).on('change', function() { let checked = $(this).prop('checked'); that.state.useIndex = checked; if (checked === true) { $(that.wrapSelector('.vp-ds-df-multiselector')).hide(); + $(that.wrapSelector('.vp-ds-df-singleselector')).hide(); $(that.wrapSelector('.vp-ds-df-index-box')).show(); } else { $(that.wrapSelector('.vp-ds-df-multiselector')).show(); + $(that.wrapSelector('.vp-ds-df-singleselector')).show(); $(that.wrapSelector('.vp-ds-df-index-box')).hide(); } }); @@ -705,6 +821,7 @@ define([ let { data, dataType, useIndex, + singleColumn, slicingStart1, slicingEnd1, slicingStart2, slicingEnd2, ndRowType, ndColType @@ -719,21 +836,39 @@ define([ code.append('.index'); } else { // use column selector - if (this._columnSelector != null) { - let result = this._columnSelector.getDataList(); - this.state.indexing = result.map(obj => obj.code); // save state - let columnList = []; - result && result.forEach(obj => { - columnList.push(obj.code); - }); - if (columnList.length > 0) { - if (columnList.length == 1) { - // return as Series - code.appendFormat('[{0}]', columnList.join(', ')); - // change datatype to Series - this.state.returnDataType = 'Series'; - } else { - code.appendFormat('[[{0}]]', columnList.join(', ')); + if (this.prop.columnSelection === 'single') { + // single selector + if (this.prop.returnFrameType === 'DataFrame') { + // return as DataFrame + code.appendFormat('[[{0}]]', singleColumn); + } else { + // return as Series + code.appendFormat('[{0}]', singleColumn); + this.state.returnDataType = 'Series'; + } + } else { + // multiple selector + if (this._columnSelector != null) { + let result = this._columnSelector.getDataList(); + this.state.indexing = result.map(obj => obj.code); // save state + let columnList = []; + result && result.forEach(obj => { + columnList.push(obj.code); + }); + if (columnList.length > 0) { + if (columnList.length == 1) { + if (this.prop.returnFrameType === 'DataFrame') { + // return as DataFrame + code.appendFormat('[[{0}]]', columnList.join(', ')); + } else { + // return as Series + code.appendFormat('[{0}]', columnList.join(', ')); + // change datatype to Series + this.state.returnDataType = 'Series'; + } + } else { + code.appendFormat('[[{0}]]', columnList.join(', ')); + } } } } @@ -743,7 +878,7 @@ define([ case 'list': code.append(data); // start / end value - if ((slicingStart1 && slicingStart1 != '') || (slicingEnd1 && slicingEnd1 != '')) { + if ((slicingStart1 != null && slicingStart1 != '') || (slicingEnd1 != null && slicingEnd1 != '')) { code.appendFormat('[{0}:{1}]', slicingStart1, slicingEnd1); } break; @@ -758,7 +893,7 @@ define([ let colCode = ''; if (ndRowType == 'slicing') { // slicing start / end value - if ((slicingStart1 && slicingStart1 != '') || (slicingEnd1 && slicingEnd1 != '')) { + if ((slicingStart1 != null && slicingStart1 != '') || (slicingEnd1 != null && slicingEnd1 != '')) { rowCode = com_util.formatString('{0}:{1}', slicingStart1, slicingEnd1); } } else { @@ -779,7 +914,7 @@ define([ } if (ndColType == 'slicing') { // slicing start / end value - if ((slicingStart2 && slicingStart2 != '') || (slicingEnd2 && slicingEnd2 != '')) { + if ((slicingStart2 != null && slicingStart2 != '') || (slicingEnd2 != null && slicingEnd2 != '')) { colCode = com_util.formatString('{0}:{1}', slicingStart2, slicingEnd2); } } else { @@ -807,7 +942,7 @@ define([ } } else { // start / end value - if ((slicingStart1 && slicingStart1 != '') || (slicingEnd1 && slicingEnd1 != '')) { + if ((slicingStart1 != null && slicingStart1 != '') || (slicingEnd1 != null && slicingEnd1 != '')) { code.appendFormat('[{0}:{1}]', slicingStart1, slicingEnd1); } } diff --git a/visualpython/js/com/component/FileNavigation.js b/visualpython/js/com/component/FileNavigation.js index 603ff8a0..506ffc0a 100644 --- a/visualpython/js/com/component/FileNavigation.js +++ b/visualpython/js/com/component/FileNavigation.js @@ -84,6 +84,10 @@ define([ this.pathStackPointer = -1; this.pathStack = []; this.currentFileList = []; + this.selectedExt = ''; + if (this.state.extensions.length > 0) { + this.selectedExt = this.state.extensions[0]; + } this.pathState = { parentPath: '', @@ -280,6 +284,32 @@ define([ // clear body $(this.wrapSelector('.fileNavigationPage-body')).html(''); + /** + * Filter file/dir which included in this.state.extensions + */ + if (Array.isArray(this.state.extensions) && this.state.extensions.length > 0 && this.state.extensions.toString() !== '') { + fileList = fileList.filter((data, index) => { + if (index == 0) { + return true; + } + + if (data.type && data.type == 'dir') { + // if directory, just show + return true; + } else if (data.name) { + var extension = data.name.substring(data.name.lastIndexOf('.') + 1); + // if (that.state.extensions.includes(extension)) { + if (that.selectedExt === '' || extension === that.selectedExt) { + return true; + } else { + return false; + } + } else { + return false; + } + }); + } + // render file items let dirArr = []; let fileArr = []; @@ -411,12 +441,12 @@ define([ page.appendFormatLine('' , 'vp_fileNavigationInput', 'New File Name', this.state.fileName); page.appendFormatLine(''); page.appendFormatLine('', 'vp-filenavi-btn', 'select', 'Select'); @@ -430,6 +460,13 @@ define([ that.handleSelectFile(filePath, fileName); }); + // bind file extension change event + $(this.wrapSelector('#vp_fileNavigationExt')).on('change', function() { + let ext = $(this).val(); + that.selectedExt = ext; + + that.renderFileList(); + }); // bind save cancel event $(this.wrapSelector('.vp-filenavi-btn')).on('click', function() { let menu = $(this).data('menu'); @@ -477,10 +514,10 @@ define([ let that = this; /** Implement after rendering */ // if save mode - if (this.state.type == 'save') { - // render saving box - this.renderSaveBox(); - } + // if (this.state.type == 'save') { + // render saving box + this.renderSaveBox(); + // } // get current path this.getCurrentDirectory().then(function(currentPath) { @@ -527,20 +564,21 @@ define([ //============================================================================ // Set selection result //============================================================================ - if (this.state.type == 'save') { - // add as saving file - this.setSelectedFile(fileInput, pathInput); - } else { - // Manage result using finish function - let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented - let status = true; - let error = null; - vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); - this.state.finish(filesPath, status, error); + this.setSelectedFile(fileInput, pathInput); + // if (this.state.type == 'save') { + // // add as saving file + // this.setSelectedFile(fileInput, pathInput); + // } else { + // // Manage result using finish function + // let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented + // let status = true; + // let error = null; + // vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); + // this.state.finish(filesPath, status, error); - // remove and close file navigation - this.close(); - } + // // remove and close file navigation + // this.close(); + // } } getCurrentDirectory() { return vpKernel.getCurrentDirectory(); @@ -588,31 +626,6 @@ define([ return a - b; }); - /** - * Filter file/dir which included in this.state.extensions - */ - if (Array.isArray(that.state.extensions) && that.state.extensions.length > 0 && that.state.extensions.toString() !== '') { - filtered_varList = filtered_varList.filter((data, index) => { - if (index == 0) { - return true; - } - - if (data.type && data.type == 'dir') { - // if file, just show - return true; - } else if (data.name) { - var extension = data.name.substring(data.name.lastIndexOf('.') + 1); - if (that.state.extensions.includes(extension)) { - return true; - } else { - return false; - } - } else { - return false; - } - }); - } - vpLog.display(VP_LOG_TYPE.DEVELOP, 'FileNavigation - getFileList: ', filtered_varList); var { currentDirStr, currentRelativePathStr } = that.splitPathStrAndSetStack(dirObj, filtered_varList); @@ -633,7 +646,7 @@ define([ let { msg, result } = resultObj; // show error using alert - if (msg.content.evalue) { + if (msg.content && msg.content.evalue) { let resultStr = msg.content.evalue; //t.match(/\[Errno [0-9]+?\] (.*)/)[1] // get error message from traceback diff --git a/visualpython/js/com/component/HelpViewer.js b/visualpython/js/com/component/HelpViewer.js index ac8a8e68..84ccfe44 100644 --- a/visualpython/js/com/component/HelpViewer.js +++ b/visualpython/js/com/component/HelpViewer.js @@ -167,14 +167,18 @@ define([ return sbSelector.toString(); } - open(content, useHelp=true) { + open(content, useHelp=true, importCode='') { this.show(); let that = this; let code = content; if (useHelp === true) { - code = `print(help(${content}))`; + if (importCode !== '') { + code = importCode + '\n' + `print(help(${content}))`; + } else { + code = `print(help(${content}))`; + } } let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); diff --git a/visualpython/js/com/component/InstanceEditor.js b/visualpython/js/com/component/InstanceEditor.js index f687faa2..c3a030c0 100644 --- a/visualpython/js/com/component/InstanceEditor.js +++ b/visualpython/js/com/component/InstanceEditor.js @@ -349,11 +349,15 @@ define([ // add targetid as state if exists if (targetId) { popupState[targetId] = targetCode; + that.state[targetId] = targetCode; + } + if (that.optionPopup) { + that.optionPopup.remove(); + that.optionPopup = null; } that.optionPopup = new LibraryComponent(popupState, { pageThis: that, - useInputVariable: true, targetSelector: that.pageThis.wrapSelector('#' + that.targetId), finish: function(code) { @@ -554,16 +558,5 @@ define([ } } - - - - - - - - - - - return InstanceEditor; }) \ No newline at end of file diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index d847043b..c2fab057 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -80,8 +80,8 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Perform modeling from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } ] }, 'predict': { @@ -90,7 +90,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -100,7 +100,7 @@ define([ code: '${pred_prob_allocate} = ${model}.predict_proba(${pred_prob_featureData})', description: 'Predict class probabilities for X.', options: [ - { name: 'pred_prob_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_prob_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_prob_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -110,7 +110,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -125,7 +125,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -134,16 +134,21 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, 'transform': { - ...defaultActions['transform'], - description: 'Transform labels to normalized encoding.' + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -153,12 +158,75 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } } } + if (modelType == 'LabelEncoder') { + actions = { + ...actions, + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Fit Encoder/Scaler to X.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' } + ] + }, + 'fit_transform': { + name: 'fit_transform', + label: 'Fit and transform', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', + description: 'Fit Encoder/Scaler to X, then transform X.', + options: [ + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -181,7 +249,7 @@ define([ code: '${dec_allocate} = ${model}.decision_function(${dec_featureData})', description: 'Compute the decision function of X.', options: [ - { name: 'dec_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'dec_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'dec_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable' } ] } @@ -198,7 +266,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Fit and predict.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -215,7 +283,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Perform clustering from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_predict': { @@ -224,7 +292,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute clusters from a data or distance matrix and predict labels.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -238,7 +306,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Compute clustering.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'predict': { @@ -247,7 +315,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -257,7 +325,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute cluster centers and predict cluster index for each sample.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -271,7 +339,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Compute clustering and transform X to cluster-distance space.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -281,7 +349,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Transform X to a cluster-distance space.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -297,7 +365,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -306,7 +374,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -321,18 +389,18 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, 'fit_transform': { name: 'fit_transform', label: 'Fit and transform', - code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}, ${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_trans_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -342,7 +410,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -352,7 +420,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -366,7 +434,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -375,7 +443,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -385,7 +453,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -395,12 +463,58 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}${fit_targetData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } @@ -415,8 +529,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: '', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, @@ -434,15 +548,67 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ - { name: 'importance_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'importance_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' }, - { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, + 'feature_importances': { + name: 'feature_importances', + label: 'Feature importances', + code: "${fi_allocate} = vp_create_feature_importances(${model}, ${fi_featureData}${sort})", + description: 'Allocate feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fi_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'df_i' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true } + ] + }, + 'plot_feature_importances': { + name: 'plot_feature_importances', + label: 'Plot feature importances', + code: "vp_plot_feature_importances(${model}, ${fi_featureData}${sort}${top_count})", + description: 'Draw feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true }, + ] } } switch (category) { @@ -517,6 +683,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] @@ -535,8 +714,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -548,7 +727,44 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'], + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'Coefficient': { name: 'coef_', label: 'Coefficient', @@ -568,6 +784,19 @@ define([ ] } } + let svcList = [ + 'DecisionTreeRegressor', + 'RandomForestRegressor', + 'GradientBoostingRegressor', + 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor' + ]; + if (svcList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } break; case 'Classification': infos = { @@ -582,8 +811,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -595,7 +824,109 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'] + 'roc_curve': { + name: 'roc_curve', + label: 'ROC Curve', + import: 'from sklearn import metrics', + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.predict_proba(${roc_featureData})[:, 1])\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()", + description: '', + options: [ + { name: 'roc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'roc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'auc': { + name: 'auc', + label: 'AUC', + import: 'from sklearn import metrics', + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.predict_proba(${auc_featureData})[:, 1])', + description: '', + options: [ + { name: 'auc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, + } + + // feature importances + let clfList = [ + 'DecisionTreeClassifier', + 'RandomForestClassifier', + 'GradientBoostingClassifier', + 'XGBClassifier', + 'LGBMClassifier', + 'CatBoostClassifier', + ] + if (clfList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } + + // use decision_function on ROC, AUC + let decisionFunctionTypes = [ + 'LogisticRegression', 'SVC', 'GradientBoostingClassifier' + ]; + if (decisionFunctionTypes.includes(modelType)) { + infos = { + ...infos, + 'roc_curve': { + ...infos['roc_curve'], + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.decision_function(${roc_featureData}))\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()" + }, + 'auc': { + ...infos['auc'], + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.decision_function(${auc_featureData}))', + } + } } break; case 'Auto ML': @@ -611,16 +942,18 @@ define([ break; case 'Clustering': infos = { - // 'Size of clusters': { - // name: 'Size of clusters', - // code: "print(f'Size of clusters: {np.bincount(pred)}')", // FIXME: model.cluster_centers_ / use model info or hide it - // options: [] - // } + 'get_params': { + ...defaultInfos['get_params'] + } } if (modelType == 'KMeans') { infos = { ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Return the mean accuracy on the given test data and labels.' + }, 'cluster_centers_': { name: 'cluster_centers', label: 'Cluster centers', @@ -642,11 +975,21 @@ define([ code: "# import\nfrom scipy.cluster.hierarchy import dendrogram, ward\n\nlinkage_array = ward(${dendro_data})\ndendrogram(linkage_array, p=3, truncate_mode='level', no_labels=True)\nplt.show()", description: 'Draw a dendrogram', options: [ - { name: 'dendro_data', label: 'Data', component: ['var_select'], var_type: ['DataFrame'] } + { name: 'dendro_data', label: 'Data', component: ['data_select'], var_type: ['DataFrame'] } ] } } } + + if (modelType == 'GaussianMixture') { + infos = { + ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Compute the per-sample average log-likelihood of the given data X.' + } + } + } break; case 'Dimension Reduction': if (modelType == 'LDA') { @@ -657,8 +1000,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } @@ -686,12 +1029,45 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index 3c9033f7..d9d3d9fa 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -77,6 +77,7 @@ define([ } = this.config; this.mode = mode; // variable / columns / index / ndarray0 / ndarray1 / methods / data(given data) this.parent = parent; + this.varType = type; // for mode:variable, variable type list to search this.selectedList = selectedList; this.includeList = includeList; this.excludeList = excludeList; @@ -88,35 +89,45 @@ define([ this.dataList = dataList; // [ { value, code, type }, ... ] this.pointer = { start: -1, end: -1 }; + this.loadDataList(); + } + + render() { + ; + } + + loadDataList() { var that = this; - if (parent === '') { - this._executeCallback([]); - return; + if (this.mode !== 'variable' && this.mode !== 'data') { + if (this.parent == null || this.parent === '' || (Array.isArray(this.parent) && this.parent.length == 0)) { + this._executeCallback([]); + return; + } } - switch (mode) { + switch (this.mode) { case 'columns': - this._getColumnList(parent, function(dataList) { + this._getColumnList(this.parent, function(dataList) { that._executeCallback(dataList); }); break; case 'variable': - this._getVariableList(type, function(dataList) { + this._getVariableList(this.varType, function(dataList) { that._executeCallback(dataList); }); break; case 'index': - this._getRowList(parent, function(dataList) { + this._getRowList(this.parent, function(dataList) { that._executeCallback(dataList); }); break; case 'ndarray0': - this._getNdarray(parent, 0, function(dataList) { + this._getNdarray(this.parent, 0, function(dataList) { that._executeCallback(dataList); }); break; case 'ndarray1': - this._getNdarray(parent, 1, function(dataList) { + this._getNdarray(this.parent, 1, function(dataList) { that._executeCallback(dataList); }); break; @@ -126,10 +137,6 @@ define([ } } - // render() { - // ; - // } - _executeCallback(dataList) { if (this.includeList && this.includeList.length > 0) { dataList = dataList.filter(data => this.includeList.includes(data.code)); @@ -241,7 +248,8 @@ define([ ndList.push({ value: i, code: i, - type: 'int' + type: 'int', + location: i }); } callback(ndList); @@ -256,7 +264,7 @@ define([ } load() { - $(this.frameSelector).html(this.render()); + $(this.frameSelector).html(this.templateForMultiSelector()); this.bindEvent(); this.bindDraggable(); this._bindItemClickEvent(); @@ -270,7 +278,7 @@ define([ var name = $(colTags[i]).data('name'); var type = $(colTags[i]).data('type'); var code = $(colTags[i]).data('code'); - if (code) { + if (code != null) { dataList.push({ name: name, type: type, code: code}); } } @@ -278,7 +286,7 @@ define([ return dataList; } - render() { + templateForMultiSelector() { var that = this; var tag = new com_String(); @@ -337,6 +345,7 @@ define([ tag.appendLine('

'); } tag.appendLine('

'); // APP_SELECT_RIGHT + tag.appendLine(''); tag.appendLine('

'); // APP_SELECT_CONTAINER return tag.toString(); } @@ -355,22 +364,28 @@ define([ info = ''; } let iconStr = ''; + let infoStr = ''; if (mode === 'columns') { if (data.isNumeric === true) { - iconStr = ''; + iconStr = ''; } else { - iconStr = ''; + iconStr = ''; } + } else if (mode === 'variable') { + infoStr = ` | ${data.type}`; } // render item box - tag.appendFormatLine('

{7}{8}

' - , APP_SELECT_ITEM, APP_DRAGGABLE, data.location, data.value, data.type, data.code, info, iconStr, data.value); + tag.appendFormat('

' + , APP_SELECT_ITEM, APP_DRAGGABLE, data.location, data.value, data.type, data.code, info); + tag.appendFormat('{0}{1}{2}', iconStr, data.value, infoStr); + tag.appendLine('

'); }); tag.appendLine('

'); // APP_SELECT_BOX return tag.toString(); } renderSelectedBox(dataList) { + let mode = this.mode; var tag = new com_String(); tag.appendFormatLine('

', APP_SELECT_BOX, 'right', APP_DROPPABLE, 'no-selection vp-scrollbar'); // get data and make draggable items @@ -383,16 +398,21 @@ define([ info = ''; } let iconStr = ''; + let infoStr = ''; if (mode === 'columns') { if (data.isNumeric === true) { - iconStr = ''; + iconStr = ''; } else { - iconStr = ''; + iconStr = ''; } + } else if (mode === 'variable') { + infoStr = ` | ${data.type}`; } // render item box - tag.appendFormatLine('

{8}{9}

' - , APP_SELECT_ITEM, APP_DRAGGABLE, 'added', data.location, data.value, data.type, data.code, info, iconStr, data.value); + tag.appendFormat('

' + , APP_SELECT_ITEM, APP_DRAGGABLE, 'added', data.location, data.value, data.type, data.code, info); + tag.appendFormat('{0}{1}{2}', iconStr, data.value, infoStr); + tag.appendLine('

'); }); tag.appendLine('

'); // APP_SELECT_BOX return tag.toString(); @@ -423,62 +443,6 @@ define([ that._bindItemClickEvent(); }); - // item indexing - $(this.wrapSelector('.' + APP_SELECT_ITEM)).on('click', function(event) { - var dataIdx = $(this).attr('data-idx'); - var idx = $(this).index(); - var added = $(this).hasClass('added'); // right side added item? - var selector = ''; - - // remove selection for select box on the other side - if (added) { - // remove selection for left side - $(that.wrapSelector('.' + APP_SELECT_ITEM + ':not(.added)')).removeClass('selected'); - // set selector - selector = '.added'; - } else { - // remove selection for right(added) side - $(that.wrapSelector('.' + APP_SELECT_ITEM + '.added')).removeClass('selected'); - // set selector - selector = ':not(.added)'; - } - - if (vpEvent.keyManager.keyCheck.ctrlKey) { - // multi-select - that.pointer = { start: idx, end: -1 }; - $(this).toggleClass('selected'); - } else if (vpEvent.keyManager.keyCheck.shiftKey) { - // slicing - var startIdx = that.pointer.start; - - if (startIdx == -1) { - // no selection - that.pointer = { start: idx, end: -1 }; - } else if (startIdx > idx) { - // add selection from idx to startIdx - var tags = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); - for (var i = idx; i <= startIdx; i++) { - $(tags[i]).addClass('selected'); - } - that.pointer = { start: startIdx, end: idx }; - } else if (startIdx <= idx) { - // add selection from startIdx to idx - var tags = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); - for (var i = startIdx; i <= idx; i++) { - $(tags[i]).addClass('selected'); - } - that.pointer = { start: startIdx, end: idx }; - } - } else { - // single-select - that.pointer = { start: idx, end: -1 }; - // un-select others - $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).removeClass('selected'); - // select this - $(this).addClass('selected'); - } - }); - // item indexing - add all $(this.wrapSelector('.' + APP_SELECT_ADD_ALL_BTN)).on('click', function(event) { $(that.wrapSelector('.' + APP_SELECT_BOX + '.left .' + APP_SELECT_ITEM)).appendTo( @@ -566,6 +530,11 @@ define([ } }); + // refresh + $(this.wrapSelector('.vp-cs-refresh')).on('click', function(event) { + that.loadDataList(); + }); + this._bindItemClickEvent(); } @@ -677,6 +646,7 @@ define([ let newItemIndex = this.dataList.length; var targetTag = $(`

${newItemName} +

`); $(targetTag).appendTo( $(this.wrapSelector('.' + APP_SELECT_BOX + '.right')) @@ -686,6 +656,8 @@ define([ $(this.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('selected'); // clear item input $(this.wrapSelector('.vp-cs-add-item-name')).val(''); + // bind click event + this._bindItemClickEvent(); // bind draggable this.bindDraggable(); } diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js index 350e9093..a808315e 100644 --- a/visualpython/js/com/component/PackageManager.js +++ b/visualpython/js/com/component/PackageManager.js @@ -17,10 +17,12 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(ifHtml, ifCss, com_util, com_Const, com_String, PopupComponent, FileNavigation, LoadingSpinner) { +], function(ifHtml, ifCss, com_util, com_Const, com_String, com_interface, SuggestInput, PopupComponent, FileNavigation, LoadingSpinner) { /** * PackageManager @@ -45,24 +47,25 @@ define([ this.packageLibTemplate = { 'numpy': { pipName: 'numpy' }, 'pandas': { pipName: 'pandas' }, + 'pyarrow': { pipName: 'pyarrow' }, 'matplotlib': { pipName: 'matplotlib' }, 'seaborn': { pipName: 'seaborn' }, 'plotly': { pipName: 'plotly' }, 'wordcloud': { pipName: 'wordcloud' }, 'sklearn': { pipName: 'scikit-learn' }, - 'scikit-posthocs': { pipName: 'scikit-posthocs' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, 'scipy': { pipName: 'scipy' }, 'statsmodels': { pipName: 'statsmodels' }, - 'factor-analyzer': { pipName: 'factor-analyzer' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, 'pingouin': { pipName: 'pingouin' }, 'category_encoders': { pipName: 'category_encoders' }, 'imblearn': { pipName: 'imblearn' }, 'xgboost': { pipName: 'xgboost' }, 'lightgbm': { pipName: 'lightgbm' }, 'catboost': { pipName: 'catboost' }, - 'auto-sklearn': { pipName: 'auto-sklearn' }, + 'autosklearn': { pipName: 'auto-sklearn' }, 'tpot': { pipName: 'tpot' }, - 'PyMuPDF': { pipName: 'PyMuPDF' }, + 'pymupdf': { pipName: 'pymupdf' }, 'sweetviz': { pipName: 'sweetviz' }, } @@ -74,33 +77,39 @@ define([ 'seaborn': { pipName: 'seaborn' }, 'plotly': { pipName: 'plotly' }, 'sklearn': { pipName: 'scikit-learn' }, - 'scikit-posthocs': { pipName: 'scikit-posthocs' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, 'scipy': { pipName: 'scipy' }, 'statsmodels': { pipName: 'statsmodels' }, - 'factor-analyzer': { pipName: 'factor-analyzer' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, 'category_encoders': { pipName: 'category_encoders' }, 'imblearn': { pipName: 'imblearn' }, 'xgboost': { pipName: 'xgboost' }, 'lightgbm': { pipName: 'lightgbm' }, 'catboost': { pipName: 'catboost' }, - 'auto-sklearn': { pipName: 'auto-sklearn' }, + 'autosklearn': { pipName: 'auto-sklearn' }, 'sweetviz': { pipName: 'sweetviz' }, } } } + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('.vp-pm-search')); + } + _bindEvent() { super._bindEvent(); /** Implement binding events */ let that = this; // search item - $(this.wrapSelector('.vp-pm-search')).on('change', function(evt) { + $(document).on('change', this.wrapSelector('.vp-pm-search'), function(evt) { var value = $(this).val(); if (value != '') { $(that.wrapSelector('.vp-pm-item')).hide(); $(that.wrapSelector('.vp-pm-item')).filter(function() { - return $(this).data('key').search(value) >= 0; + let key = $(this).data('key'); + return key.search(value.toLowerCase()) >= 0; }).show(); } else { $(that.wrapSelector('.vp-pm-item')).show(); @@ -116,7 +125,14 @@ define([ // sort item $(this.wrapSelector('.vp-pm-sort-menu-item')).on('click', function() { var menu = $(this).data('menu'); - if (menu === 'name') { + if (menu === 'registered') { + // sort by name + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var keyA = parseInt($(a).data('seq')); + var keyB = parseInt($(b).data('seq')); + return keyA > keyB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'name') { // sort by name $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { var keyA = $(a).data('key'); @@ -147,6 +163,16 @@ define([ return insA > insB ? 1 : -1 }).appendTo($(that.wrapSelector('.vp-pm-table'))) } + $(that.wrapSelector('.vp-pm-sort-menu-box')).hide(); + }); + + // reload package list + $(this.wrapSelector('.vp-pm-func-reload')).on('click', function() { + // reset search keyword + $(that.wrapSelector('.vp-pm-search')).val(''); + + // load package list + that.loadPackageList(); }); // add package @@ -173,33 +199,40 @@ define([ if (vpConfig.extensionType === 'lite') { code = com_util.formatString("%pip uninstall {0}", pipName); } + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); } else if (menu === 'upgrade') { var pipName = that.packageLib[key].pipName; var code = com_util.formatString("!pip install --upgrade {0}", pipName); if (vpConfig.extensionType === 'lite') { code = com_util.formatString("%pip install {0}", pipName); } + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); } else if (menu === 'delete') { $(item).remove(); delete that.packageLib[key]; vpConfig.removeData('packageList', 'vppackman').then(function() { vpConfig.setData({ 'packageList': that.packageLib }, 'vppackman'); + that.loadPackageList(); }); } evt.stopPropagation(); @@ -307,14 +340,19 @@ define([ return false; } } + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + + // load package list + this.loadPackageList(); break; } @@ -355,11 +393,12 @@ define([ * * @param {String} key * @param {Object} info installed, version, path + * @param {number} index sequence of initial package list * @returns */ - renderPackageItem(key, info) { + renderPackageItem(key, info, index) { var item = new com_String(); - item.appendFormatLine('

', 'vp-pm-item', key, info.installed===true?'1':'0'); + item.appendFormatLine('

', 'vp-pm-item', key, info.installed===true?'1':'0', index); item.appendFormatLine('

', 'vp-pm-item-header', (info.path?info.path:'')); item.appendFormatLine('{0}', key); if (info.installed === true) { @@ -401,6 +440,14 @@ define([ $(this.wrapSelector('.vp-pm-table')).html(''); let packageList = Object.keys(this.packageLib); + + // set auto search + let searchInput = new SuggestInput(); + searchInput.addClass('vp-pm-search vp-input'); + searchInput.setPlaceholder("Search"); + searchInput.setSuggestList(function () { return packageList; }); + $(this.wrapSelector('.vp-pm-search')).replaceWith(searchInput.toTagString()); + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); vpKernel.getPackageList(packageList).then(function(resultObj) { let { result } = resultObj; @@ -408,10 +455,10 @@ define([ // load code list var innerFuncCode = new com_String(); - Object.keys(packageInfo).forEach(key => { + Object.keys(packageInfo).forEach((key, idx) => { let info = packageInfo[key]; // installed, version, path if (info) { - var item = that.renderPackageItem(key, info); + var item = that.renderPackageItem(key, info, idx); innerFuncCode.append(item); } }); diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 90d79712..60b37260 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -99,7 +99,7 @@ define([ this.category = category; this.config = { - sizeLevel: 0, // 0: 400x400 / 1: 500x500 / 2: 600x500 / 3: 750x500 + sizeLevel: 0, // 0: 400x400 / 1: 500x500 / 2: 600x500 / 3: 750x500 / 4: executeMode: 'code', // cell execute mode // show header bar buttons installButton: false, // install button (#popupInstall) // FIXME: after creating packagemanager, deprecate it @@ -112,14 +112,21 @@ define([ // show footer runButton: true, footer: true, + // run type : run / run-save / add + runType: 'run', + // position and size position: { right: 10, top: 120 }, size: { width: 400, height: 550 }, + // additional modes and infos saveOnly: false, // apply mode + resizable: true, + autoScroll: true, checkModules: [], // module aliases or function names docs: 'https://visual-python.gitbook.io/docs/getting-started/welcome-to-visual-python', helpInfo: { - content: '', // method to show using help() ex) pd.DataFrame - useHelp: true // custom text to show on help viewer + content: '', // method to show using help() ex) pd.DataFrame + useHelp: true, // custom text to show on help viewer + importCode: '' // import code for help }, ...restConfig }; @@ -304,6 +311,22 @@ define([ _bindEvent() { var that = this; + // Popup click / focus event + $(this.wrapSelector()).on('click focus', function(evt) { + // Close on blur + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-codeview-box') + && $(that.wrapSelector('.vp-popup-codeview-box')).find(evt.target).length == 0) { + that.closeView('code'); + } + } + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-dataview-box') + && $(that.wrapSelector('.vp-popup-dataview-box')).find(evt.target).length == 0) { + that.closeView('data'); + } + } + }); // Close popup event $(this.wrapSelector('.vp-popup-close')).on('click', function(evt) { if (that.getTaskType() === 'task') { @@ -322,10 +345,7 @@ define([ // Toggle operation (minimize) $(this.wrapSelector('.vp-popup-toggle')).on('click', function(evt) { evt.stopPropagation(); - $(that.eventTarget).trigger({ - type: 'close_option_page', - component: that - }); + that.hide(); }); // Maximize operation $(this.wrapSelector('.vp-popup-maximize')).on('click', function(evt) { @@ -456,33 +476,75 @@ define([ } break; case 'run': - let result = that.run(); + var result = that.run(); if (result) { - that.save(); + // that.save(); + that.close(); } break; case 'show-detail': $(that.wrapSelector('.vp-popup-run-detailbox')).show(); + // set run button + vpConfig.getData('runType', 'vpcfg').then(function(data) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); + that._setDefaultRunType(data); + }); evt.stopPropagation(); break; case 'save': that.save(); break; + case 'run-save': + var result = that.run(); + if (result) { + that.save(); + } + break; + case 'add': + var result = that.run(false); + if (result) { + // that.save(); + that.close(); + } + break; } }); + // Click detail radio + $(this.wrapSelector('.vp-popup-run-type')).on('click', function(evt) { + let runType = $(that.wrapSelector('.vp-popup-run-type[name=runType]:checked')).val(); + // save as vpConfig + vpConfig.setData({runType: runType}, 'vpcfg'); + that._setDefaultRunType(runType); + }); // Click detail buttons - $(this.wrapSelector('.vp-popup-detail-button')).on('click', function(evt) { + $(this.wrapSelector('.vp-popup-detail-action-button')).on('click', function(evt) { var btnType = $(this).data('type'); + vpConfig.setData({runType: btnType}, 'vpcfg'); + that._setDefaultRunType(btnType); // run, run-save, add switch(btnType) { - case 'apply': - that.save(); + // case 'apply': + // that.save(); + // break; + case 'run': + var result = that.run(); + if (result) { + // that.save(); + that.close(); + } break; - case 'add': - let result = that.run(false); + case 'run-save': + var result = that.run(); if (result) { that.save(); } break; + case 'add': + var result = that.run(false); + if (result) { + // that.save(); + that.close(); + } + break; } }); // Close event for inner popup @@ -502,6 +564,14 @@ define([ } }); + // blur on code, dataview + $(this.wrapSelector('.vp-popup-codeview-box')).on('hide', function() { + that.closeView('code'); + }); + $(this.wrapSelector('.vp-popup-dataview-box')).on('hide', function() { + that.closeView('data'); + }); + // focus on data selector input $(this.wrapSelector('.vp-data-selector')).on('focus', function(evt) { @@ -599,7 +669,7 @@ define([ let { installButton, importButton, packageButton, codeview, dataview, helpview, runButton, footer, - sizeLevel, position, docs + sizeLevel, position, autoScroll, docs } = this.config; // apply link to docs @@ -650,7 +720,15 @@ define([ $(this.wrapSelector('.vp-popup-body')).css({ 'height': 'calc(100% - 30px)' }) + } else { + // set run button + let that = this; + vpConfig.getData('runType', 'vpcfg').then(function(data) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); + that._setDefaultRunType(data); + }); } + // popup-frame size switch (sizeLevel) { @@ -664,10 +742,17 @@ define([ this.config.size = { width: 760, height: 550 }; break; case 4: + this.config.size = { width: 840, height: 550 }; + break; + case 5: this.config.size = { width: 1064, height: 550 }; break; } + if (autoScroll) { + $(this.wrapSelector('.vp-popup-body')).addClass('vp-scrollbar'); + } + // set detailed size $(this.wrapSelector()).css({ width: this.config.size.width + 'px', @@ -683,7 +768,9 @@ define([ } this._bindDraggable(); - this._bindResizable(); + if (this.config.resizable) { + this._bindResizable(); + } } templateForInnerPopup() { @@ -785,9 +872,27 @@ define([ vpLog.display(VP_LOG_TYPE.DEVELOP, 'savedState', that.state); } - _saveSingleState(tag) { + _setDefaultRunType(runType) { + if (runType == undefined || runType == null || runType === '') { + runType = 'run'; // default = run + } + this.config.runType = runType; + $(this.wrapSelector(`.vp-popup-run-type[value="${runType}"]`)).prop('checked', true); + $(this.wrapSelector('.vp-popup-run-button')).attr('data-type', runType); + let runTitle = 'Run code'; // when runType is 'run' + switch (runType) { + case 'run-save': + runTitle = 'Save to block & Run code'; + break; + case 'add': + runTitle = 'Add code to cell'; + break; + } + $(this.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + } + + _getTagValue(tag) { let id = tag.id; - let customKey = $(tag).data('key'); let tagName = $(tag).prop('tagName'); // returns with UpperCase let newValue = ''; switch(tagName) { @@ -818,7 +923,14 @@ define([ } break; } + return newValue; + } + + _saveSingleState(tag) { + let id = tag.id; + let customKey = $(tag).data('key'); + let newValue = this._getTagValue(tag); // if custom key is available, use it if (customKey && customKey != '') { // allow custom key until level 2 @@ -876,9 +988,23 @@ define([ } } + if (requiredFilled) { + requiredFilled = this.checkBeforeRun(); + } + return requiredFilled; } + /** + * Check options and do some operation before run + * @returns {boolean} check options and returns true or false (false will stop code running) + */ + checkBeforeRun() { + /** Implementation needed */ + + return true; + } + checkAndRunModules(execute=true, background=false) { let sigText = this.getSigText(); @@ -947,30 +1073,52 @@ define([ * - focus popup * - bind codemirror */ - open() { + open(targetFrame=undefined) { vpLog.display(VP_LOG_TYPE.DEVELOP, 'open popup', this); this.loadState(); this._beforeOpen(); - this.show(); - - // set popup position if its top-left side is outside of view - let pos = $(this.wrapSelector()).position(); - if (pos) { - if (pos.top < 0) { - $(this.wrapSelector()).css({ top: 0 }); - } - if (pos.left < 0) { - $(this.wrapSelector()).css({ left: 0 }); + if (targetFrame !== undefined) { + // hide popup frame + $(this.wrapSelector('.vp-popup-header')).hide(); + $(this.wrapSelector('.vp-popup-footer')).hide(); + // set width and height to 100% + $(this.wrapSelector()).css({ + width: '100%', + height: '100%', + 'min-height': 'unset', + display: 'block', + position: 'initial', + border: '0px' + }); + $(this.wrapSelector('.vp-popup-body')).css({ + padding: '0px' + }); + // show on targetFrame + $(this.wrapSelector()).appendTo($(targetFrame)); + } else { + this.show(); + + // set popup position if its top-left side is outside of view + let pos = $(this.wrapSelector()).position(); + if (pos) { + if (pos.top < 0) { + $(this.wrapSelector()).css({ top: 0 }); + } + if (pos.left < 0) { + $(this.wrapSelector()).css({ left: 0 }); + } } } this._bindCodemirror(); - $(this.eventTarget).trigger({ - type: 'focus_option_page', - component: this - }); + if (targetFrame == undefined) { + $(this.eventTarget).trigger({ + type: 'focus_option_page', + component: this + }); + } } setSaveOnlyMode() { @@ -987,7 +1135,15 @@ define([ close() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'close popup', this); this.saveState(); - this.hide(); + if (this.getTaskType() === 'task') { + $(this.eventTarget).trigger({ + type: 'remove_option_page', + component: this + }); + } else { + // if it's block, just hide it + this.hide(); + } } save() { @@ -1004,6 +1160,7 @@ define([ remove() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'remove popup', this); + this._unbindResizable(); this._unbindEvent(); this.closeHelpView(); $(this.wrapSelector()).remove(); @@ -1022,6 +1179,10 @@ define([ blur() { $(this.wrapSelector()).removeClass('vp-focused'); + // blur on its block + if (this.taskItem) { + this.taskItem.blurItem(); + } } show() { @@ -1093,20 +1254,36 @@ define([ setTimeout(function() { that.cmCodeview.refresh(); }, 1); - $(this.wrapSelector('.vp-popup-dataview-box')).hide(); + this.closeView('data'); $(this.wrapSelector('.vp-popup-codeview-box')).show(); } else if (viewType === 'data') { this.renderDataView(); - $(this.wrapSelector('.vp-popup-codeview-box')).hide(); + this.closeView('code'); $(this.wrapSelector('.vp-popup-dataview-box')).show(); } else if (viewType === 'help') { - $(this.wrapSelector('.vp-popup-codeview-box')).hide(); - $(this.wrapSelector('.vp-popup-dataview-box')).hide(); + this.closeView('code'); + this.closeView('data'); this.openHelpView(); } } closeView(viewType) { + if (viewType === 'code') { + // reset codeview + if (this.cmCodeview) { + this.cmCodeview.setValue(''); + this.cmCodeview.save(); + + var that = this; + setTimeout(function() { + that.cmCodeview.refresh(); + }, 1); + } + } else if (viewType === 'data') { + // reset dataview + $(this.wrapSelector('.vp-popup-dataview-box')).html(''); + } + // hide view $(this.wrapSelector('.vp-popup-'+viewType+'view-box')).hide(); } @@ -1117,7 +1294,7 @@ define([ openHelpView() { this.closeHelpView(); this.helpViewer = new HelpViewer(); - this.helpViewer.open(this.config.helpInfo.content, this.config.helpInfo.useHelp); + this.helpViewer.open(this.config.helpInfo.content, this.config.helpInfo.useHelp, this.config.helpInfo.importCode); } /** @@ -1133,11 +1310,13 @@ define([ * Set HelpViewer content * @param {string} content * @param {boolean} useHelp + * @param {string} importCode */ - setHelpContent(content, useHelp=true) { + setHelpContent(content, useHelp=true, importCode='') { this.config.helpInfo = { content: content, - useHelp: useHelp + useHelp: useHelp, + importCode: importCode }; } @@ -1149,7 +1328,7 @@ define([ $(this.wrapSelector('.vp-inner-popup-box')).show(); // focus on first input - $(this.wrapSelector('.vp-inner-popup-box input:not(:disabled):visible:first')).focus(); + $(this.wrapSelector('.vp-inner-popup-box input:not(:readonly):not(:disabled):visible:first')).focus(); // disable Jupyter key com_interface.disableOtherShortcut(); } diff --git a/visualpython/js/com/lib/__init__.py b/visualpython/js/com/lib/__init__.py deleted file mode 100644 index 1e8c9919..00000000 --- a/visualpython/js/com/lib/__init__.py +++ /dev/null @@ -1 +0,0 @@ -print('Visual Python') diff --git a/visualpython/js/com/template/__init__.py b/visualpython/js/com/template/__init__.py deleted file mode 100644 index 1e8c9919..00000000 --- a/visualpython/js/com/template/__init__.py +++ /dev/null @@ -1 +0,0 @@ -print('Visual Python') diff --git a/visualpython/js/loadVisualpython.js b/visualpython/js/loadVisualpython.js index bd8b1781..b06ffb62 100644 --- a/visualpython/js/loadVisualpython.js +++ b/visualpython/js/loadVisualpython.js @@ -287,8 +287,10 @@ define([ } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: if widget is ready or changed, ready for lab kernel connected, and restart vp vpLab.shell._currentChanged.connect(function(s1, value) { - var { newValue } = value; - vpLog.display(VP_LOG_TYPE.DEVELOP, 'jupyterlab shell currently changed', s1, value); + // var { newValue } = value; + // LAB 4.x: get currentWidget + var newValue = s1.currentWidget; + vpLog.display(VP_LOG_TYPE.DEVELOP, 'jupyterlab shell currently changed', s1, newValue); // kernel restart for notebook and console if (newValue && newValue.sessionContext) { vpConfig.hideProtector(); diff --git a/visualpython/js/m_apps/Bind.js b/visualpython/js/m_apps/Bind.js index 8b19c96a..8d5e42af 100644 --- a/visualpython/js/m_apps/Bind.js +++ b/visualpython/js/m_apps/Bind.js @@ -339,7 +339,7 @@ define([ renderDataPage(renderedText, isHtml = true) { var tag = new com_String(); - tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area + tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area if (isHtml) { tag.appendLine(renderedText); } else { @@ -355,7 +355,7 @@ define([ // if not, get output of all data in selected pandasObject vpKernel.execute(code).then(function(resultObj) { let { msg } = resultObj; - if (msg.content.data) { + if (msg.content && msg.content.data) { var htmlText = String(msg.content.data["text/html"]); var codeText = String(msg.content.data["text/plain"]); if (htmlText != 'undefined') { @@ -368,20 +368,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } diff --git a/visualpython/js/m_apps/File.js b/visualpython/js/m_apps/File.js index 19bbc796..7e6bd848 100644 --- a/visualpython/js/m_apps/File.js +++ b/visualpython/js/m_apps/File.js @@ -34,16 +34,17 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 1; + this.config.sizeLevel = 2; this.config.checkModules = ['pd']; this.fileExtensions = { - 'csv': 'csv', - 'excel': 'xlsx', - 'json': 'json', - 'pickle': '', - 'sas': '', // xport or sas7bdat - 'spss': '' + 'csv': ['csv', 'tsv', 'txt'], + 'excel': ['xlsx', 'xls'], + 'json': ['json'], + 'pickle': [], + 'sas': [], // xport or sas7bdat + 'spss': [], + 'parquet': ['parquet'] } this.package = { @@ -67,7 +68,8 @@ define([ } this.state = { - fileExtension: 'csv', + fileType: 'csv', + fileExtension: ['csv'], selectedFile: '', selectedPath: '', vp_fileioType: 'Read', @@ -89,7 +91,8 @@ define([ 'json': 'pd_readJson', 'pickle': 'pd_readPickle', 'sas': 'pd_readSas', - 'spss': 'pd_readSpss' + 'spss': 'pd_readSpss', + 'parquet':'pd_readParquet' }, selectedType: 'csv', package: null, @@ -103,7 +106,8 @@ define([ 'csv': 'pd_toCsv', 'excel': 'pd_toExcel', 'json': 'pd_toJson', - 'pickle': 'pd_toPickle' + 'pickle': 'pd_toPickle', + 'parquet':'pd_toParquet' }, selectedType: 'csv', package: null, @@ -184,6 +188,11 @@ define([ } } + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#fileReadAs')); + } + _bindEvent() { super._bindEvent(); /** Implement binding events */ @@ -193,38 +202,74 @@ define([ that.state['vp_fileioType'] = pageType; $(that.wrapSelector('.vp-fileio-box')).hide(); $(that.wrapSelector('#vp_file' + pageType)).show(); - + + if (pageType === 'Read' && that.fileState[pageType].selectedType === 'spss') { + // show install button + that.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); + } else { + that.hideInstallButton(); + $(that.wrapSelector('.vp-spss-note')).remove(); + } + + //set fileExtensions that.fileResultState = { ...that.fileState[pageType].fileResultState }; }); + + // fileReadAs change Event, Use PyArrow + $(document).on('change', this.wrapSelector('#fileReadAs'), function() { + let isChecked = $(this).prop('checked'); + var fileioType = that.state.vp_fileioType; + var prefix = '#vp_file' + fileioType + ' '; + var selectedType = that.fileState[fileioType]['selectedType']; + var fileioTypePrefix = fileioType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } + let fileId = that.fileState[fileioType].fileTypeId[selectedType]; + + if (isChecked) { // pyArrow + fileId = "pa_" + fileioTypePrefix + selectedType[0].toUpperCase() + selectedType.slice(1); + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pa_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); + } else { // pandas + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + if (that.state.fileType != 'parquet'){ // parquet has no options area + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').show(); + } + } + + let pdLib = pandasLibrary.PANDAS_FUNCTION; + let thisPkg = JSON.parse(JSON.stringify(pdLib[fileId])); + + that.fileState[fileioType].package = thisPkg; + }); + } _bindEventByType(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + + var fileioTypePrefix = pageType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } // select file type $(this.wrapSelector(prefix + '#fileType')).change(function() { - var value = $(this).val(); - that.fileState[pageType].selectedType = value; + var fileType = $(this).val(); + that.fileState[pageType].selectedType = fileType; // reload that.renderPage(pageType); that._bindEventByType(pageType); - - if (value === 'spss') { - // show install button - that.showInstallButton(); - // show install note below File type selection - $(` - NOTE: - pyreadstat package is required to read spss file. - `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); - } else { - that.hideInstallButton(); - } }); // open file navigation @@ -236,8 +281,8 @@ define([ } let extensionList = []; - if (that.state.fileExtension !== '') { - extensionList = [ that.state.fileExtension ]; + if (that.state.fileExtension && that.state.fileExtension.length > 0) { + extensionList = that.state.fileExtension; } let fileNavi = new FileNavigation({ @@ -326,7 +371,7 @@ define([ renderPage(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + // clear $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).html(''); $(this.wrapSelector(prefix + '#vp_optionBox table tbody')).html(''); @@ -343,7 +388,7 @@ define([ ...this.fileState[pageType].fileResultState }; - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { // hide additional option box $(this.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); } else { @@ -354,7 +399,7 @@ define([ if (selectedType == 'json') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path_or_buf'); } - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path'); } } @@ -364,11 +409,21 @@ define([ // pdGen.vp_showInterfaceOnPage(this.wrapSelector('#vp_file' + pageType), thisPkg); pdGen.vp_showInterfaceOnPage(this, thisPkg, this.state, parent=('#vp_file' + pageType)); + // pyarrow can r/w parquet, csv and only read json. + if ((pageType == 'Read' && selectedType == 'json') || selectedType == 'parquet'|| selectedType == 'csv') { + // add checkbox 'Use PyArrow', next to File Type + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' Use PyArrow')) + ); + } else { + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' ')) + ); + } + // prepend file type selector - $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( - $('').append($(`File Type`)) - .append($('')) - ); var fileTypeList = Object.keys(fileTypeObj); fileTypeList.forEach(type => { $(this.wrapSelector(prefix + '#fileType')).append( @@ -389,9 +444,9 @@ define([ .append($('')) ) } - + $(this.wrapSelector(prefix + '#fileType')).val(selectedType); - + // add file navigation button if (pageType == 'Write') { if (selectedType == 'json') { @@ -399,7 +454,7 @@ define([ com_util.formatString('

' , 'vp-file-browser-button') ); - } else if (selectedType == 'pickle') { + } else if (selectedType == 'pickle' || selectedType == 'parquet') { $(prefix + '#path').parent().html( com_util.formatString('

' , 'vp-file-browser-button') @@ -418,6 +473,18 @@ define([ , 'vp-file-browser-button') ); } + + if (pageType === 'Read' && selectedType === 'spss') { + // show install button + this.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(this.wrapSelector('#fileType')).closest('tr')); + } else { + this.hideInstallButton(); + } // encoding suggest input $(this.wrapSelector('#encoding')).replaceWith(function() { @@ -430,6 +497,18 @@ define([ suggestInput.setPlaceholder('encoding option'); return suggestInput.toTagString(); }); + + // separator suggest input + $(this.wrapSelector('#sep')).replaceWith(function() { + // separator list : + var sepList = [',', '|', '\\t', '\\n', ':', ';', '-', '_', '&', '/', '\\']; + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('sep'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return sepList; }); + suggestInput.setPlaceholder('Input separator'); + return suggestInput.toTagString(); + }); } render() { @@ -492,7 +571,6 @@ define([ var result = pdGen.vp_codeGenerator(this, thisPkg, this.state, userOption.toString(), parent='#vp_fileWrite'); sbCode.append(result); } - return sbCode.toString(); } diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index e8b98611..4e91083d 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -21,8 +21,9 @@ define([ 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/m_apps/Subset' -], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, Subset) { +], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, MultiSelector, Subset) { /** * Frame @@ -30,7 +31,7 @@ define([ class Frame extends PopupComponent { _init() { super._init(); - this.config.sizeLevel = 4; + this.config.sizeLevel = 5; this.config.checkModules = ['pd']; // state @@ -86,7 +87,8 @@ define([ { id: 'add_row', label: 'Add row', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.ADD_ROW }, { id: 'delete', label: 'Delete', selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP }, { id: 'rename', label: 'Rename', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RENAME }, - { id: 'asType', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'as_type', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'to_datetime', label: 'To Datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, { id: 'replace', label: 'Replace', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.REPLACE }, { id: 'discretize', label: 'Discretize', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, numeric_only: true, menuType: FRAME_EDIT_TYPE.DISCRETIZE } ] @@ -113,10 +115,10 @@ define([ id: 'encoding', label: 'Encoding', axis: FRAME_AXIS.COLUMN, - selection: FRAME_SELECT_TYPE.SINGLE, + selection: FRAME_SELECT_TYPE.MULTI, child: [ - { id: 'label_encoding', label: 'Label encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.SINGLE, menuType: FRAME_EDIT_TYPE.LABEL_ENCODING }, - { id: 'one_hot_encoding', label: 'Onehot encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.SINGLE, menuType: FRAME_EDIT_TYPE.ONE_HOT_ENCODING }, + { id: 'label_encoding', label: 'Label encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.LABEL_ENCODING }, + { id: 'one_hot_encoding', label: 'Onehot encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.ONE_HOT_ENCODING }, ] }, { @@ -617,6 +619,7 @@ define([ case FRAME_EDIT_TYPE.RENAME: case FRAME_EDIT_TYPE.REPLACE: case FRAME_EDIT_TYPE.AS_TYPE: + case FRAME_EDIT_TYPE.TO_DATETIME: case FRAME_EDIT_TYPE.DISCRETIZE: case FRAME_EDIT_TYPE.DATA_SHIFT: case FRAME_EDIT_TYPE.SORT_INDEX: @@ -710,10 +713,7 @@ define([ return; } } else if (type === FRAME_EDIT_TYPE.REPLACE) { - if (content.replacetype === 'condition' && content.value === '') { - $(this.wrapSelector('.vp-inner-popup-input3')).focus(); - return; - } + ; } else if (type === FRAME_EDIT_TYPE.FILL_NA) { if (content.method === 'value' && content.value === '') { $(this.wrapSelector('.vp-inner-popup-value')).focus(); @@ -724,6 +724,10 @@ define([ $(this.wrapSelector('.vp-inner-popup-fillvalue')).focus(); return; } + } else if (type === FRAME_EDIT_TYPE.TO_DATETIME) { + if (content.newcol === '') { + $(this.wrapSelector('.vp-inner-popup-todt-new-col')).focus(); + } } // run check modules for outliers and load codes if (type === FRAME_EDIT_TYPE.FILL_OUT) { @@ -790,6 +794,7 @@ define([ if (tab === 'apply') { let colName = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colName); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); } }); @@ -830,6 +835,7 @@ define([ if (tab === 'apply') { let colName = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colName); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); } }); } @@ -945,10 +951,10 @@ define([ $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', false); } $(operTag).replaceWith(function () { - return that.templateForConditionOperator(colDtype, 'vp-inner-popup-apply-oper-list'); + return that.templateForConditionOperator(colDtype, 'vp-inner-popup-apply-oper-list', $(this).val()); }); $(condTag).replaceWith(function () { - return that.templateForConditionCondInput(category, colDtype, 'vp-inner-popup-apply-condition'); + return that.templateForConditionCondInput(category, colDtype, 'vp-inner-popup-apply-condition', $(this).val()); }); } catch { $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', false); @@ -962,6 +968,23 @@ define([ }); }); + // change operator selection + $(document).off('change', this.wrapSelector('.vp-inner-popup-apply-oper-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-apply-oper-list'), function () { + var oper = $(this).val(); + var condTag = $(this).closest('td').find('.vp-inner-popup-apply-condition'); + var useTextTag = $(this).closest('td').find('.vp-inner-popup-apply-cond-usetext'); + + // if operator is isnull(), notnull(), disable condition input + if (oper == 'isnull()' || oper == 'notnull()') { + $(condTag).prop('disabled', true); + $(useTextTag).prop('disabled', true); + } else { + $(condTag).prop('disabled', false); + $(useTextTag).prop('disabled', false); + } + }); + $(this.wrapSelector('.vp-inner-popup-toggle-else')).on('click', function() { // toggle else on/off let elseOn = $(this).attr('data-else'); // on / off @@ -983,6 +1006,7 @@ define([ $(this.wrapSelector('.vp-inner-popup-add-case')).on('click', function() { // add case $(this).parent().find('.vp-inner-popup-apply-case-box').append($(that.templateForApplyCase())); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); }); $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-add-cond')); @@ -991,6 +1015,7 @@ define([ $(this).parent().find('.vp-inner-popup-apply-cond-box').append($(that.templateForApplyCondition())); // show operator except last operator $(this).parent().find('.vp-inner-popup-apply-oper-connect:not(:last)').show(); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); }); $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-del-cond')); @@ -1104,6 +1129,70 @@ define([ $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); } }); + } else if (menuType === FRAME_EDIT_TYPE.TO_DATETIME) { + // bind event for selecting format + $(this.wrapSelector('.vp-inner-popup-todt-format')).on('change', function() { + let format = $(this).val(); + if (format === 'auto') { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(''); + } + + if (format === 'typing') { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', true); + } + }); + + // Add column set event + $(this.wrapSelector('.vp-inner-popup-todt-addcol')).on('click', function() { + let dateTypeList = [ // df[col].dt[{dateType}] + { label: 'Year', value: 'year' }, + { label: 'Month', value: 'month' }, + { label: 'Day', value: 'day' }, + { label: 'Date', value: 'date' }, + { label: 'DayOfWeek', value: 'dayofweek' }, + { label: 'DayOfYear', value: 'dayofyear' }, + { label: 'DaysInMonth', value: 'daysinmonth' }, + { label: 'Quarter', value: 'quarter' }, + { label: 'Time', value: 'time' }, + { label: 'Hour', value: 'hour' }, + { label: 'Minute', value: 'minute' }, + { label: 'Second', value: 'second' }, + { label: 'Nanosecond', value: 'nanosecond' }, + ]; + let dateTypeOptionTag = new com_String(); + dateTypeList.forEach(opt => { + dateTypeOptionTag.appendFormat('', opt.value, opt.label); + }); + + let addColItemTag = $(``); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-content')).append(addColItemTag); + $(addColItemTag)[0].scrollIntoView(); + + // bind event for selecting date type option + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).off('change'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).on('change', function() { + let dateTypeVal = $(this).val(); + $(this).parent().find('.vp-inner-popup-todt-addcol-colname').val(dateTypeVal); + }); + + // bind event for deleting + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).off('click'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).on('click', function() { + // delete item + $(this).closest('.vp-inner-popup-todt-addcol-item').remove(); + }); + }); } } @@ -1430,6 +1519,7 @@ define([ content.appendLine('Add type'); content.appendFormatLine('', 'vp-inner-popup-stats-method'); + let methodList = [ + 'sum', 'mean', 'median', 'max', 'min', 'std', 'var' + ]; + methodList.forEach(method => { + content.appendFormatLine('', + method, method, method); + }); + content.appendLine(''); + + content.appendLine('

'); + content.appendLine('

'); + + // tab 3. replace content.appendFormatLine(''); - // tab 3. condition + // tab 4. condition // replace page - 2. condition content.appendFormatLine(''); - // tab 4. apply + // tab 5. apply content.appendFormatLine('

', 'vp-inner-popup-tab', 'apply'); content.appendLine('

'); content.appendLine('

'); @@ -1520,12 +1629,12 @@ define([ // content.appendFormatLine('' // , 'vp_popupAddApply', 'vp-input vp-inner-popup-apply-lambda', 'Type code manually'); // render condition - content.appendLine('

'); + content.appendLine('

'); content.appendFormatLine('

', 'vp-inner-popup-apply-case-box'); content.appendLine(this.templateForApplyCase()); content.appendLine('

'); - content.appendFormatLine('', 'vp-inner-popup-add-case'); content.appendLine('

'); + content.appendFormatLine('', 'vp-inner-popup-add-case'); content.appendLine('

'); // else on/off content.appendFormatLine('', 'vp-inner-popup-toggle-else'); @@ -2016,7 +2125,7 @@ define([ return content.toString(); } - templateForConditionOperator(dtype='object', className='vp-inner-popup-oper-list') { + templateForConditionOperator(dtype='object', className='vp-inner-popup-oper-list', prevValue='') { var content = new com_String(); content.appendFormatLine(''); return content.toString(); } - templateForConditionCondInput(category, dtype='object', className='vp-inner-popup-condition') { + templateForConditionCondInput(category, dtype='object', className='vp-inner-popup-condition', prevValue='') { var vpCondSuggest = new SuggestInput(); vpCondSuggest.addClass('vp-input m ' + className); - + if (prevValue !== '') { + vpCondSuggest.setValue(prevValue); + } if (category && category.length > 0) { vpCondSuggest.setPlaceholder((dtype=='object'?'Categorical':dtype) + " dtype"); vpCondSuggest.setSuggestList(function () { return category; }); @@ -2085,6 +2200,74 @@ define([ return content.toString(); } + renderToDatetime() { + var content = new com_String(); + let formatList = [ + { label: 'Auto', value: 'auto' }, + { label: '%Y/%m/%d', value: '%Y/%m/%d' }, + { label: '%Y-%m-%d', value: '%Y-%m-%d' }, + { label: '%y/%m/%d', value: '%y/%m/%d' }, + { label: '%y-%m-%d', value: '%y-%m-%d' }, + { label: '%d/%m/%Y', value: '%d/%m/%Y' }, + { label: '%d-%m-%Y', value: '%d-%m-%Y' }, + { label: '%d/%m/%y', value: '%d/%m/%y' }, + { label: '%d-%m-%y', value: '%d-%m-%y' }, + { label: 'Typing', value: 'typing' } + ]; + let formatOptionTag = new com_String(); + formatList.forEach(opt => { + formatOptionTag.appendFormat('', opt.value, opt.label); + }); + + content.appendFormat(` + `, this.state.selected[0].label, formatOptionTag.toString(), this.state.selected[0].label); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + renderFillNAPage() { var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-fillna-page'); @@ -2266,6 +2449,11 @@ define([ }); }); + // bind multiselector + this.statsSelector = new MultiSelector(this.wrapSelector('.vp-inner-popup-stats-col-list'), + { mode: 'columns', parent: this.state.tempObj, showDescription: false } + ); + // bind codemirror for apply textarea this.applyCm = this.initCodemirror({ key: 'vp-inner-popup-apply-lambda', @@ -2421,6 +2609,11 @@ define([ title = 'Convert type'; content = this.renderAsType(); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + title = 'Convert to Datetime'; + size = { width: 500, height: 450 }; + content = this.renderToDatetime(); + break; case FRAME_EDIT_TYPE.FILL_NA: title = 'Fill NA'; content = this.renderFillNAPage(); @@ -2550,6 +2743,10 @@ define([ }); content['values'] = values; content['opers'] = opers; + } else if (tab == 'statistics') { + var colList = this.statsSelector.getDataList(); + content['columns'] = colList.map(x => x.code); + content['method'] = $(this.wrapSelector('.vp-inner-popup-stats-method')).val(); } else if (tab == 'replace') { content['target'] = $(this.wrapSelector('.vp-inner-popup-value-col-list option:selected')).data('code'); var useregex = $(this.wrapSelector('.vp-inner-popup-use-regex')).prop('checked'); @@ -2587,7 +2784,7 @@ define([ var condText = $(condTextTag[i]).prop('checked'); var operConn = $(operConnTag[i]).val(); var condObj = {}; - if (col !== '' && oper !== '' && cond !== '') { + if (col !== '' && oper !== '' && (oper == 'isnull()' || oper === 'notnull()' || cond !== '')) { condObj = { oper: oper, cond: com_util.convertToStr(cond, condText) @@ -2628,7 +2825,7 @@ define([ var condText = $(condTextTag[i]).prop('checked'); var operConn = $(operConnTag[i]).val(); var condObj = {}; - if (col !== '' && oper !== '' && cond !== '') { + if (col !== '' && oper !== '' && (oper == 'isnull()' || oper === 'notnull()' || cond !== '')) { condObj = { colName: col, oper: oper, @@ -2681,7 +2878,7 @@ define([ var condText = $(condTextTag[i]).prop('checked'); var operConn = $(operConnTag[i]).val(); var condObj = {}; - if (col !== '' && oper !== '' && cond !== '') { + if (col !== '' && oper !== '' && (oper == 'isnull()' || oper === 'notnull()' || cond !== '')) { condObj = { colName: col, oper: oper, @@ -2714,7 +2911,7 @@ define([ var condText = $(condTextTag[i]).prop('checked'); var operConn = $(operConnTag[i]).val(); var condObj = {}; - if (col !== '' && oper !== '' && cond !== '') { + if (col !== '' && oper !== '' && (oper == 'isnull()' || oper === 'notnull()' || cond !== '')) { condObj = { oper: oper, cond: com_util.convertToStr(cond, condText) @@ -2786,6 +2983,22 @@ define([ } }); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + content['format'] = $(this.wrapSelector('.vp-inner-popup-todt-format')).val(); + content['format_typing'] = $(this.wrapSelector('.vp-inner-popup-todt-format-typing')).val(); + content['dayfirst'] = $(this.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(); + content['newcol'] = $(this.wrapSelector('.vp-inner-popup-todt-new-col')).val(); + var colList = []; + var addcolItemTags = $(this.wrapSelector('.vp-inner-popup-todt-addcol-item')); + addcolItemTags && addcolItemTags.each((idx, tag) => { + let colName = $(tag).find('.vp-inner-popup-todt-addcol-colname').val(); + let dateType = $(tag).find('.vp-inner-popup-todt-addcol-type').val(); + if (colName !== '' && dateType !== '') { + colList.push({ colName: colName, dateType: dateType }); + } + }); + content['collist'] = colList; + break; case FRAME_EDIT_TYPE.DISCRETIZE: content['input'] = $(this.wrapSelector('.vp-inner-popup-input')).val(); content['inputastext'] = $(this.wrapSelector('.vp-inner-popup-inputastext')).prop('checked'); @@ -2922,10 +3135,10 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(that.wrapSelector('.' + VP_FE_INFO_CONTENT)).replaceWith(function() { return that.renderInfoPage(errorContent); }); @@ -2933,10 +3146,10 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(that.wrapSelector('.' + VP_FE_INFO_CONTENT)).replaceWith(function() { return that.renderInfoPage(errorContent); }); @@ -3041,13 +3254,18 @@ define([ break; case FRAME_EDIT_TYPE.LABEL_ENCODING: if (axis == FRAME_AXIS.COLUMN) { - let encodedColName = this.state.selected.map(col=> { + let encodedColNameList = this.state.selected.map(col=> { if (col.code !== col.label) { - return com_util.formatString("'{0}'", col.label + '_label'); + return { 'origin': com_util.formatString("'{0}'", col.label), 'encoded': com_util.formatString("'{0}'", col.label + '_label') }; + } + return { 'origin': col.label, 'encoded': col.label + '_label' }; + }); + encodedColNameList.forEach((encodedColObj, idx) => { + if (idx > 0) { + code.appendLine(); } - return col.label + '_label' - }).join(','); - code.appendFormat("{0}[{1}] = pd.Categorical({2}[{3}]).codes", tempObj, encodedColName, tempObj, selectedName); + code.appendFormat("{0}[{1}] = pd.Categorical({2}[{3}]).codes", tempObj, encodedColObj['encoded'], tempObj, encodedColObj['origin']); + }); } break; case FRAME_EDIT_TYPE.ONE_HOT_ENCODING: @@ -3101,10 +3319,18 @@ define([ }); let valueStr = values.join(' '); if (valueStr === "" || valueStr === "''") { - code.appendFormat("{0}[{1}] = np.NaN", tempObj, content.name); + code.appendFormat("{0}[{1}] = np.nan", tempObj, content.name); } else { code.appendFormat("{0}[{1}] = {2}", tempObj, content.name, valueStr); } + } else if (tab == 'statistics') { + code.appendFormat("{0}[{1}] = ", tempObj, content.name); + code.append(tempObj); + if (content['columns'].length > 0) { + code.appendFormat("[[{0}]]", content['columns'].join(',')); + } + code.appendFormat(".apply(lambda x: x.{0}(numeric_only=True), axis=1)", content.method); + // code.appendFormat(".{0}(axis=1)", content.method); // FIXME: for pandas version upper 2.0.0 } else if (tab == 'replace') { var replaceStr = new com_String(); var targetName = content['target']; @@ -3134,7 +3360,21 @@ define([ let condCode = ''; obj.condList.forEach((condObj, idx) => { let { oper, cond, connector } = condObj; - condCode += `(x ${oper} ${cond})`; + if (oper === 'isnull()') { + condCode += `(pd.isnull(x))`; + } else if (oper === 'notnull()') { + condCode += `(pd.notnull(x))`; + } else if (oper === 'contains') { + condCode += `(${cond} in x)`; + } else if (oper === 'not contains') { + condCode += `(${cond} not in x)`; + } else if (oper === 'starts with') { + condCode += `(x.startswith(${cond}))`; + } else if (oper === 'ends with') { + condCode += `(x.endswith(${cond}))`; + } else { + condCode += `(x ${oper} ${cond})`; + } if (connector !== undefined) { condCode += ` ${connector} `; } @@ -3178,6 +3418,9 @@ define([ } }); var value = com_util.convertToStr(content.value, content.valueastext); + if (value === '') { + value = 'np.nan'; + } code.appendFormat(", {0}] = {1}", content.name, value); } break; @@ -3223,9 +3466,10 @@ define([ code.append(')'); } else if (tab === 'condition') { code.appendFormat("{0}.loc[", tempObj); + var condCode = new com_String(); content['list'].forEach((obj, idx) => { let { colName, oper, cond, condAsText, connector } = obj; - code.append('('); + condCode.append('('); let colValue = tempObj; if (colName && colName != '') { @@ -3237,25 +3481,31 @@ define([ } let condValue = com_util.convertToStr(cond, condAsText); if (oper == 'contains') { - code.appendFormat('{0}.str.contains({1})', colValue, condValue); + condCode.appendFormat('{0}.str.contains({1})', colValue, condValue); } else if (oper == 'not contains') { - code.appendFormat('~{0}.str.contains({1})', colValue, condValue); + condCode.appendFormat('~{0}.str.contains({1})', colValue, condValue); } else if (oper == 'starts with') { - code.appendFormat('{0}.str.startswith({1})', colValue, condValue); + condCode.appendFormat('{0}.str.startswith({1})', colValue, condValue); } else if (oper == 'ends with') { - code.appendFormat('{0}.str.endswith({1})', colValue, condValue); + condCode.appendFormat('{0}.str.endswith({1})', colValue, condValue); } else if (oper == 'isnull()' || oper == 'notnull()') { - code.appendFormat('{0}.{1}', colValue, oper); + condCode.appendFormat('{0}.{1}', colValue, oper); } else { - code.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); + condCode.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); } - code.append(')'); + condCode.append(')'); if (idx < (content['list'].length - 1)) { - code.append(connector); + condCode.append(connector); } }); + if (condCode.toString() === '') { + condCode.append(':'); + } var value = com_util.convertToStr(content.value, content.valueastext); - code.appendFormat(", {0}] = {1}", content.name, value); + if (value === '') { + value = 'np.nan'; + } + code.appendFormat("{0}, {1}] = {2}", condCode.toString(), content.name, value); } else if (tab == 'apply') { // code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); let lambdaCode = 'lambda x: '; @@ -3265,7 +3515,21 @@ define([ let condCode = ''; obj.condList.forEach((condObj, idx) => { let { oper, cond, connector } = condObj; - condCode += `(x ${oper} ${cond})`; + if (oper === 'isnull()') { + condCode += `pd.isnull(x)`; + } else if (oper === 'notnull()') { + condCode += `pd.notnull(x)`; + } else if (oper === 'contains') { + condCode += `(${cond} in x)`; + } else if (oper === 'not contains') { + condCode += `(${cond} not in x)`; + } else if (oper === 'starts with') { + condCode += `(x.startswith(${cond}))`; + } else if (oper === 'ends with') { + condCode += `(x.endswith(${cond}))`; + } else { + condCode += `(x ${oper} ${cond})`; + } if (connector !== undefined) { condCode += ` ${connector} `; } @@ -3288,6 +3552,31 @@ define([ }); code.appendFormat("{0} = {1}.astype({{2}})", tempObj, tempObj, astypeStr.toString()); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + code.appendFormat("{0}['{1}'] = pd.to_datetime({2}[{3}]", tempObj, content['newcol'], tempObj, selectedName); + let optionList = []; + if (content['format'] === 'auto') { + if (content['dayfirst'] !== '') { + optionList.push(`dayfirst=${content['dayfirst']}`); + } + } else if (content['format'] === 'typing') { + if (content['format_typing'] !== '') { + optionList.push(`format='${content['format_typing']}'`); + } + } else { + optionList.push(`format='${content['format']}'`); + } + if (optionList.length > 0) { + code.appendFormat(', {0}', optionList.join(', ')); + } + code.append(')'); + if (content['collist'].length > 0) { + content['collist'].forEach(obj => { + code.appendLine(); + code.appendFormat("{0}['{1}'] = {2}['{3}'].dt.{4}", tempObj, obj.colName, tempObj, content['newcol'], obj.dateType); + }); + } + break; case FRAME_EDIT_TYPE.DISCRETIZE: let newColumn = com_util.convertToStr(content['input'], content['inputastext']); let method = content['type']; @@ -3331,7 +3620,12 @@ define([ code.appendFormat(", limit={0}", content['limit']); } } else { - code.appendFormat("{0}.{1}()", subsetObjStr, content['method']); + if (content['method'] === 'mode') { + // get mode()'s first element (mode returns Series) + code.appendFormat("{0}.{1}()[0]", subsetObjStr, content['method']); + } else { + code.appendFormat("{0}.{1}()", subsetObjStr, content['method']); + } } code.append(')'); break; @@ -3360,8 +3654,9 @@ define([ code.appendLine(codeStr); code.appendFormat("{0}.iloc[{1}:{2}].to_json(orient='{3}')", tempObj, prevLines, lines, 'split'); + this.loading = true; - vpKernel.execute(code.toString()).then(function(resultObj) { + vpKernel.execute(com_util.ignoreWarning(code.toString())).then(function(resultObj) { let { result } = resultObj; try { if (!result || result.length <= 0) { @@ -3381,7 +3676,7 @@ define([ var indexList = data.index; var dataList = data.data; - columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value } }); + columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value, isNumeric: col.is_numeric } }); indexList = indexList.map(idx => { return { label: idx, code: idx } }); if (!more) { @@ -3397,6 +3692,12 @@ define([ while (colIdx < columnList.length) { let col = columnList[colIdx]; let colCode = col.code.slice(0, colLevIdx + 1).join(','); + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } let nextCol = columnList[colIdx + 1]; if (nextCol && nextCol.code.slice(0, colLevIdx + 1).join(',') === colCode) { colSpan++; @@ -3411,8 +3712,8 @@ define([ } else { colClass = VP_FE_TABLE_COLUMN_GROUP; } - table.appendFormatLine('{8}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, col.label[colLevIdx]); + table.appendFormatLine('{8}{9}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, colIcon, col.label[colLevIdx]); colSpan = 1; } colIdx++; @@ -3431,12 +3732,18 @@ define([ table.appendLine(''); columnList && columnList.forEach(col => { var colCode = col.code; + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } var colClass = ''; if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code).includes(colCode)) { colClass = 'selected'; } - table.appendFormatLine('{6}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, col.label); + table.appendFormatLine('{6}{7}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, colIcon, col.label); }); // // add column table.appendFormatLine('

', VP_FE_ADD_COLUMN, 'vp-icon-plus'); @@ -3555,11 +3862,21 @@ define([ that.loading = false; } }).catch(function(resultObj) { - let { result, type, msg } = resultObj; - vpLog.display(VP_LOG_TYPE.ERROR, result.ename + ': ' + result.evalue, msg, code.toString()); - if (that.isHidden() == false) { - // show alert modal only if this popup is visible - com_util.renderAlertModal(result.ename + ': ' + result.evalue); + let { result, msg, ename, evalue, status } = resultObj; + if (result) { + // NOTEBOOK: notebook error FIXME: migrate it on com_Kernel.execute + vpLog.display(VP_LOG_TYPE.ERROR, result?.ename + ': ' + result?.evalue, msg, code.toString()); + if (that.isHidden() == false) { + // show alert modal only if this popup is visible + com_util.renderAlertModal(result?.ename + ': ' + result?.evalue, { content: code.toString(), type: 'code' }); + } + } else { + // LAB: lab error FIXME: migrate it on com_Kernel.execute + vpLog.display(VP_LOG_TYPE.ERROR, ename + ': ' + evalue, status, code.toString()); + if (that.isHidden() == false) { + // show alert modal only if this popup is visible + com_util.renderAlertModal(ename + ': ' + evalue, { content: code.toString(), type: 'code' }); + } } that.loading = false; }); @@ -3647,6 +3964,7 @@ define([ DROP: 3, RENAME: 2, AS_TYPE: 10, + TO_DATETIME: 19, REPLACE: 9, DISCRETIZE: 15, diff --git a/visualpython/js/m_apps/Groupby.js b/visualpython/js/m_apps/Groupby.js index 41ef61f7..589882f0 100644 --- a/visualpython/js/m_apps/Groupby.js +++ b/visualpython/js/m_apps/Groupby.js @@ -177,7 +177,7 @@ define([ var colList = event.dataList; that.state.display = colList; - if (colList && colList.length == 1) { + if ((colList && colList.length == 1) || that.state.method === 'size') { $(that.wrapSelector('#vp_gbToFrame')).parent().show(); } else { $(that.wrapSelector('#vp_gbToFrame')).parent().hide(); @@ -196,6 +196,12 @@ define([ var method = $(this).val(); that.state.method = method; $(that.wrapSelector('#vp_gbMethod')).val(method); + + if (method === 'size' || (that.state.display && that.state.display.length == 1)) { + $(that.wrapSelector('#vp_gbToFrame')).parent().show(); + } else { + $(that.wrapSelector('#vp_gbToFrame')).parent().hide(); + } }); // advanced checkbox event @@ -543,20 +549,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } @@ -643,7 +649,7 @@ define([ page.appendFormatLine('

', 'vp-gb-method-user'); page.appendFormatLine('{0}', 'User option'); page.appendFormatLine('', 'Type user method'); - page.appendFormatLine('{0}' + page.appendFormatLine('{0}' , 'Text'); page.appendLine(''); page.appendLine('

'); @@ -744,7 +750,7 @@ define([ // Display columns //==================================================================== var colStr = ''; - if (display) { + if (display && display.length > 0) { if (toFrame || display.length > 1) { // over 2 columns colStr = '[[' + display.join(',') + ']]'; @@ -891,6 +897,10 @@ define([ methodStr.appendFormat('{0}(numeric_only=True)', method); } else { methodStr.appendFormat('{0}()', method); + if (method === 'size' && toFrame === true) { + // if to_Frame on size() method + methodStr.append(".to_frame(name='size')"); + } } } } diff --git a/visualpython/js/m_apps/Import.js b/visualpython/js/m_apps/Import.js index 38b6aec6..bc0321d5 100644 --- a/visualpython/js/m_apps/Import.js +++ b/visualpython/js/m_apps/Import.js @@ -22,26 +22,27 @@ define([ const importTemplates = { 'data-analysis': [ - { i0: 'numpy', i1: 'np', type: 'module'}, - { i0: 'pandas', i1: 'pd', type: 'module'}, + { i0: 'numpy', i1: 'np', type: 'module', checked: 'checked'}, + { i0: 'pandas', i1: 'pd', type: 'module', checked: 'checked'}, { i0: 'matplotlib.pyplot', i1: 'plt', type: 'module' , include: [ '%matplotlib inline' - ] + ], checked: 'checked' }, - { i0: 'seaborn', i1: 'sns', type: 'module'}, + { i0: 'seaborn', i1: 'sns', type: 'module', checked: 'checked'}, { i0: 'plotly.express', i1: 'px', type: 'module' , include: [ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' - ], checked: false - } + ], checked: '' + }, + { i0: 'pyarrow', i1: 'pa', type: 'module', checked: ''}, ], 'machine-learning': [ - { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function' }, - { i0: 'sklearn', i1: 'metrics', type: 'function' } + { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function', checked: 'checked' }, + { i0: 'sklearn', i1: 'metrics', type: 'function', checked: 'checked' } ] } @@ -174,9 +175,9 @@ define([ let that = this; libraries && libraries.forEach((lib, idx) => { if (lib.type == 'function') { - page.appendLine(that.templateForFunction(idx, lib.i0, lib.i1, lib.checked)); + page.appendLine(that.templateForFunction(idx, lib.i0, lib.i1, (lib.checked === 'checked' || lib.checked === true))); } else { - page.appendLine(that.templateForModule(idx, lib.i0, lib.i1, lib.checked)); + page.appendLine(that.templateForModule(idx, lib.i0, lib.i1, (lib.checked === 'checked' || lib.checked === true))); } }); page.appendLine(''); @@ -245,7 +246,7 @@ define([ if (pacI0 == "") { continue; } - if (pacChecked) { + if (pacChecked === true) { if (sbCode.toString().trim().length > 0) { sbCode.appendLine(); } @@ -269,7 +270,7 @@ define([ } } - importMeta.push({ i0: pacI0, i1: pacI1, type: pacType, checked: pacChecked }); + importMeta.push({ i0: pacI0, i1: pacI1, type: pacType, checked: (pacChecked?'checked':'') }); } this.state.importMeta = importMeta; diff --git a/visualpython/js/m_apps/Information.js b/visualpython/js/m_apps/Information.js index 025caf32..319c8877 100644 --- a/visualpython/js/m_apps/Information.js +++ b/visualpython/js/m_apps/Information.js @@ -17,10 +17,11 @@ define([ __VP_CSS_LOADER__('vp_base/css/m_apps/information'), // INTEGRATION: unified version of css loader 'vp_base/js/com/com_String', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/DataSelector', 'vp_base/js/com/component/LoadingSpinner' -], function(varHtml, varCss, com_String, com_util, PopupComponent, DataSelector, LoadingSpinner) { +], function(varHtml, varCss, com_String, com_util, com_interface, PopupComponent, DataSelector, LoadingSpinner) { /** * Information @@ -30,7 +31,7 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 4; + this.config.sizeLevel = 5; this.config.dataview = false; this.config.runButton = false; this.config.checkModules = ['pd', 'plt']; @@ -103,6 +104,7 @@ define([ \n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'category'): _value_counts.index})], axis=1)\ \n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'count'): _value_counts.values})], axis=1)\ \n_dfr.replace(np.nan,'')", + code2: "${data}.value_counts()", dtype: ['DataFrame', 'Series'], toframe: true }, ] }, @@ -153,7 +155,7 @@ define([ , code: "pd.plotting.scatter_matrix(${data}, marker='o', hist_kwds={'bins': 30}, s=30, alpha=.8)\nplt.show()" , dtype: ['DataFrame'] }, { id: 'boxplot', label: 'Box plot', code: "${data}.plot(kind='box')\nplt.show()", dtype: ['DataFrame', 'Series'] }, - { id: 'count_plot', label: 'Count plot', code: "${data}.value_counts().plot(kind='bar')\nplt.show()", dtype: ['DataFrame', 'Series'] } + { id: 'count_plot', label: 'Count plot', code: "${data}.value_counts().plot(kind='bar', rot=30)\nplt.show()", dtype: ['DataFrame', 'Series'] } ] } @@ -406,15 +408,17 @@ define([ // click run button $(this.wrapSelector('.vp-information-run-button')).click(function(event) { // get code - var code = that.generateCode(); + var code = that.generateCodeForInfo(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Data Info'); }); } @@ -426,6 +430,7 @@ define([ this.state.selected = []; this.state.selection = { start: -1, end: -1 }; this.renderMenu(); + $(this.wrapSelector('.' + VP_FE_TABLE)).html(''); this.loadCode(data); this.loadInfo(data, this.state.menu); } @@ -531,6 +536,10 @@ define([ } generateCode() { + return this.tempCode; + } + + generateCodeForInfo() { let { data, dtype, menu, menuItem } = this.state; var selected = []; @@ -597,18 +606,26 @@ define([ // only one method selected if (menuItem.length > 0 && infoObj.child) { let childObj = infoObj.child.find(obj=>obj.id === menuItem[0]); - if (childObj.toframe === true) { - if (dtype === 'Series') { - dataVar = new com_String(); - dataVar.appendFormat("{0}.to_frame()", data); - currentDtype = 'DataFrame'; - } else if (currentDtype === 'Series') { - dataVar = new com_String(); - dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); - currentDtype = 'DataFrame'; + if (menuItem[0] === 'value_counts') { + if (currentDtype === 'Series') { + codePattern = childObj.code2; + } else { + codePattern = childObj.code; } + } else { + if (childObj.toframe === true) { + if (dtype === 'Series') { + dataVar = new com_String(); + dataVar.appendFormat("{0}.to_frame()", data); + currentDtype = 'DataFrame'; + } else if (currentDtype === 'Series') { // DataFrame with single column selected + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } + } + codePattern = childObj.code; } - codePattern = childObj.code; } else { codePattern = infoObj.code; } @@ -809,11 +826,11 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue, msg.content?.detail); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(variablePreviewTag).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); that.loading = false; }); @@ -838,7 +855,8 @@ define([ // load preview content let $infoPreviewTag = $(this.wrapSelector('#informationPreview')); $infoPreviewTag.html(''); - let code = this.generateCode(); + let code = this.generateCodeForInfo(); + this.tempCode = code; // use default pandas option // let defaultPOCode = new com_String(); @@ -868,18 +886,18 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $infoPreviewTag.html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg, ename, evalue } = resultObj; var errorContent = ''; if (msg && msg?.content?.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } else if (ename && evalue) { errorContent = com_util.templateForErrorBox(ename, evalue); vpLog.display(VP_LOG_TYPE.ERROR, ename, evalue, resultObj); diff --git a/visualpython/js/m_apps/Instance.js b/visualpython/js/m_apps/Instance.js index 2dda58e9..1095ef31 100644 --- a/visualpython/js/m_apps/Instance.js +++ b/visualpython/js/m_apps/Instance.js @@ -33,7 +33,7 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 4; + this.config.sizeLevel = 5; this.config.checkModules = ['pd']; this.state = { @@ -263,7 +263,8 @@ define([ let cmObj = this.getCodemirror('vp_instanceVariable'); let rightCode = (cmObj && cmObj.cm)?cmObj.cm.getValue():''; if (leftCode && leftCode != '') { - sbCode.appendFormat('{0} = {1}', leftCode, rightCode); + sbCode.appendFormatLine('{0} = {1}', leftCode, rightCode); + sbCode.append(leftCode); // show allocation (from version 2.4.10) } else { sbCode.appendFormat('{0}', rightCode); } @@ -331,20 +332,25 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#instancePreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { - let { msg } = resultObj; + let { msg, ename, evalue, status } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg && msg.content && msg.content.ename) { + // NOTEBOOK: notebook error FIXME: migrate it on com_Kernel.execute errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + } else { + // LAB: lab error FIXME: migrate it on com_Kernel.execute + errorContent = com_util.templateForErrorBox(ename, evalue, ''); + vpLog.display(VP_LOG_TYPE.ERROR, ename, evalue); } $(that.wrapSelector('#instancePreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } diff --git a/visualpython/js/m_apps/PandasOption.js b/visualpython/js/m_apps/PandasOption.js index cb96e92c..359de409 100644 --- a/visualpython/js/m_apps/PandasOption.js +++ b/visualpython/js/m_apps/PandasOption.js @@ -30,6 +30,7 @@ define([ this.config.sizeLevel = 2; this.config.dataview = false; this.config.checkModules = ['pd']; + this.config.docs = 'https://pandas.pydata.org/docs/user_guide/options.html'; this.state = { filter_warning: '', diff --git a/visualpython/js/m_apps/Reshape.js b/visualpython/js/m_apps/Reshape.js index 022739a3..995ddd1e 100644 --- a/visualpython/js/m_apps/Reshape.js +++ b/visualpython/js/m_apps/Reshape.js @@ -277,7 +277,7 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); @@ -285,7 +285,7 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); diff --git a/visualpython/js/m_apps/Snippets.js b/visualpython/js/m_apps/Snippets.js index 3c031163..1fecfce7 100644 --- a/visualpython/js/m_apps/Snippets.js +++ b/visualpython/js/m_apps/Snippets.js @@ -18,10 +18,11 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(snHtml, snCss, com_util, com_Const, com_String, PopupComponent, FileNavigation, LoadingSpinner) { +], function(snHtml, snCss, com_util, com_Const, com_String, com_interface, PopupComponent, FileNavigation, LoadingSpinner) { /** * Snippets @@ -471,14 +472,16 @@ define([ let cmCode = that.codemirrorList[title]; cmCode.save(); var code = cmCode.getValue(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Snippets'); } else if (menu == 'duplicate') { var dupNo = 1; var timestamp = new Date().getTime(); @@ -522,7 +525,7 @@ define([ // vp-multilang for success message com_util.renderSuccessMessage('Successfully removed!'); }).catch(function(err) { - com_util.renderAlertModal('Failed to remove data...', err); + com_util.renderAlertModal('Failed to remove data...', { content: err }); // load again that.loadUdfList(); }).finally(function() { @@ -540,7 +543,7 @@ define([ // vp-multilang for success message com_util.renderSuccessMessage('Successfully removed!'); }).catch(function(err) { - com_util.renderAlertModal('Failed to remove data...', err); + com_util.renderAlertModal('Failed to remove data...', { content: err }); // load again that.loadUdfList(); }).finally(function() { diff --git a/visualpython/js/m_apps/Subset.js b/visualpython/js/m_apps/Subset.js index 5c923a62..41af3ef4 100644 --- a/visualpython/js/m_apps/Subset.js +++ b/visualpython/js/m_apps/Subset.js @@ -343,10 +343,13 @@ define([ $(this.wrapSelector('.select-row .vp-ds-select-box.left')).on('scroll', function() { if ($(this).scrollTop() + $(this).innerHeight() >= ($(this)[0].scrollHeight - 2)) { let scrollPos = $(this).scrollTop(); + if (that.state.rowLimit > that.state.rowList.length){ + return; // Prevents scroll from being fixed downwards + } let start = that.state.rowLimit; let end = start + 10; let subsetVariable = com_util.formatString('{0}.iloc[{1}:{2}]', that.state.pandasObject, start, end); - vpKernel.getRowList(subsetVariable).then(function (resultObj) { + vpKernel.getRowList(subsetVariable, start).then(function (resultObj) { let { result } = resultObj; var { list:rowList } = JSON.parse(result); rowList = rowList.map(function (x) { @@ -361,9 +364,9 @@ define([ rowList = rowList.map(function (x) { return { ...x, - label: x.location + '', - value: x.location + '', - code: x.location + '', + label: x.label + '', + value: x.value + '', + code: x.code + '', }; }); } @@ -737,20 +740,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); } /** @@ -1742,7 +1745,7 @@ define([ var connector = i > 0 ? $(condList[i - 1]).find('.vp-oper-connect').val() : undefined; // if no variable selected, pass - if (varName == "") + if (varName === "" || oper === "") continue; if (useCondition) { rowSelection.append(connector); @@ -1810,6 +1813,10 @@ define([ rowSelection.append(')'); } else { rowSelection.appendFormat('({0}', varName); + if (colName == '.index') { + // index + rowSelection.append('.index'); + } oper && rowSelection.appendFormat(' {0}', oper); if (cond) { // condition value as text @@ -1863,7 +1870,7 @@ define([ var colList = []; for (var i = 0; i < colTags.length; i++) { var colValue = $(colTags[i]).data('code'); - if (colValue) { + if (colValue !== undefined) { colList.push(colValue); } } @@ -1873,7 +1880,7 @@ define([ $(this.wrapSelector('.' + VP_DS_TO_FRAME)).parent().show(); // to frame - if (this.state.toFrame) { + if (this.state.toFrame === true) { colSelection.appendFormat('[{0}]', colList.toString()); this.state.returnType = 'DataFrame'; } else { diff --git a/visualpython/js/m_library/m_pandas/readFile.js b/visualpython/js/m_library/m_pandas/readFile.js index 296d3509..c9144db9 100644 --- a/visualpython/js/m_library/m_pandas/readFile.js +++ b/visualpython/js/m_library/m_pandas/readFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -60,7 +61,8 @@ define([ 'csv': 'pd_readCsv', 'excel': 'pd_readExcel', 'json': 'pd_readJson', - 'pickle': 'pd_readPickle' + 'pickle': 'pd_readPickle', + 'parquet': 'pd_readParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_library/m_pandas/toFile.js b/visualpython/js/m_library/m_pandas/toFile.js index 41a58ab3..08bb45c8 100644 --- a/visualpython/js/m_library/m_pandas/toFile.js +++ b/visualpython/js/m_library/m_pandas/toFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -60,7 +61,8 @@ define([ 'csv': 'pd_toCsv', 'excel': 'pd_toExcel', 'json': 'pd_toJson', - 'pickle': 'pd_toPickle' + 'pickle': 'pd_toPickle', + 'parquet': 'pd_toParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_ml/Classification.js b/visualpython/js/m_ml/Classification.js index f487be39..f327b611 100644 --- a/visualpython/js/m_ml/Classification.js +++ b/visualpython/js/m_ml/Classification.js @@ -50,7 +50,7 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Classfication': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], + 'Classification': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], } diff --git a/visualpython/js/m_ml/Clustering.js b/visualpython/js/m_ml/Clustering.js index c3781201..95ee735c 100644 --- a/visualpython/js/m_ml/Clustering.js +++ b/visualpython/js/m_ml/Clustering.js @@ -51,7 +51,7 @@ define([ this.modelTypeList = { // 'Regression': ['ln-rgs', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], - // 'Classfication': ['lg-rgs', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], + // 'Classification': ['lg-rgs', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], // 'Auto ML': ['tpot-rgs', 'tpot-clf'], 'Clustering': ['k-means', 'agg-cls', 'gaus-mix', 'dbscan'], // 'Dimension Reduction': ['pca', 'lda', 'svd', 'nmf'] diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index 983a8168..10b9625f 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -57,13 +57,13 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], - 'ETC': ['make-column-transformer'] + 'ETC': ['prep-simple-imputer', 'prep-smote', 'make-column-transformer'] } this.mctEstimator = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], } @@ -94,10 +94,19 @@ define([ $(that.wrapSelector('#vp_installLibrary')).hide(); } - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { // load mct-targetData that.loadVariableList(); that.bindMCT(); + } else if (modelType === 'prep-simple-imputer') { + $(that.wrapSelector('#missing_values')).replaceWith(`

+ ${$(that.wrapSelector('#missing_values'))[0].outerHTML} + Text +

`); + $(that.wrapSelector('#fill_value')).replaceWith(`

+ ${$(that.wrapSelector('#fill_value'))[0].outerHTML} + Text +

`); } }); @@ -411,6 +420,7 @@ define([ generateCode() { let { modelControlType, modelType, userOption, allocateToCreation, model } = this.state; + let state = JSON.parse(JSON.stringify(this.state)); let code = new com_String(); if (modelControlType == 'creation') { /** @@ -422,22 +432,33 @@ define([ let config = this.modelConfig[modelType]; code.appendLine(config.import); + if (modelType === 'prep-simple-imputer') { + let checkList = ['missing_values', 'fill_value']; + checkList.forEach(checkKey => { + try { + state[checkKey] = com_util.convertToStr(state[checkKey], state[checkKey + '_istext']); + } catch(e) { + ; + } + }); + } + // model code let modelCode = config.code; - modelCode = com_generator.vp_codeGenerator(this, config, this.state, (userOption != ''? ', ' + userOption : '')); + modelCode = com_generator.vp_codeGenerator(this, config, state, (userOption != ''? ', ' + userOption : '')); // generate mct code - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { let mctCodes = []; - let { mct_estimator1, mct_columns1, mct_estimator2, mct_columns2 } = this.state; + let { mct_estimator1, mct_columns1='', mct_estimator2, mct_columns2='' } = state; if (mct_estimator1 != undefined && mct_estimator1 != '') { code.appendLine(this.modelConfig[mct_estimator1].import); - let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator1code, mct_columns1)); } if (mct_estimator2 != undefined && mct_estimator2 != '') { code.appendLine(this.modelConfig[mct_estimator2].import); - let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator2code, mct_columns2)); } modelCode = modelCode.replace('${mct_code}', mctCodes.join(', ')); diff --git a/visualpython/js/m_ml/FitPredict.js b/visualpython/js/m_ml/FitPredict.js index 8cf91de5..d48c20c5 100644 --- a/visualpython/js/m_ml/FitPredict.js +++ b/visualpython/js/m_ml/FitPredict.js @@ -41,6 +41,8 @@ define([ method: '', action: {}, optionConfig: {}, + modelEditorType: '', + modelEditorName: '', userOption: '', ...this.state } @@ -315,7 +317,7 @@ define([ } generateCode() { - let { model } = this.state; + let { model, modelType, modelEditorName } = this.state; let code = new com_String(); let replaceDict = {'${model}': model}; @@ -336,6 +338,31 @@ define([ code.appendLine(); code.append(allocateCode); } + // Data Preparation > Scaling + const scalingTypeList = ['StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer']; + // Dimension Reduction + const dimensionTypeList = ['PCA', 'NMF']; + if (scalingTypeList.includes(modelType) || dimensionTypeList.includes(modelType)) { + // fit_transform, transform returns df_trans also + switch (modelEditorName) { + case 'fit_transform': + const allocatedFitTrans = this.state.fit_trans_allocate || 'trans'; + code.appendLine(); + code.appendLine(); + code.appendFormatLine("df_{0} = pd.DataFrame({1}, columns=[{2}])", allocatedFitTrans, allocatedFitTrans, this.state.fit_trans_featureData); + code.append("df_" + allocatedFitTrans); + break; + case 'transform': + const allocatedTrans = this.state.trans_allocate || 'trans'; + code.appendLine(); + code.appendLine(); + code.appendFormatLine("df_{0} = pd.DataFrame({1}, columns=[{2}])", allocatedTrans, allocatedTrans, this.state.trans_featureData); + code.append("df_" + allocatedTrans); + break; + default: + break; + } + } } return code.toString(); @@ -409,7 +436,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -418,16 +445,21 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, 'transform': { - ...defaultActions['transform'], - description: 'Transform labels to normalized encoding.' + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -437,12 +469,75 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } } } + if (modelType == 'LabelEncoder') { + actions = { + ...actions, + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Fit Encoder/Scaler to X.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' } + ] + }, + 'fit_transform': { + name: 'fit_transform', + label: 'Fit and transform', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', + description: 'Fit Encoder/Scaler to X, then transform X.', + options: [ + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -581,7 +676,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -590,7 +685,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -605,7 +700,7 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, @@ -615,7 +710,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] @@ -626,7 +721,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -636,7 +731,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -650,7 +745,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -659,7 +754,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -669,7 +764,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -679,12 +774,58 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}${fit_targetData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } diff --git a/visualpython/js/m_ml/GridSearch.js b/visualpython/js/m_ml/GridSearch.js new file mode 100644 index 00000000..1dfdbaa4 --- /dev/null +++ b/visualpython/js/m_ml/GridSearch.js @@ -0,0 +1,564 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : GridSearch.js + * Author : Black Logic + * Note : GridSearch + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] GridSearch +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/gridSearch.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/gridSearch'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, com_util, com_interface, com_String, com_generator, ML_LIBRARIES, PopupComponent, MultiSelector, SuggestInput, ModelEditor) { + + /** + * GridSearch + */ + class GridSearch extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 3; + this.config.dataview = false; + this.config.docs = 'https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html'; + + this.state = { + modelType: 'ln-rgs', + userOption: '', + allocateToCreation: 'model', + ...this.state + } + + this.targetSetTag = null; + + this.modelConfig = ML_LIBRARIES; + + this.modelTypeList = { + 'Regression': ['ln-rgs', 'ridge', 'lasso', 'elasticnet', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], + 'Classification': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'] + } + + + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('click', this.wrapSelector('.vp-param-set-del')); + $(document).off('click', this.wrapSelector('.vp-param-item-add')); + $(document).off('click', this.wrapSelector('.vp-param-item-del')); + $(document).off('keyup', this.wrapSelector('.vp-param-val')); + $(document).off('click', this.wrapSelector('.vp-param-result-item-del')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select model type + $(this.wrapSelector('#modelType')).on('change', function() { + let modelType = $(this).val(); + that.state.modelType = modelType; + + // show install button + if (that.modelConfig[modelType].install != undefined) { + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + + that.handleScoringOptions(modelType); + + // reset model param set + $(that.wrapSelector('.vp-param-grid-box')).html(''); + $(that.wrapSelector('.vp-param-grid-box')).html(that.templateForParamSet()); + }); + + // Add param set + $(this.wrapSelector('#vp_addParamSet')).on('click', function() { + let newSet = $(that.templateForParamSet()); + $(that.wrapSelector('.vp-param-grid-box')).append(newSet); + // focus + $(newSet)[0].scrollIntoView(); + }); + + // delete param set + $(document).on('click', this.wrapSelector('.vp-param-set-del'), function() { + $(this).closest('.vp-param-set-box').remove(); + + // rename param set + $(that.wrapSelector('.vp-param-set-name')).each((i, tag) => { + $(tag).text('Param set ' + (i + 1)); + }); + }); + + // Add param item + $(document).on('click', this.wrapSelector('.vp-param-item-add'), function() { + that.targetSetTag = $(this).parent(); // target param-set-box + that.openParamPopup(); + }); + + // Delete param item + $(document).on('click', this.wrapSelector('.vp-param-item-del'), function() { + $(this).closest('.vp-param-item').remove(); + }); + + // add param value - using enter + $(document).on('keyup', this.wrapSelector('.vp-param-val'), function(event) { + var keycode = event.keyCode + ? event.keyCode + : event.which; + if (keycode == vpEvent.keyManager.keyCode.enter) { // enter + that.handleAddParamValue($(this)); + } + if (keycode === 188) {// ,< + let val = $(this).val(); + $(this).val(val.split(',')[0]); // remove , and add param + that.handleAddParamValue($(this)); + } + }); + + // delete param set item + $(document).on('click', this.wrapSelector('.vp-param-result-item-del'), function() { + $(this).closest('.vp-param-result-item').remove(); + }); + } + + handleAddParamValue(thisTag) { + let parentTag = $(thisTag).parent(); + let paramIsText = $(parentTag).find('.vp-param-val').data('type') === 'text'; // text / var + let paramVal = $(parentTag).find('.vp-param-val').val(); + let reservedKeywordList = ['None', 'True', 'False', 'np.nan', 'np.NaN']; + if (reservedKeywordList.includes(paramVal)) { + paramIsText = false; + } + // check , and split it + let paramSplit = paramVal.split(','); + paramSplit && paramSplit.forEach(val => { + // add only if it is not empty value + if (val.trim() !== '') { + val = com_util.convertToStr(val.trim(), paramIsText); + $(parentTag).find('.vp-param-result-box').append(` + ${val} + + + `); + // clear param val + $(parentTag).find('.vp-param-val').val(''); + } + }); + } + + bindEventForInnerPopup() { + let that = this; + // Inner popup: Select param + $(document).off('click', this.wrapSelector('.vp-inner-param-list-item')); + $(document).on('click', this.wrapSelector('.vp-inner-param-list-item'), function() { + if ($(this).hasClass('selected')) { + $(this).removeClass('selected'); + } else { + $(this).addClass('selected'); + } + }); + + // Inner popup: Add param + $(this.wrapSelector('.vp-add-param-btn')).on('click', function() { + let newParam = $(that.wrapSelector('.vp-add-param-name')).val(); + if (newParam != undefined && newParam.trim() != '') { + // check if exist + let checkTag = $(that.wrapSelector(`.vp-inner-param-list-item[data-name="${newParam}"]`)); + if (checkTag.length > 0) { + if (!checkTag.hasClass('selected')) { + checkTag.addClass('selected'); + } + checkTag[0].scrollIntoView(); + } else { + // add to param list + $(that.wrapSelector('.vp-inner-param-list-box')).append( + `

${newParam}

` + ); + // scroll to added item + $(that.wrapSelector('.vp-inner-param-list-item.selected:last'))[0].scrollIntoView(); + } + + // clear input value + $(that.wrapSelector('.vp-add-param-name')).val(''); + } + }); + } + + handleScoringOptions(modelType) { + let options = { + 'Classification': [ + "'accuracy'", + "'balanced_accuracy'", + "'top_k_accuracy'", + "'average_precision'", + "'neg_brier_score'", + "'f1'", + "'f1_micro'", + "'f1_macro'", + "'f1_weighted'", + "'f1_samples'", + "'neg_log_loss'", + "'precision' etc.", + "'recall' etc.", + "'jaccard' etc.", + "'roc_auc'", + "'roc_auc_ovr'", + "'roc_auc_ovo'", + "'roc_auc_ovr_weighted'", + "'roc_auc_ovo_weighted'", + ], + 'Regression': [ + "'explained_variance'", + "'max_error'", + "'neg_mean_absolute_error'", + "'neg_mean_squared_error'", + "'neg_root_mean_squared_error'", + "'neg_mean_squared_log_error'", + "'neg_median_absolute_error'", + "'r2'", + "'neg_mean_poisson_deviance'", + "'neg_mean_gamma_deviance'", + "'neg_mean_absolute_percentage_error'", + "'d2_absolute_error_score'", + "'d2_pinball_score'", + "'d2_tweedie_score'" + ] + } + let modelCategory = this.modelTypeList['Regression'].includes(modelType)?'Regression':'Classification'; + + // Set suggestInput on scoring option + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('scoring'); + suggestInput.setPlaceholder('Select option'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return options[modelCategory]; }); + suggestInput.setNormalFilter(true); + $(this.wrapSelector('#scoring')).replaceWith(suggestInput.toTagString()); + } + + templateForParamSet() { + let paramSetNo = 1; + // set param set number + paramSetNo += $(this.wrapSelector('.vp-param-set-box')).length; + return `

+ Param set ${paramSetNo} +

+ +

`; + } + + templateForParamPopup() { + let config = this.modelConfig[this.state.modelType]; + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('

{2}

' + , opt.name, opt.name, opt.name); + }); + + return ` +

NOTE: Select parameters to add.

+ ${optBox.toString()} +

+ + +

+ `; + } + + openParamPopup() { + let size = { width: 400, height: 300 }; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); + + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); + + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForParamPopup()); + + this.bindEventForInnerPopup(); + + // show popup box + this.openInnerPopup('Add parameter'); + } + + handleInnerOk() { + // get selected param list + let paramList = []; + $(this.wrapSelector('.vp-inner-param-list-item.selected')).each((i, tag) => { + paramList.push($(tag).data('name')); + }); + + if (paramList.length > 0) { + // add param box + $(this.targetSetTag).find('.vp-param-item-box').append(this.templateForParamItemList(paramList)); + + this.closeInnerPopup(); + } else { + com_util.renderAlertModal('No params selected. Please select more than one param.'); + } + + this.targetSetTag = null; + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Model creation + //================================================================ + // model types + let modelTypeTag = new com_String(); + Object.keys(this.modelTypeList).forEach(modelCategory => { + let modelOptionTag = new com_String(); + that.modelTypeList[modelCategory].forEach(opt => { + let optConfig = that.modelConfig[opt]; + let selectedFlag = ''; + if (opt == that.state.modelType) { + selectedFlag = 'selected'; + } + modelOptionTag.appendFormatLine('', + opt, selectedFlag, optConfig.name); + }) + modelTypeTag.appendFormatLine('{1}', + modelCategory, modelOptionTag.toString()); + }); + $(page).find('#modelType').html(modelTypeTag.toString()); + + //================================================================ + // GridSearch option + //================================================================ + $(page).find('.vp-model-option-box').html(this.templateForOption('grid-search', ['estimator', 'param_grid'])); + + // show install button + if (this.modelConfig[this.state.modelType].install != undefined) { + this.showInstallButton(); + } else { + this.hideInstallButton(); + } + + // render option page + // $(page).find('.vp-model-param-box').html(this.templateForOption(this.state.modelType)); + + return page; + } + + templateForOption(modelType, excludeOptList=[]) { + let config = this.modelConfig[modelType]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + if (!excludeOptList.includes(opt.name)) { + optBox.appendFormatLine('{2}' + , opt.name, opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + } + }); + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + return optBox.toString(); + } + + /** + * template for param box + * @param {*} paramList ['param-name1', 'param-name2', ...] + */ + templateForParamItemList(paramList) { + let that = this; + let config = this.modelConfig[this.state.modelType]; + let paramSet = new com_String(); + paramList && paramList.forEach(name => { + // get option component + let componentType = 'input'; // default + let paramObj = config.options.filter(x => x.name === name).at(0); + if ((paramObj !== undefined) && (['option_select', 'bool_select'].includes(paramObj.component[0]))) { + componentType = 'option'; + } + paramSet.appendLine('

'); + paramSet.appendFormatLine('{1}', name, com_util.optionToLabel(name)); + paramSet.appendLine('

'); + paramSet.appendFormatLine('

', name); + paramSet.appendFormatLine('', name); + if (componentType === 'input') { + paramSet.appendFormatLine('', name); + + } else { + // paramSet.appendFormatLine('', name); + var suggestInput = new SuggestInput(); + suggestInput.setPlaceholder('Type here'); + suggestInput.addClass('vp-param-val'); + suggestInput.addAttribute('data-name', name); + suggestInput.addAttribute('data-type', (paramObj.type === 'text' ? 'text' : 'var')); + let optionList = paramObj.options; + if (paramObj.component[0] === 'bool_select') { + optionList = ['True', 'False']; + } + suggestInput.setSuggestList(function() { return optionList; }); + suggestInput.setNormalFilter(false); + suggestInput.setSelectEvent(function(selectedValue) { + // trigger change + let thisTag = $(that.wrapSelector('.' + suggestInput.uuid)); + that.handleAddParamValue($(thisTag)); + $(thisTag).val(''); + return false; + }); + paramSet.appendLine(suggestInput.toTagString()); + } + paramSet.appendLine('

'); + paramSet.appendLine('

'); // vp-param-item + }); + return paramSet.toString(); + } + + render() { + super.render(); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + this.handleScoringOptions(this.state.modelType); + } + + generateInstallCode() { + let installCode = this.modelConfig[this.state.modelType].install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + return [ installCode ]; + } + + checkBeforeRun() { + // if no param is registered, stop and show alert + if ($(this.wrapSelector('.vp-param-result-item')).length <= 0) { + com_util.renderAlertModal('No params added. Please add params.'); + return false; + } + + return true; + } + + generateCode() { + let { modelType, userOption, allocateToCreation, model } = this.state; + let code = new com_String(); + /** + * Model Creation + * --- + * from module import model_function + * model = Model(key=value, ...) + */ + let gsConfig = this.modelConfig['grid-search']; + + + let state = JSON.parse(JSON.stringify(this.state)); + let estConfig = this.modelConfig[this.state.modelType]; + let estimator = com_generator.vp_codeGenerator(null, estConfig, {}, ''); + state['estimator'] = estimator; + state['param_grid'] = '{}'; + + let paramGrid = []; + // generate param_grid + $(this.wrapSelector('.vp-param-set-box')).each((i, tag) => { + let paramSet = {}; + $(tag).find('.vp-param-result-box').each((j, resTag) => { + let paramName = $(resTag).data('name'); + let paramValList = []; + $(resTag).find('.vp-param-result-item').each((k, itemTag) => { + let val = $(itemTag).data('value'); + if (!isNaN(val)) { + // numeric string -> numeric + val = parseFloat(val); + } + paramValList.push(val); + }); + if (paramValList.length > 0) { + // Add only its result exists + paramSet[paramName] = paramValList; + } + }); + if (Object.keys(paramSet).length > 0) { + paramGrid.push(paramSet); + } + }); + + state['param_grid'] = this.paramStringify(paramGrid); + + // import code + code.appendLine(gsConfig.import); + code.appendLine(estConfig.import); + + // model code + let gsCode = gsConfig.code; + gsCode = com_generator.vp_codeGenerator(this, gsConfig, state, (userOption != ''? ', ' + userOption : '')); + code.appendLine(); + code.appendFormat('{0} = {1}', allocateToCreation, gsCode); + + return code.toString(); + } + + paramStringify(paramGridList=[]) { + let paramGridCode = new com_String(); + if (paramGridList.length > 1) { + paramGridCode.append('['); + } + paramGridList.forEach((paramSet, i) => { + if (i > 0) { + paramGridCode.append(', \n '); + } + paramGridCode.append('{'); + Object.keys(paramSet).forEach((paramName, j) => { + if (j > 0) { + paramGridCode.append(', '); + } + paramGridCode.appendFormat("'{0}': [{1}]", paramName, paramSet[paramName].toString()); + }); + paramGridCode.append('}'); + }) + + if (paramGridList.length > 1) { + paramGridCode.append(']'); + } + return paramGridCode.toString(); + } + + } + + return GridSearch; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 00cdcf4a..23bf679b 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -260,6 +260,10 @@ define([ that.config.checkModules = ['pd', 'vp_create_feature_importances']; } else if (name == 'plot_feature_importances') { that.config.checkModules = ['pd', 'plt', 'vp_create_feature_importances', 'vp_plot_feature_importances']; + } else if (name == 'permutation_importance') { + that.config.checkModules = ['pd', 'vp_create_permutation_importances']; + } else if (name == 'plot_permutation_importance') { + that.config.checkModules = ['pd', 'vp_create_permutation_importances', 'vp_plot_permutation_importances']; } else { that.config.checkModules = ['pd']; } @@ -409,16 +413,46 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, - { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', @@ -514,6 +548,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] @@ -545,7 +592,44 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'], + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'Coefficient': { name: 'coef_', label: 'Coefficient', @@ -631,7 +715,44 @@ define([ { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'] + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, } // feature importances @@ -779,6 +900,39 @@ define([ } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js new file mode 100644 index 00000000..856c33db --- /dev/null +++ b/visualpython/js/m_ml/Pipeline.js @@ -0,0 +1,798 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Pipeline.js + * Author : Black Logic + * Note : Pipeline + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] Pipeline +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/pipeline.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/pipeline'), + __VP_RAW_LOADER__('vp_base/data/libraries.json'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, librariesJson, com_util, com_String, com_generator, ML_LIBRARIES, PopupComponent, SuggestInput, ModelEditor) { + + /** + * Pipeline + */ + class Pipeline extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 4; + this.config.dataview = false; + this.config.autoScroll = false; + + this.state = { + templateType: '', + modelStep: -1, // get modelStep sequence + modelType: '', // get modelType + modelTypeName: '', // get modelTypeName + model: 'model', + // now selected pipeline info + pipeline: [ + // copy of step + info + // { name, label, useApp, state, app } + ], + ...this.state + } + + /** + * useApp + - Data Split + - Data Prep + - Regressor + - Classifier + - Clustering + - Dimension + - GridSearch + - Evaluation + + - Fit + - Transform + - Predict + - Fit and Predict + - Fit and Transform + - Fit and Resample + */ + this.templateList = { + 'data-prep': { + label: 'Data Preparation', + modelStep: 0, + step: [ + /** + * ml_* is pre-defined app + * pp_* is defined only for Pipeline + */ + { name: 'ml_dataPrep', label: 'Data Prep', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_resample'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_resample', label: 'Fit and Resample' } + ] + }, + 'regression': { + label: 'Regression', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_regression', label: 'Regressor', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'rgs' } }, + ] + }, + 'classification': { + label: 'Classification', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_classification', label: 'Classifier', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'clf' } }, + ] + }, + 'clustering': { + label: 'Clustering', + modelStep: 0, + step: [ + { name: 'ml_clustering', label: 'Clustering', useApp: true, child: ['pp_fit', 'pp_predict', 'pp_fit_predict', 'pp_transform'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'pp_fit_predict', label: 'Fit and Predict' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'cls' } }, + ] + }, + 'dimension': { + label: 'Dimension Reduction', + modelStep: 0, + step: [ + { name: 'ml_dimensionReduction', label: 'Dimension Reduction', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_transform'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_transform', label: 'Fit and Transform' }, + ] + }, + 'gridSearch': { + label: 'GridSearch', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_gridSearch', label: 'GridSearch', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true }, + ] + } + } + + // menu libraries for ml + let libObj = {}; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + libObj = librariesJson; + + this.MlAppComponent = {}; + this.MlAppComponent['ml_dataSplit'] = require('./dataSplit'); + this.MlAppComponent['ml_dataPrep'] = require('./DataPrep'); + this.MlAppComponent['ml_regression'] = require('./Regression'); + this.MlAppComponent['ml_classification'] = require('./Classification'); + this.MlAppComponent['ml_clustering'] = require('./Clustering'); + this.MlAppComponent['ml_dimensionReduction'] = require('./DimensionReduction'); + this.MlAppComponent['ml_gridSearch'] = require('./GridSearch'); + this.MlAppComponent['ml_evaluation'] = require('./evaluation'); + } else { + libObj = JSON.parse(librariesJson); + } + this.mlAppList = libObj.library.item.filter(x => x.id === 'pkg_ml')[0].item; + + this.modelConfig = ML_LIBRARIES; + } + + _unbindEvent() { + super._unbindEvent(); + + $(document).off('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]')); + $(document).off('click', this.wrapSelector('.vp-pp-item-toggle')); + $(document).off('change', this.wrapSelector(`#modelType`)); + $(document).off('change', this.wrapSelector('#allocateToCreation')); + + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select template + $(this.wrapSelector('#templateType')).on('change', function() { + let type = $(this).val(); + that.state.templateType = type; + that.state.model = 'model'; + that.state.modelType = ''; + that.state.modelTypeName = ''; + that.state.modelStep = -1; + + // reset pp-page + $(that.wrapSelector('.vp-pp-template')).html(''); + $(that.wrapSelector('.vp-pp-step-content')).html(''); + $(that.wrapSelector('.vp-pp-step-title')).text(''); + that.state.pipeline = []; + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + + that.handleChangeTemplate(type); + }); + + // select pipeline item + $(document).on('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]'), function() { + if (!$(this).hasClass('selected')) { + let title = $(this).data('label'); + let stepSeq = parseInt($(this).data('step')); // 0 ~ n + let name = $(this).data('name'); + let ppObj = that.state.pipeline[stepSeq]; + // set title + $(that.wrapSelector('.vp-pp-step-title')).text(title); + + // show page + $(that.wrapSelector(`.vp-pp-step-page:not([data-name="${name}"])`)).hide(); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).show(); + if (ppObj.useApp === true) { + ppObj.app && that.handleAppView(name, ppObj.app); + ppObj.app && ppObj.app.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`))); + } else { + that.renderApp(name); + } + + // check selected + $(that.wrapSelector('.vp-pp-item')).removeClass('selected'); + $(this).addClass('selected'); + + // check prev/next + let prevTagList = $(this).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(this).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } + } + }); + + // pipeline item toggle (enable/disable) + $(document).on('click', this.wrapSelector('.vp-pp-item-toggle'), function(evt) { + evt.stopPropagation(); + let itemTag = $(this).closest('.vp-pp-item'); + let name = $(itemTag).attr('data-name'); + let flag = $(itemTag).attr('data-flag'); // enabled / disabled + if (flag === 'enabled') { + $(itemTag).attr('data-flag', 'disabled'); + $(this).prop('checked', false); + } else if (flag === 'disabled') { + $(itemTag).attr('data-flag', 'enabled'); + $(this).prop('checked', true); + } + + if ($(itemTag).hasClass('selected')) { + // check if this page is this item's, then hide it. + $(that.wrapSelector('.vp-pp-step-title')).text(''); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + $(itemTag).removeClass('selected'); + + // disable prev/next + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + // check prev/next + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } + } + + }); + + // model type change event + $(document).on('change', this.wrapSelector(`#modelType`), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelCreatorList = ['ml_dataPrep', 'ml_regression', 'ml_classification', 'ml_clustering', 'ml_dimensionReduction', 'ml_gridSearch']; + if (modelCreatorList.includes(name)) { + let modelType = $(this).val(); + let modelObj = that.modelConfig[modelType]; + let modelTypeName = modelObj.returnType; + + that.state.modelType = modelType; + that.state.modelTypeName = modelTypeName; + + // show fit / predict / transform depends on model selection + let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform', 'fit_resample']; + let actions = that.modelEditor.getAction(modelTypeName); + defaultActions.forEach(actKey => { + if (actions[actKey] === undefined) { + // if undefined, hide step + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).show(); + if (actKey === 'fit_predict') { + if (actions['fit'] === undefined || actions['predict'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_predict"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_transform') { + if (actions['fit'] === undefined || actions['transform'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_resample') { + // for SMOTE: show fit_resample only + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } + } + $(that.wrapSelector('.vp-pp-item')).removeClass('vp-last-visible'); + $(that.wrapSelector('.vp-pp-item:visible:last')).addClass('vp-last-visible'); + }); + + // change evaluation page's default selection + if (name === 'ml_gridSearch') { + let modelCategory = that.modelEditor.getModelCategory(modelTypeName); // Regression / Classification / Clustering + let evalModelTypeValue = 'rgs'; + switch (modelCategory) { + case 'Regression': + evalModelTypeValue = 'rgs'; + break; + case 'Classification': + evalModelTypeValue = 'clf'; + break; + case 'Clustering': + evalModelTypeValue = 'cls'; + break; + } + let evalPageTag = $(that.wrapSelector('.vp-pp-step-page[data-name="ml_evaluation"]')); + if (evalPageTag.is(':empty') === true) { + // evaluation page is not opened yet + // set state for evaluation page + let stepSeq = $(evalPageTag).data('step'); + let ppObj = that.state.pipeline[stepSeq]; + if (ppObj.app) { + ppObj.app.state.modelType = evalModelTypeValue; + } + } else { + let evalModelTypeTag = $(evalPageTag).find('#modelType'); + $(evalModelTypeTag).val(evalModelTypeValue); + $(evalModelTypeTag).trigger('change'); + } + } + } + }); + + // model allocation variable change + $(document).on('change', this.wrapSelector('#allocateToCreation'), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelAllocation = $(this).val(); + that.state.model = modelAllocation; + }); + + // click prev button + $(this.wrapSelector('.vp-pp-step-prev:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(prevTagList[0]).trigger('click'); + }); + + // click next button + $(this.wrapSelector('.vp-pp-step-next:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(nextTagList[0]).trigger('click'); + }); + } + + /** + * + * @param {*} type template type + */ + handleChangeTemplate(type) { + let that = this; + if (type !== '') { + let tplObj = this.templateList[type]; + this.state.modelStep = tplObj.modelStep; + + let ppTag = new com_String(); + let appFileList = []; + // load pipeline items + tplObj.step.forEach((stepObj, idx) => { + let { name, label, useApp=false, child=[], state={} } = stepObj; + ppTag.appendFormatLine(`

+ {3} + +

`, name, idx, label, label); + + // get pages + if (useApp === true) { + let mlObj = that.mlAppList.filter(x => x.id === name)[0]; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } else { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } + } + let pipeObj = { + name: name, + label: label, + useApp: useApp, + state: state + }; + if (tplObj.modelStep === idx) { + pipeObj.modelStep = true; + pipeObj.child = child; + } + that.state.pipeline.push(pipeObj); + // append pages + $(that.wrapSelector('.vp-pp-step-content')).append( + `

`); + if (useApp === false) { + that.renderApp(name); + // hide + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + } + + }); + $(that.wrapSelector('.vp-pp-template')).html(ppTag.toString()); + + // render pages + // for lite and lab + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.forEach((obj, argIdx) => { + let MlComponent = that.MlAppComponent[obj.name]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: that.state.pipeline[index].label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + mlComponent.loadState(); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + + that.state.pipeline[index].child.forEach(childId => { + that.renderApp(childId); + }); + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); + // end of DUP AREA: pp-1 + } + }); + + } else { + // for notebook and others + window.require(appFileList.map(x => x.file), function() { + appFileList.forEach((obj, argIdx) => { + let MlComponent = arguments[argIdx]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: that.state.pipeline[index].label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + mlComponent.loadState(); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + + that.state.pipeline[index].child.forEach(childId => { + that.renderApp(childId); + }); + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); + // end of DUP AREA: pp-1 + } + }) + + }); + } + } + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Template list creation + //================================================================ + let tplTag = new com_String(); + tplTag.appendLine(''); + Object.keys(this.templateList).forEach(tplKey => { + let tplObj = that.templateList[tplKey]; + let selectedFlag = ''; + if (tplKey == that.state.templateType) { + selectedFlag = 'selected'; + } + tplTag.appendFormatLine('', + tplKey, selectedFlag, tplObj.label); + }); + $(page).find('#templateType').html(tplTag.toString()); + + return page; + } + + render() { + super.render(); + + // resize codeview + $(this.wrapSelector('.vp-popup-codeview-box')).css({'height': '300px'}); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + // load state + if (this.state.templateType !== '') { + $(this.wrapSelector('#templateType')).trigger('change'); + } else { + $(this.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(this.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } + } + + /** + * Handle app view before open + * @param {string} appId + * @param {*} mlApp + */ + handleAppView(appId, mlApp) { + switch (appId) { + case 'ml_dataSplit': + $(mlApp.wrapSelector('#inputData')).parent().hide(); + break; + case 'ml_evaluation': + // for pipeline + $(mlApp.wrapSelector('.vp-upper-box')).hide(); + $(mlApp.wrapSelector('.vp-upper-box.' + mlApp.state.modelType)).show(); + + $(mlApp.wrapSelector('.vp-eval-box')).hide(); + $(mlApp.wrapSelector('.vp-eval-' + mlApp.state.modelType)).show(); + + if (mlApp.state.modelType == 'rgs') { + // Regression + + } else if (mlApp.state.modelType == 'clf') { + // Classification + // if (this.state.roc_curve == false && this.state.auc == false) { + // $(page).find('.vp-ev-model.roc-auc').prop('disabled', true); + // } + } else { + // Clustering + if (mlApp.state.silhouetteScore == false) { + $(mlApp.wrapSelector('.vp-ev-model.silhouette')).prop('disabled', true); + } + if (mlApp.state.ari == false && mlApp.state.nmi == false) { + $(mlApp.wrapSelector('.vp-ev-model.ari-nmi')).prop('disabled', true); + } + } + break; + } + } + + /** + * Custom pages + * @param {*} appId + */ + renderApp(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let tag = ''; + switch (appId) { + case 'pp_fit': + tag = this.templateForOptionPage(actions['fit']); + break; + case 'pp_predict': + tag = this.templateForOptionPage(actions['predict']); + break; + case 'pp_transform': + tag = this.templateForOptionPage(actions['transform']); + break; + case 'pp_fit_predict': + tag = this.templateForOptionPage(actions['fit_predict']); + break; + case 'pp_fit_transform': + tag = this.templateForOptionPage(actions['fit_transform']); + break; + case 'pp_fit_resample': + tag = this.templateForOptionPage(actions['fit_resample']); + break; + } + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(` +

${tag}

+ `); + } + + templateForOptionPage(packageObj) { + let optBox = new com_String(); + // render tag + packageObj && packageObj.options && packageObj.options.forEach(opt => { + let label = opt.name; + if (opt.label != undefined) { + label = opt.label; + } + // fix label + label = com_util.optionToLabel(label); + optBox.appendFormatLine('{2}' + , opt.name, opt.name, label); + let tmpState = {}; + if (opt.value && opt.value !== '') { + tmpState[opt.name] = opt.value; + } + let content = com_generator.renderContent(this, opt.component[0], opt, tmpState); + optBox.appendLine(content[0].outerHTML); + }); + return optBox.toString(); + } + + checkBeforeRun() { + let that = this; + var result = true; + for (let idx = 0; idx < this.state.pipeline.length; idx++) { + let ppObj = this.state.pipeline[idx]; + var { name, label, useApp, app } = ppObj; + let requiredList = []; + result = true; + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; + if (isVisible && isEnabled) { + switch (name) { + case 'ml_dataSplit': + requiredList = ['featureData', 'targetData']; + // check required data + for (let i = 0; i < requiredList.length; i++) { + let reqKey = requiredList[i]; + result = that._checkIsEmpty($(app.wrapSelector('#' + reqKey))); + if (result === false) { + // show page and focus it + $(that.wrapSelector(`.vp-pp-item[data-name="${name}"]`)).click(); + $(app.wrapSelector('#' + reqKey)).focus(); + break; + } + } + break; + case 'ml_gridSearch': + result = app.checkBeforeRun(); + if (result === false) { + // show page + $(that.wrapSelector(`.vp-pp-item[data-name="${name}"]`)).click(); + break; + } + break; + } + } + if (result === false) { + break; + } + } + return result; + + } + + _checkIsEmpty(tag) { + let requiredFilled = true; + // if it's empty, focus on it + if (tag && $(tag) && $(tag).val() == '') { + requiredFilled = false; + } + return requiredFilled; + } + + generateCodeForOptionPage(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let actObj = {}; + switch (appId) { + case 'pp_fit': + actObj = actions['fit']; + break; + case 'pp_predict': + actObj = actions['predict']; + break; + case 'pp_transform': + actObj = actions['transform']; + break; + case 'pp_fit_predict': + actObj = actions['fit_predict']; + break; + case 'pp_fit_transform': + actObj = actions['fit_transform']; + break; + case 'pp_fit_resample': + actObj = actions['fit_resample']; + break; + } + + let code = new com_String(); + if (actObj.import != undefined) { + code.appendLine(actObj.import); + code.appendLine(); + } + + let that = this; + let state = { + model: this.state.model + }; + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"] .vp-state`)).each((idx, tag) => { + let id = tag.id; + let value = that._getTagValue(tag); + state[id] = value; + }); + + let modelCode = com_generator.vp_codeGenerator(this, actObj, state); + modelCode = modelCode.replace('${model}', state.model); + code.append(modelCode); + return { state: state, code: code.toString() }; + } + + generateCode() { + let that = this; + let { template } = this.state; + let code = new com_String(); + + let stepNo = 1; + this.state.pipeline.forEach((ppObj, idx) => { + let { name, label, useApp, app } = ppObj; + + // check disabled + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; + if (isVisible && isEnabled) { + if (code.toString() !== '') { + code.appendLine(); + code.appendLine(); + } + if (useApp === true) { + let appCode = app.generateCode(); + if (appCode instanceof Array) { + appCode = appCode.join('\n'); + } + if (appCode && appCode.trim() !== '') { + code.appendFormatLine("# [{0}] {1}", stepNo++, label); + if (name === 'ml_evaluation') { + // import auto generate + code.appendLine(app.generateImportCode().join('\n')); + } + code.append(appCode); + } + // save state + that.state.pipeline[idx].state = app.state; + } else { + let ppResult = that.generateCodeForOptionPage(name); + if (ppResult && ppResult?.code?.trim() !== '') { + code.appendFormatLine("# [{0}] {1}", stepNo++, label); + code.append(ppResult.code); + } + // save state + that.state.pipeline[idx].state = ppResult.state; + } + } + }); + + return code.toString(); + } + + } + + return Pipeline; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/SaveLoad.js b/visualpython/js/m_ml/SaveLoad.js index 0940b028..454f468c 100644 --- a/visualpython/js/m_ml/SaveLoad.js +++ b/visualpython/js/m_ml/SaveLoad.js @@ -86,7 +86,7 @@ define([ // render tag config.options.forEach(opt => { optBox.appendFormatLine('{2}' - , opt.name, opt.name, opt.name); + , opt.name, opt.name, com_util.optionToLabel(opt.name)); let content = com_generator.renderContent(this, opt.component[0], opt, state); optBox.appendLine(content[0].outerHTML); }); diff --git a/visualpython/js/m_ml/dataSplit.js b/visualpython/js/m_ml/dataSplit.js index bfa2aac2..a8c9417f 100644 --- a/visualpython/js/m_ml/dataSplit.js +++ b/visualpython/js/m_ml/dataSplit.js @@ -175,7 +175,7 @@ define([ options.appendFormat(', shuffle={0}', shuffle); } if (shuffle != 'False' && stratify && stratify != '') { - options.appendFormat(', startify={0}', stratify); + options.appendFormat(', stratify={0}', stratify); } let code = new com_String(); diff --git a/visualpython/js/m_ml/evaluation.js b/visualpython/js/m_ml/evaluation.js index 6231b4a3..ed63120c 100644 --- a/visualpython/js/m_ml/evaluation.js +++ b/visualpython/js/m_ml/evaluation.js @@ -33,7 +33,7 @@ define([ this.config.checkModules = ['metrics']; this.state = { - modelType: 'rgs', + modelType: 'rgs', // rgs / clf / cls predictData: 'pred', targetData: 'y_test', // regression @@ -156,37 +156,11 @@ define([ }); $(page).find('#targetData2').replaceWith(targetData2Selector.toTagString()); - // load state - let that = this; - Object.keys(this.state).forEach(key => { - let tag = $(page).find('#' + key); - let tagName = $(tag).prop('tagName'); // returns with UpperCase - let value = that.state[key]; - if (value == undefined) { - return; - } - switch(tagName) { - case 'INPUT': - let inputType = $(tag).prop('type'); - if (inputType == 'text' || inputType == 'number' || inputType == 'hidden') { - $(tag).val(value); - break; - } - if (inputType == 'checkbox') { - $(tag).prop('checked', value); - break; - } - break; - case 'TEXTAREA': - case 'SELECT': - default: - $(tag).val(value); - break; - } - }); - $(page).find('.vp-upper-box').hide(); $(page).find('.vp-upper-box.' + this.state.modelType).show(); + + $(page).find('.vp-eval-box').hide(); + $(page).find('.vp-eval-' + this.state.modelType).show(); if (this.state.modelType == 'rgs') { // Regression @@ -228,15 +202,20 @@ define([ sizeOfClusters, silhouetteScore, ari, nmi, clusteredIndex, featureData2, targetData2 } = this.state; + // add import code for display and Markdown + let needDisplay = false; + let needMarkdown = false; //==================================================================== - // Classfication + // Classification //==================================================================== if (modelType == 'clf') { if (confusion_matrix) { code = new com_String(); code.appendLine("# Confusion Matrix"); - code.appendFormat('pd.crosstab({0}, {1}, margins=True)', targetData, predictData); + code.appendLine("display(Markdown('### Confusion Matrix'))"); + code.appendFormat('display(pd.crosstab({0}, {1}, margins=True))', targetData, predictData); + needMarkdown = true; codeCells.push(code.toString()); } if (report) { @@ -248,25 +227,25 @@ define([ if (accuracy) { code = new com_String(); code.appendLine("# Accuracy"); - code.appendFormat('metrics.accuracy_score({0}, {1})', targetData, predictData); + code.appendFormat("print('Accuracy: {}'.format(metrics.accuracy_score({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (precision) { code = new com_String(); code.appendLine("# Precision"); - code.appendFormat("metrics.precision_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('Precision: {}'.format(metrics.precision_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } if (recall) { code = new com_String(); code.appendLine("# Recall"); - code.appendFormat("metrics.recall_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('Recall: {}'.format(metrics.recall_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } if (f1_score) { code = new com_String(); code.appendLine("# F1-score"); - code.appendFormat("metrics.f1_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('F1-score: {}'.format(metrics.f1_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } // if (roc_curve) { @@ -300,14 +279,14 @@ define([ // } if (r_squared) { code = new com_String(); - code.appendLine("# R square"); - code.appendFormat('metrics.r2_score({0}, {1})', targetData, predictData); + code.appendLine("# R squared"); + code.appendFormat("print('R squared: {}'.format(metrics.r2_score({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (mae) { code = new com_String(); code.appendLine("# MAE(Mean Absolute Error)"); - code.appendFormat('metrics.mean_absolute_error({0}, {1})', targetData, predictData); + code.appendFormat("print('MAE: {}'.format(metrics.mean_absolute_error({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (mape) { @@ -316,22 +295,24 @@ define([ code.appendLine('def MAPE(y_test, y_pred):'); code.appendLine(' return np.mean(np.abs((y_test - pred) / y_test)) * 100'); code.appendLine(); - code.appendFormat('MAPE({0}, {1})', targetData, predictData); + code.appendFormat("print('MAPE: {}'.format(MAPE({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (rmse) { code = new com_String(); code.appendLine("# RMSE(Root Mean Squared Error)"); - code.appendFormat('metrics.mean_squared_error({0}, {1})**0.5', targetData, predictData); + code.appendFormat("print('RMSE: {}'.format(metrics.mean_squared_error({0}, {1})**0.5))", targetData, predictData); codeCells.push(code.toString()); } if (scatter_plot) { code = new com_String(); code.appendLine('# Regression plot'); + code.appendLine("display(Markdown('### Regression plot'))"); code.appendFormatLine('plt.scatter({0}, {1})', targetData, predictData); code.appendFormatLine("plt.xlabel('{0}')", targetData); code.appendFormatLine("plt.ylabel('{0}')", predictData); code.append('plt.show()'); + needMarkdown = true; codeCells.push(code.toString()); } } @@ -346,23 +327,34 @@ define([ if (silhouetteScore) { code = new com_String(); code.appendLine("# Silhouette score"); - code.appendFormat("print(f'Silhouette score: {metrics.cluster.silhouette_score({0}, {1})}')", featureData2, clusteredIndex); + code.appendFormat("print('Silhouette score: {}'.format(metrics.cluster.silhouette_score({0}, {1})))", featureData2, clusteredIndex); codeCells.push(code.toString()); } if (ari) { code = new com_String(); code.appendLine("# ARI(Adjusted Rand score)"); - code.appendFormat("print(f'ARI: {metrics.cluster.adjusted_rand_score({0}, {1})}')", targetData2, clusteredIndex); + code.appendFormat("print('ARI: {}'.format(metrics.cluster.adjusted_rand_score({0}, {1})))", targetData2, clusteredIndex); codeCells.push(code.toString()); } if (nmi) { code = new com_String(); code.appendLine("# NMI(Normalized Mutual Info Score)"); - code.appendFormat("print(f'NM: {metrics.cluster.normalized_mutual_info_score({0}, {1})}')", targetData2, clusteredIndex); + code.appendFormat("print('NM: {}'.format(metrics.cluster.normalized_mutual_info_score({0}, {1})))", targetData2, clusteredIndex); codeCells.push(code.toString()); } } - // return as seperated cells + if (needMarkdown === true) { + codeCells = [ + "from IPython.display import display, Markdown", + ...codeCells + ]; + } else if (needDisplay === true) { + codeCells = [ + "from IPython.display import display", + ...codeCells + ]; + } + // return as separated cells return codeCells; } diff --git a/visualpython/js/m_stats/Anova.js b/visualpython/js/m_stats/Anova.js index 66ad16ce..4d6f901f 100644 --- a/visualpython/js/m_stats/Anova.js +++ b/visualpython/js/m_stats/Anova.js @@ -31,7 +31,7 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.sizeLevel = 2; - this.config.checkModules = ['pd', 'np', 'scipy.stats', 'vp_confidence_interval', 'vp_sem']; + this.config.checkModules = ['pd', 'np', 'stats', 'vp_confidence_interval', 'vp_sem']; this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; this.state = { @@ -60,6 +60,8 @@ define([ this.columnBindList = ['depVar', 'factor', 'factorA', 'factorB', 'covariate']; + this.tmpInstallCode = []; // install codes + this.subsetEditor = {}; } @@ -74,6 +76,29 @@ define([ $(that.wrapSelector('.vp-st-option')).hide(); $(that.wrapSelector('.vp-st-option.' + testType)).show(); + + that.tmpInstallCode = []; + that.hideInstallButton(); + + if (testType === 'one-way' || testType === 'two-way') { + if (that.state.tukey || that.state.scheffe || that.state.duncan) { + // Add installation code + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install scikit-posthocs"]; + } else { + that.tmpInstallCode = ["!pip install scikit-posthocs"]; + } + that.showInstallButton(); + } + } else if (testType === 'ancova') { + // Add installation code : # pip install pingouin + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install pingouin"]; + } else { + that.tmpInstallCode = ["!pip install pingouin"]; + } + that.showInstallButton(); + } }); $(this.wrapSelector('#data')).on('change', function() { @@ -91,6 +116,28 @@ define([ com_generator.vp_bindColumnSource(that, 'data', that.columnBindList, 'select', false, false); } }); + + $(this.wrapSelector('.vp-st-posthoc-box .vp-state')).on('change', function() { + let id = $(this)[0].id; + let checked = $(this).prop('checked') === true; + that.state[id] = checked; + let { testType, tukey, scheffe, duncan } = that.state; + if (testType === 'one-way' || testType === 'two-way') { + if (tukey || scheffe || duncan) { + // Add installation code + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install scikit-posthocs"]; + } else { + that.tmpInstallCode = ["!pip install scikit-posthocs"]; + } + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + } + }); + + $(this.wrapSelector('')) } templateForBody() { @@ -147,6 +194,10 @@ define([ $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); } + generateInstallCode() { + return this.tmpInstallCode; + } + generateCode() { let { testType, data, depVar, factor, factorA, factorB, covariate, sigLevel, @@ -253,12 +304,6 @@ define([ } if (tukey === true || scheffe === true || duncan === true) { - if (vpConfig.extensionType === 'lite') { - codeList.push("%pip install scikit-posthocs"); - } else { - codeList.push("!pip install scikit-posthocs"); - } - // Post hoc analysis - Tukey if (tukey === true) { code.appendLine(); @@ -383,13 +428,6 @@ define([ code.append("display(_res[0])"); } if (tukey === true || scheffe === true || duncan === true) { - // Add installation code - if (vpConfig.extensionType === 'lite') { - codeList.push("%pip install scikit-posthocs"); - } else { - codeList.push("!pip install scikit-posthocs"); - } - // Post hoc analysis - Tukey if (tukey === true) { code.appendLine(); @@ -453,13 +491,6 @@ define([ code.append(" plt.show()"); } - // Add installation code : # pip install pingouin - if (vpConfig.extensionType === 'lite') { - codeList.push("%pip install pingouin"); - } else { - codeList.push("!pip install pingouin"); - } - code.appendLine(); code.appendLine(); code.appendLine("# ANCOVA - Analysis of covariance"); diff --git a/visualpython/js/m_stats/FactorAnalysis.js b/visualpython/js/m_stats/FactorAnalysis.js index 6d5ca900..188efe19 100644 --- a/visualpython/js/m_stats/FactorAnalysis.js +++ b/visualpython/js/m_stats/FactorAnalysis.js @@ -160,7 +160,12 @@ define([ } generateInstallCode() { - return [ '!pip install factor-analyzer']; + let installCode = '!pip install factor-analyzer'; + // Add installation code + if (vpConfig.extensionType === 'lite') { + installCode = '%pip install factor-analyzer'; + } + return [ installCode ]; } generateCode() { @@ -168,13 +173,6 @@ define([ let codeList = []; let code = new com_String(); - // Add installation code FIXME: - if (vpConfig.extensionType === 'lite') { - codeList.push('%pip install factor-analyzer'); - } else { - codeList.push('!pip install factor-analyzer'); - } - // data declaration code.appendFormat("vp_df = {0}", data); if (this.columnSelector) { diff --git a/visualpython/js/m_stats/ProbDist.js b/visualpython/js/m_stats/ProbDist.js index 387c59b1..245cd9f6 100644 --- a/visualpython/js/m_stats/ProbDist.js +++ b/visualpython/js/m_stats/ProbDist.js @@ -34,6 +34,7 @@ define([ this.config.sizeLevel = 2; this.config.checkModules = ['pd', 'plt']; this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + this.config.helpview = true; this.state = { distType: 'normal', @@ -85,22 +86,44 @@ define([ // discrete option $(that.wrapSelector('.vp-pd-display-option.dist')).show(); - // hide continuous action - if (that.state.action === 'stats-to-pvalue' || that.state.action === 'pvalue-to-stats') { - $(that.wrapSelector('#action')).val('random-number'); - $(that.wrapSelector('#action')).trigger('change'); + // set size to 1000 + $(that.wrapSelector('#size')).val(1000); + that.state.size = 1000; + + // hide distribution plot for multinomial + if (distType === 'multinomial') { + $(that.wrapSelector('.vp-pd-display-option.dist-plot')).hide(); + // hide other actions + if (that.state.action !== 'random-number') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } + } else { + // hide continuous action + if (that.state.action === 'stats-to-pvalue' || that.state.action === 'pvalue-to-stats') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } } } else { // continuous option $(that.wrapSelector('.vp-pd-display-option.cont')).show(); + + // set size to 10000 + $(that.wrapSelector('#size')).val(10000); + that.state.size = 10000; } // show install button - if (STATS_LIBRARIES[distType].install != undefined) { + let thisDistObj = STATS_LIBRARIES[distType]; + if (thisDistObj.install != undefined) { $(that.wrapSelector('#vp_installLibrary')).show(); } else { $(that.wrapSelector('#vp_installLibrary')).hide(); } + + // set help content + that.setHelpContent(thisDistObj.help); }); $(this.wrapSelector('#action')).on('change', function() { @@ -198,6 +221,8 @@ define([ } }); $(this.wrapSelector('#allocatedTo')).replaceWith(allocateSelector.toTagString()); + + this.setHelpContent(STATS_LIBRARIES[this.state.distType].help, true, 'import scipy.stats as _vp_stats'); } generateCode() { @@ -248,10 +273,25 @@ define([ code.appendLine("import warnings"); code.appendLine("with warnings.catch_warnings():"); code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); - code.appendFormatLine(" sns.histplot({0}, stat='density', kde=True)", allocateTo); - code.appendFormatLine(" plt.title('Generate random numbers: {0}')", label.replace("'", "\\'")); - code.appendLine(" plt.xlabel('$x$')"); - code.append(" plt.show()"); + if (distType === 'multinomial') { + // code.appendFormatLine(" plt.boxplot(x={0})", allocateTo); + code.appendFormatLine(" for i in range(0, {0}.shape[1]):", allocateTo); + code.appendLine(" plt.subplot(2, 2, i+1)"); + code.appendLine(" plt.title('$x$=' + str(i))"); + code.appendFormatLine(" sns.countplot(x=[ x[i] for x in {0} ])", allocateTo); + code.appendLine(" plt.suptitle('Generate random numbers: Multinomial')"); + code.appendLine(" plt.tight_layout()"); + code.appendLine(" plt.show()"); + } else { + if (this.distList[0].child.includes(distType)) { + code.appendFormatLine(" sns.countplot(x={0})", allocateTo); + } else { + code.appendFormatLine(" sns.histplot({0}, stat='density', kde=True)", allocateTo); + } + code.appendFormatLine(" plt.title('Generate random numbers: {0}')", label.replace("'", "\\'")); + code.appendLine(" plt.xlabel('$x$')"); + code.append(" plt.show()"); + } } break; case 'distribution-plot': @@ -263,19 +303,64 @@ define([ code.appendLine(); code.appendLine(); code.appendFormatLine("# Probability mass function ({0})", label); - code.appendLine("plt.bar([0,1], _rv.pmf([0,1]))"); - code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); - code.appendLine("plt.xlim(-1, 2)"); - code.appendLine("plt.ylim(0, 1)"); - code.appendLine("plt.xticks([0, 1], ['x=0', 'x=1'])"); - code.appendLine("plt.xlabel('$x$')"); - code.appendLine("plt.ylabel('$p(x)$')"); - code.append("plt.show()"); + if (distType === 'bernoulli') { + code.appendLine("plt.bar([0,1], _rv.pmf([0,1]))"); + code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); + code.appendLine("plt.xlim(-1, 2)"); + code.appendLine("plt.ylim(0, 1)"); + code.appendLine("plt.xticks([0, 1], ['x=0', 'x=1'])"); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } else if (distType === 'binomial') { + var { n=10 } = this.state; + code.appendFormatLine("plt.bar(range(0,{0}), _rv.pmf(range(0,{1})))", n, n); + code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); + code.appendFormatLine("plt.xlim(-1, {0})", n); + code.appendFormatLine("plt.xticks(range(0, {0}), ['x='+str(i) for i in range(0, {1})])", n, n); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } else if (distType === 'multinomial') { + code.appendFormatLine("for i in range(0, {0}.shape[1]):", allocateTo); + code.appendLine(" plt.subplot(2, 2, i+1)"); + code.appendLine(" plt.title('$x$=' + str(i))"); + code.appendFormatLine(" sns.countplot(x=[ x[i] for x in {0} ])", allocateTo); + code.appendLine("plt.suptitle('Probability mass function: Multinomial')"); + code.appendLine("plt.tight_layout()"); + code.append("plt.show()"); + } } } else { + let start = -5; + let end = 5; + switch (distType) { + case 'normal': + case 'studentst': + case 'multivariate_normal': + start = -5; + end = 5; + break; + case 'uniform': + case 'beta': + case 'dirichlet': + start = 0; + end = 1; + break; + case 'gamma': + case 'chi2': + start = 0; + end = 30; + break; + case 'f': + start = 0; + end = 10; + break; + } + if (probDensityFunc === true || cumDistFunc === true) { code.appendLine(); - code.append("x = np.linspace(-5, 5, 100)"); + code.appendFormat("x = np.linspace({0}, {1}, 100)", start, end); if (probDensityFunc === true) { this.addCheckModules('np'); this.addCheckModules('plt'); @@ -298,7 +383,7 @@ define([ code.appendFormatLine("# Cumulative distribution function ({0})", label); code.appendLine("import warnings"); code.appendLine("with warnings.catch_warnings():"); - code.appendLine(" _x = np.linspace(-5, 5, 100)"); + code.appendFormatLine(" _x = np.linspace({0}, {1}, 100)", start, end); code.appendLine(" plt.plot(_x, _rv.cdf(_x))"); code.appendLine(); code.appendFormatLine(" plt.title('Cumulative distribution function: {0}')", label.replace("'", "\\'")); diff --git a/visualpython/js/m_stats/Regression.js b/visualpython/js/m_stats/Regression.js index 68b2c49d..a38c811b 100644 --- a/visualpython/js/m_stats/Regression.js +++ b/visualpython/js/m_stats/Regression.js @@ -58,6 +58,7 @@ define([ normTest: true, histogram: true, scatterplot: true, + rmse: false, ...this.state }; @@ -259,7 +260,7 @@ define([ // Multi-collinearity multiCollinearity, // Residual option - statistics, normTest, histogram, scatterplot + statistics, normTest, histogram, scatterplot, rmse, } = this.state; let codeList = []; let code = new com_String(); @@ -325,7 +326,7 @@ define([ code.append("print(_result.summary())"); } else if (method === 'stepwise') { // Inner function : vp_stepwise_select - this.addCheckModules('statsmodels.api'); + this.addCheckModules('sm'); this.addCheckModules('vp_stepwise_select'); code.appendFormatLine("_selected_stepwise = vp_stepwise_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); @@ -373,7 +374,7 @@ define([ lastModelNum = 2; } else if (method === 'backward') { // Inner function : vp_backward_select - this.addCheckModules('statsmodels.api'); + this.addCheckModules('sm'); this.addCheckModules('vp_backward_select'); code.appendFormatLine("_selected_backward = vp_backward_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); @@ -421,7 +422,7 @@ define([ lastModelNum = 2; } else if (method === 'forward') { // Inner function : vp_forward_select - this.addCheckModules('statsmodels.api'); + this.addCheckModules('sm'); this.addCheckModules('vp_forward_select'); code.appendFormatLine("_selected_forward = vp_forward_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); @@ -508,7 +509,7 @@ define([ code.appendLine("# Mean Centering "); independentValue = com_util.formatString("{0}_MC", independentValue); moderatedValue = com_util.formatString("{0}_MC", moderatedValue); - code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", independentValue, independent, independent); + code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", independentValue, independent, independent); code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", moderatedValue, moderated, moderated); } // Model 1 to 3 @@ -578,7 +579,7 @@ define([ case 'mediated': // 5. Mediated if (sobelTest === true) { - this.addCheckModules('scipy.stats'); + this.addCheckModules('stats'); this.addCheckModules('vp_sobel'); } code.appendLine(); @@ -697,7 +698,7 @@ define([ } // Residual option - if (statistics === true || normTest === true || histogram === true || scatterplot === true) { + if (statistics === true || normTest === true || histogram === true || scatterplot === true || rmse === true) { let residualTitle = 'Residual' if (lastModelNum > 0) { residualTitle += ' - Model ' + lastModelNum; @@ -768,6 +769,14 @@ define([ code.appendLine(" plt.tight_layout()"); code.append(" plt.show()"); } + if (rmse === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# RMSE (Root Mean Squared Error)"); + code.appendLine("_rmse = np.sqrt(_result.mse_resid)"); + code.appendLine("display(Markdown('### RMSE (Root Mean Squared Error)'))"); + code.append("display(Markdown(f'RMSE: {_rmse}'))"); + } } codeList.push(code.toString()); diff --git a/visualpython/js/m_stats/StudentstTest.js b/visualpython/js/m_stats/StudentstTest.js index fc0c7341..d9b5b1a4 100644 --- a/visualpython/js/m_stats/StudentstTest.js +++ b/visualpython/js/m_stats/StudentstTest.js @@ -20,8 +20,9 @@ define([ 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/SuggestInput', 'vp_base/js/m_apps/Subset' -], function(stHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { +], function(stHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, SuggestInput, Subset) { /** * StudentstTest @@ -46,6 +47,8 @@ define([ groupingVariable: '', group1: '', group2: '', + group1_istext: true, + group2_istext: true, pairedVariable1: '', pairedVariable2: '', testValue: '', @@ -123,27 +126,61 @@ define([ var colName = $(this).find('option:selected').text(); var colDtype = $(this).find('option:selected').attr('data-type'); that.state.groupingVariable = colCode; - $(that.wrapSelector('#group1')).html(''); - $(that.wrapSelector('#group2')).html(''); // get result and load column list vpKernel.getColumnCategory(that.state.data, colCode).then(function(resultObj) { let { result } = resultObj; + $(that.wrapSelector('#group1')).val(''); + $(that.wrapSelector('#group2')).val(''); + that.state.group1 = ''; + that.state.group2 = ''; + that.state.group1_istext = true; + that.state.group2_istext = true; try { var category = JSON.parse(result); - if (category && category.length > 0 && colDtype == 'object') { - // if it's categorical column and its dtype is object, check 'Text' as default - category.forEach(obj => { - let selected1 = obj.value === that.state.group1; - let selected2 = obj.value === that.state.group1; - $(that.wrapSelector('#group1')).append(``); - $(that.wrapSelector('#group2')).append(``); - }); + // if (category && category.length > 0 && colDtype == 'object') { + // // if it's categorical column and its dtype is object, check 'Text' as default + // category.forEach(obj => { + // let selected1 = obj.value === that.state.group1; + // let selected2 = obj.value === that.state.group2; + // $(that.wrapSelector('#group1')).append(``); + // $(that.wrapSelector('#group2')).append(``); + // }); + // } + var groupSuggest1 = new SuggestInput(); + groupSuggest1.setComponentID('group1'); + groupSuggest1.addClass('vp-input vp-state'); + groupSuggest1.setSuggestList(function() { return category; }); + groupSuggest1.setNormalFilter(true); + groupSuggest1.setPlaceholder('Select value'); + $(that.wrapSelector('#group1')).replaceWith(groupSuggest1.toTagString()); + var groupSuggest2 = new SuggestInput(); + groupSuggest2.setComponentID('group2'); + groupSuggest2.addClass('vp-input vp-state'); + groupSuggest2.setSuggestList(function() { return category; }); + groupSuggest2.setNormalFilter(true); + groupSuggest2.setPlaceholder('Select value'); + $(that.wrapSelector('#group2')).replaceWith(groupSuggest2.toTagString()); + + if (category && category.length > 0) { that.state.group1 = category[0].value; that.state.group2 = category[0].value; } + + if (colDtype == 'object') { + // check as default + $(that.wrapSelector('#group1_istext')).prop('checked', true); + $(that.wrapSelector('#group2_istext')).prop('checked', true); + that.state.group1_istext = true; + that.state.group2_istext = true; + } else { + $(that.wrapSelector('#group1_istext')).prop('checked', false); + $(that.wrapSelector('#group2_istext')).prop('checked', false); + that.state.group1_istext = false; + that.state.group2_istext = false; + } } catch { - $(that.wrapSelector('#group1')).html(''); - $(that.wrapSelector('#group2')).html(''); + $(that.wrapSelector('#group1')).val(''); + $(that.wrapSelector('#group2')).val(''); } }); }); @@ -231,7 +268,7 @@ define([ testType, inputType, data, testVariable, testVariable1, testVariable2, groupingVariable, pairedVariable1, pairedVariable2, - group1, group2, + group1, group2, group1_istext, group2_istext, testValue, alterHypo, confInt } = this.state; let codeList = []; @@ -276,8 +313,8 @@ define([ code.appendLine("# Independent two-sample t-test"); // variable declaration if (inputType === 'long-data') { - code.appendFormatLine("vp_df1 = {0}[({1}[{2}] == '{3}')][{4}].dropna().copy()", data, data, groupingVariable, group1, testVariable); - code.appendFormatLine("vp_df2 = {0}[({1}[{2}] == '{3}')][{4}].dropna().copy()", data, data, groupingVariable, group2, testVariable); + code.appendFormatLine("vp_df1 = {0}[({1}[{2}] == {3})][{4}].dropna().copy()", data, data, groupingVariable, com_util.convertToStr(group1, group1_istext), testVariable); + code.appendFormatLine("vp_df2 = {0}[({1}[{2}] == {3})][{4}].dropna().copy()", data, data, groupingVariable, com_util.convertToStr(group2, group2_istext), testVariable); } else if (inputType === 'wide-data') { code.appendFormatLine("vp_df1 = {0}[{1}].dropna().copy()", data, testVariable1); code.appendFormatLine("vp_df2 = {0}[{1}].dropna().copy()", data, testVariable2); diff --git a/visualpython/js/m_visualize/Chart.js b/visualpython/js/m_visualize/Chart.js index f5336bb9..d979a6e1 100644 --- a/visualpython/js/m_visualize/Chart.js +++ b/visualpython/js/m_visualize/Chart.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/com_generator', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_Const, com_util, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { +], function(chartHTml, chartCss, com_String, com_Const, com_util, com_interface, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { /** * Chart @@ -268,14 +269,16 @@ define([ $(this.wrapSelector('#vp_plImportRun')).click(function() { // generateImportCode var code = that.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Chart'); }); } diff --git a/visualpython/js/m_visualize/Plotly.js b/visualpython/js/m_visualize/Plotly.js index 9136883f..684840b3 100644 --- a/visualpython/js/m_visualize/Plotly.js +++ b/visualpython/js/m_visualize/Plotly.js @@ -451,20 +451,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_ptPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_ptPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } diff --git a/visualpython/js/m_visualize/Seaborn.js b/visualpython/js/m_visualize/Seaborn.js index b153c3b0..bea6cfd1 100644 --- a/visualpython/js/m_visualize/Seaborn.js +++ b/visualpython/js/m_visualize/Seaborn.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/data/m_visualize/seabornLibrary', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_generator, com_util, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { +], function(chartHTml, chartCss, com_String, com_generator, com_util, com_interface, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { class Seaborn extends PopupComponent { _init() { @@ -31,6 +32,7 @@ define([ this.config.dataview = false; this.config.size = { width: 1064, height: 550 }; + this.config.autoScroll = false; this.config.checkModules = ['plt', 'sns']; this.config.docs = 'https://seaborn.pydata.org/index.html'; @@ -52,6 +54,7 @@ define([ stat: '', showValues: false, showValuesPrecision: '', + errorbar: '', sortBy: 'y', sortType: '', sortHue: '', @@ -199,7 +202,9 @@ define([ $(that.wrapSelector('#kde')).closest('.sb-option').show(); $(that.wrapSelector('#stat')).closest('.sb-option').show(); } else if (chartType == 'barplot') { + $(that.wrapSelector('#orient')).closest('.sb-option').show(); $(that.wrapSelector('#showValues')).closest('.sb-option').show(); + $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); if (that.state.setXY === false) { if (that.state.x !== '' && that.state.y !== '') { $(that.wrapSelector('#sortBy')).closest('.sb-option').show(); @@ -220,6 +225,8 @@ define([ } } else if (chartType == 'heatmap') { $(that.wrapSelector('#annot')).closest('.sb-option').show(); + } else if (chartType === 'lineplot') { + $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); } }); @@ -361,29 +368,33 @@ define([ let val = $(this).val(); if (val !== '') { // enable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', false); + $(that.wrapSelector('#xticks_label')).attr('readonly', false); } else { // disable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', true); + $(that.wrapSelector('#xticks_label')).attr('readonly', true); } }); $(this.wrapSelector('#yticks')).on('change', function() { let val = $(this).val(); if (val !== '') { // enable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', false); + $(that.wrapSelector('#yticks_label')).attr('readonly', false); } else { // disable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', true); + $(that.wrapSelector('#yticks_label')).attr('readonly', true); } }); // axes - ticks label: inform user to type location option to use label - $(this.wrapSelector('#xticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#xticks')).focus(); + $(this.wrapSelector('#xticks_label')).on('click', function() { + if ($(that.wrapSelector('#xticks')).val() === '') { + $(that.wrapSelector('#xticks')).focus(); + } }); - $(this.wrapSelector('#yticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#yticks')).focus(); + $(this.wrapSelector('#yticks_label')).on('click', function() { + if ($(that.wrapSelector('#yticks')).val() === '') { + $(that.wrapSelector('#yticks')).focus(); + } }); // preview refresh @@ -522,6 +533,15 @@ define([ }); $(page).find('#sampleCount').html(sampleCountTag.toString()); + // set errorbar list + var vpErrorbarSuggest = new SuggestInput(); + vpErrorbarSuggest.setComponentID('errorbar'); + vpErrorbarSuggest.addClass('vp-input vp-state'); + vpErrorbarSuggest.setPlaceholder("('ci', 95)"); + vpErrorbarSuggest.setValue(this.state.errorbar); + vpErrorbarSuggest.setSuggestList(["None", "'ci'", "'pi'", "'sd'", "'se'"]); + $(page).find('#errorbar').replaceWith(vpErrorbarSuggest.toTagString()); + // data options depend on chart type $(page).find('.sb-option').hide(); if (this.state.chartType == 'histplot') { @@ -529,7 +549,9 @@ define([ $(page).find('#kde').closest('.sb-option').show(); $(page).find('#stat').closest('.sb-option').show(); } else if (this.state.chartType == 'barplot') { + $(page).find('#orient').closest('.sb-option').show(); $(page).find('#showValues').closest('.sb-option').show(); + $(page).find('#errorbar').closest('.sb-option').show(); if (this.state.setXY === false) { if (this.state.x !== '' && this.state.y !== '') { $(page).find('#sortBy').closest('.sb-option').show(); @@ -550,6 +572,8 @@ define([ } } else if (this.state.chartType == 'heatmap') { $(page).find('#annot').closest('.sb-option').show(); + } else if (this.state.chartType === 'lineplot') { + $(page).find('#errorbar').closest('.sb-option').show(); } //================================================================ @@ -766,14 +790,16 @@ define([ handleInnerOk() { // generateImportCode var code = this.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Seaborn'); this.closeInnerPopup(); @@ -810,20 +836,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#chartPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#chartPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } @@ -867,8 +893,8 @@ define([ generateCode(preview=false) { let { - chartType, data, x, y, setXY, hue, kde, stat, - showValues, showValuesPrecision, + chartType, data, x, y, setXY, hue, orient, kde, stat, + showValues, showValuesPrecision, errorbar, sortType, sortBy, sortHue, sortHueText, userOption='', x_limit_from, x_limit_to, y_limit_from, y_limit_to, @@ -933,6 +959,12 @@ define([ if (showValues === true && chartType === 'barplot') { // etcOptionCode.push('ci=None'); // changed to errorbar after 0.12 version etcOptionCode.push('errorbar=None'); + } else { + if (chartType === 'barplot' || chartType === 'lineplot') { + if (errorbar !== '') { + etcOptionCode.push(com_util.formatString("errorbar={0}", errorbar)); + } + } } if (setXY === false && sortType !== '') { let sortCode = ''; diff --git a/visualpython/js/m_visualize/WordCloud.js b/visualpython/js/m_visualize/WordCloud.js index 005e306b..96ee22a1 100644 --- a/visualpython/js/m_visualize/WordCloud.js +++ b/visualpython/js/m_visualize/WordCloud.js @@ -265,20 +265,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_wcPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_wcPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } diff --git a/visualpython/js/menu/MenuFrame.js b/visualpython/js/menu/MenuFrame.js index 05813295..386916a3 100644 --- a/visualpython/js/menu/MenuFrame.js +++ b/visualpython/js/menu/MenuFrame.js @@ -205,19 +205,19 @@ define([ return {}; } - getMenuLibrary(menuId, libraries=this.menuLibraries) { + getMenuLibrary(menuId, libraries=this.menuLibrariesFlatten) { for (var i=0; i < libraries.length; i++) { var item = libraries[i]; if (item) { if (item.id === menuId) { return item; } - if (item.type === 'package') { - var result = this.getMenuLibrary(menuId, item.item); - if (result) { - return result; - } - } + // if (item.type === 'package') { + // var result = this.getMenuLibrary(menuId, item.item); + // if (result) { + // return result; + // } + // } } } return null; @@ -252,7 +252,12 @@ define([ var that = this; var body = group.getBody(); var item = group.getItem(); + var state = group.getState(); item && item.forEach(child => { + // remember parent to navigate its parent menu + var category = state.category?(state.category + ' > ' + state.name):state.name; + child['category'] = category; + if (child.type == 'package') { // packages : MenuGroup var menuGroup = new MenuGroup($(body), child); @@ -298,7 +303,7 @@ define([ }); let functionList = this.menuLibrariesFlatten.map(menu => { - return { label: menu.name, value: menu.name, ...menu } + return { label: menu.name, value: menu.name, dtype: menu.category, ...menu } }); // render searchbox let searchBox = new SuggestInput(); diff --git a/visualpython/js/menu/TaskItem.js b/visualpython/js/menu/TaskItem.js index dcb63181..c8e2be64 100644 --- a/visualpython/js/menu/TaskItem.js +++ b/visualpython/js/menu/TaskItem.js @@ -39,10 +39,7 @@ define([ if (isOpen) { // hide task if it's already opened // open task - $('#vp_wrapper').trigger({ - type: 'close_option_page', - component: that.state.task - }); + that.state.task.hide(); } else { // open task $('#vp_wrapper').trigger({ diff --git a/visualpython/lib/codemirror/lib/codemirror.css b/visualpython/lib/codemirror/lib/codemirror.css index f4d5718a..0a80f68d 100644 --- a/visualpython/lib/codemirror/lib/codemirror.css +++ b/visualpython/lib/codemirror/lib/codemirror.css @@ -3,7 +3,7 @@ .CodeMirror { /* Set height, width, borders, and global font properties here */ font-family: monospace; - height: 300px; + /* height: 300px; */ color: black; direction: ltr; } diff --git a/visualpython/python/pandasCommand.py b/visualpython/python/pandasCommand.py index aee1338d..4f8301c6 100644 --- a/visualpython/python/pandasCommand.py +++ b/visualpython/python/pandasCommand.py @@ -11,7 +11,7 @@ # from IPython.core.display is deprecated since IPython 7.14 from IPython.display import display -def _vp_get_rows_list(df): +def _vp_get_rows_list(df, start_idx=0): """ Get Rows List with Detail Information """ @@ -19,7 +19,8 @@ def _vp_get_rows_list(df): indexType = str(df.index.dtype) # make dict for rows info for i, r in enumerate(df.index): - rInfo = { 'label': r, 'value': r, 'location': i } + rInfo = { 'label': r, 'value': r, 'location': start_idx + i } + # value if type(r).__name__ == 'str': rInfo['value'] = "'{}'".format(r) diff --git a/visualpython/python/userCommand.py b/visualpython/python/userCommand.py index 586eea36..78034078 100644 --- a/visualpython/python/userCommand.py +++ b/visualpython/python/userCommand.py @@ -100,7 +100,7 @@ def vp_create_feature_importances(model, X_train=None, sort=False): feature_names = [ 'X{}'.format(i) for i in range(len(model.feature_importances_)) ] df_i = _vp_pd.DataFrame(model.feature_importances_, index=feature_names, columns=['Feature_importance']) - df_i['Percentage'] = 100 * (df_i['Feature_importance'] / df_i['Feature_importance'].max()) + df_i['Percentage'] = 100 * df_i['Feature_importance'] if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) df_i = df_i.round(2) @@ -123,6 +123,41 @@ def vp_plot_feature_importances(model, X_train=None, sort=False, top_count=0): _vp_plt.show() ###### +# Visual Python: Machine Learning > Model Info +###### +def vp_create_permutation_importances(model, X_train, y_train, scoring=None, sort=False): + from sklearn.inspection import permutation_importance + if isinstance(X_train, _vp_pd.core.frame.DataFrame): + feature_names = X_train.columns + else: + feature_names = [ 'X{}'.format(i) for i in range(X_train.shape[1]) ] + + imp = permutation_importance(model, X_train, y_train, scoring=scoring) + + df_i = _vp_pd.DataFrame(imp['importances_mean'], index=feature_names, columns=['Feature_importance']) + df_i['Percentage'] = 100 * df_i['Feature_importance'] + if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) + df_i = df_i.round(2) + + return df_i +###### +# Visual Python: Machine Learning > Model Info +###### +def vp_plot_permutation_importances(model, X_train, y_train, scoring=None, sort=False, top_count=0): + df_i = vp_create_permutation_importances(model, X_train, y_train, scoring=scoring, sort=sort) + + if sort: + if top_count > 0: + df_i['Percentage'].sort_values().tail(top_count).plot(kind='barh') + else: + df_i['Percentage'].sort_values().plot(kind='barh') + else: + df_i['Percentage'].plot(kind='barh') + _vp_plt.xlabel('Feature importance Percentage') + _vp_plt.ylabel('Features') + + _vp_plt.show() +###### # Visual Python: Visualization > Seaborn ###### def vp_seaborn_show_values(axs, precision=1, space=0.01): diff --git a/visualpython/python/variableCommand.py b/visualpython/python/variableCommand.py index 6069dda9..6a0df33c 100644 --- a/visualpython/python/variableCommand.py +++ b/visualpython/python/variableCommand.py @@ -132,11 +132,15 @@ def _vp_check_package_list(pack_list): Check package info : name, version, path """ import importlib as _vp_ilib + import warnings _pack_info = {} - for pack in pack_list: - try: - _vp_pack = _vp_ilib.import_module(pack) - _pack_info[pack] = { 'name': _vp_pack.__name__, 'installed': True, 'version': _vp_pack.__version__, 'path': _vp_pack.__path__ } - except: - _pack_info[pack] = { 'name': pack, 'installed': False } + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + for pack in pack_list: + try: + _vp_pack = _vp_ilib.import_module(pack) + _vp_ilib.reload(_vp_pack) + _pack_info[pack] = { 'name': _vp_pack.__name__, 'installed': True, 'version': _vp_pack.__version__, 'path': _vp_pack.__path__ } + except: + _pack_info[pack] = { 'name': pack, 'installed': False } return _pack_info \ No newline at end of file diff --git a/visualpython/visualpython.yaml b/visualpython/visualpython.yaml index 11ae71f3..a4e3d5bb 100644 --- a/visualpython/visualpython.yaml +++ b/visualpython/visualpython.yaml @@ -5,28 +5,4 @@ Link: README.md Icon: img/logo.png Main: visualpython.js Compatibility: 4.x, 5.x, 6.x -Parameters: -- name: vpcfg.run_code_without_asking - description: Run code without asking - input_type: checkbox - default: false -- name: vpcfg.change_task_without_asking - description: Change task without asking - input_type: checkbox - default: false -- name: vpcfg.code_insert_position - description: Code insert position - input_type: text - default: below -- name: vpcfg.api_list_sort_by - description: Api list sorting by... - input_type: text - default: frequency -- name: vpcfg.default_variable_for_required - description: Default variable for required input - input_type: checkbox - default: false -- name: vpcfg.auto_import_package - description: Automatically import packages - input_type: checkbox - default: false +Parameters: \ No newline at end of file