	sepal_length	sepal_width	petal_length	petal_width	variety
0	5.1	3.5	1.4	0.2	Setosa
1	4.9	3.0	1.4	0.2	Setosa
2	4.7	3.2	1.3	0.2	Setosa
3	4.6	3.1	1.5	0.2	Setosa
4	5.0	3.6	1.4	0.2	Setosa
...	...	...	...	...	...
145	6.7	3.0	5.2	2.3	Virginica
146	6.3	2.5	5.0	1.9	Virginica
147	6.5	3.0	5.2	2.0	Virginica
148	6.2	3.4	5.4	2.3	Virginica
149	5.9	3.0	5.1	1.8	Virginica

sepal_length

sepal_width

petal_length

petal_width

variety

5.1

3.5

1.4

0.2

Setosa

4.9

3.0

1.4

0.2

Setosa

4.7

3.2

1.3

0.2

Setosa

4.6

3.1

1.5

0.2

Setosa

5.0

3.6

1.4

0.2

Setosa

...

145

6.7

3.0

5.2

2.3

Virginica

146

6.3

2.5

5.0

1.9

Virginica

147

6.5

3.0

5.2

2.0

Virginica

148

6.2

3.4

5.4

2.3

Virginica

149

5.9

3.0

5.1

1.8

Virginica

	variety	sepal_length	sepal_width	petal_length	petal_width
0	Setosa	5.1	3.5	1.4	0.2
1	Setosa	4.9	3.0	1.4	0.2
2	Setosa	4.7	3.2	1.3	0.2
3	Setosa	4.6	3.1	1.5	0.2
4	Setosa	5.0	3.6	1.4	0.2
...	...	...	...	...	...
145	Virginica	6.7	3.0	5.2	2.3
146	Virginica	6.3	2.5	5.0	1.9
147	Virginica	6.5	3.0	5.2	2.0
148	Virginica	6.2	3.4	5.4	2.3
149	Virginica	5.9	3.0	5.1	1.8

variety

sepal_length

sepal_width

petal_length

petal_width

Setosa

5.1

3.5

1.4

0.2

Setosa

4.9

3.0

1.4

0.2

Setosa

4.7

3.2

1.3

0.2

Setosa

4.6

3.1

1.5

0.2

Setosa

5.0

3.6

1.4

0.2

...

145

Virginica

6.7

3.0

5.2

2.3

146

Virginica

6.3

2.5

5.0

1.9

147

Virginica

6.5

3.0

5.2

2.0

148

Virginica

6.2

3.4

5.4

2.3

149

Virginica

5.9

3.0

5.1

1.8

Type Data

@@ -22,7 +22,7 @@

Option Page -

diff --git a/visualpython/html/component/fileNavigation.html b/visualpython/html/component/fileNavigation.html index ea2b2369..e02507b4 100644 --- a/visualpython/html/component/fileNavigation.html +++ b/visualpython/html/component/fileNavigation.html @@ -10,7 +10,7 @@ --> -

diff --git a/visualpython/html/component/helpViewer.html b/visualpython/html/component/helpViewer.html new file mode 100644 index 00000000..295370ac --- /dev/null +++ b/visualpython/html/component/helpViewer.html @@ -0,0 +1,41 @@ + + + + + \ No newline at end of file diff --git a/visualpython/html/component/packageManager.html b/visualpython/html/component/packageManager.html new file mode 100644 index 00000000..1e8ca640 --- /dev/null +++ b/visualpython/html/component/packageManager.html @@ -0,0 +1,39 @@ + +

+ +

+ Package + Version + +

+ +

+ \ No newline at end of file diff --git a/visualpython/html/component/popupComponent.html b/visualpython/html/component/popupComponent.html index 7fc8ce52..961f83e8 100644 --- a/visualpython/html/component/popupComponent.html +++ b/visualpython/html/component/popupComponent.html @@ -62,7 +62,7 @@

Install Package @@ -91,28 +91,40 @@ diff --git a/visualpython/html/menuFrame.html b/visualpython/html/menuFrame.html index a46f4d37..87938120 100644 --- a/visualpython/html/menuFrame.html +++ b/visualpython/html/menuFrame.html @@ -47,7 +47,9 @@

- +

+ +

diff --git a/visualpython/img/apps/apps_anova.svg b/visualpython/img/apps/apps_anova.svg new file mode 100644 index 00000000..6564ac04 --- /dev/null +++ b/visualpython/img/apps/apps_anova.svg @@ -0,0 +1,7 @@ + diff --git a/visualpython/img/apps/apps_chi2test.svg b/visualpython/img/apps/apps_chi2test.svg new file mode 100644 index 00000000..605e5169 --- /dev/null +++ b/visualpython/img/apps/apps_chi2test.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/apps/apps_corrAnalysis.svg b/visualpython/img/apps/apps_corrAnalysis.svg new file mode 100644 index 00000000..c433dd49 --- /dev/null +++ b/visualpython/img/apps/apps_corrAnalysis.svg @@ -0,0 +1,9 @@ + diff --git a/visualpython/img/apps/apps_descStats.svg b/visualpython/img/apps/apps_descStats.svg new file mode 100644 index 00000000..0ff04348 --- /dev/null +++ b/visualpython/img/apps/apps_descStats.svg @@ -0,0 +1,13 @@ + diff --git a/visualpython/img/apps/apps_equalVarTest.svg b/visualpython/img/apps/apps_equalVarTest.svg new file mode 100644 index 00000000..45667cbb --- /dev/null +++ b/visualpython/img/apps/apps_equalVarTest.svg @@ -0,0 +1,7 @@ + diff --git a/visualpython/img/apps/apps_factorAnalysis.svg b/visualpython/img/apps/apps_factorAnalysis.svg new file mode 100644 index 00000000..2d1a9d42 --- /dev/null +++ b/visualpython/img/apps/apps_factorAnalysis.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/apps/apps_gridSearch.svg b/visualpython/img/apps/apps_gridSearch.svg new file mode 100644 index 00000000..0d6ba9b5 --- /dev/null +++ b/visualpython/img/apps/apps_gridSearch.svg @@ -0,0 +1,6 @@ + diff --git a/visualpython/img/apps/apps_logisticRegression.svg b/visualpython/img/apps/apps_logisticRegression.svg new file mode 100644 index 00000000..d98a6c50 --- /dev/null +++ b/visualpython/img/apps/apps_logisticRegression.svg @@ -0,0 +1,12 @@ + diff --git a/visualpython/img/apps/apps_pipeline.svg b/visualpython/img/apps/apps_pipeline.svg new file mode 100644 index 00000000..aab67d91 --- /dev/null +++ b/visualpython/img/apps/apps_pipeline.svg @@ -0,0 +1,9 @@ + diff --git a/visualpython/img/apps/apps_probDist.svg b/visualpython/img/apps/apps_probDist.svg new file mode 100644 index 00000000..754436ad --- /dev/null +++ b/visualpython/img/apps/apps_probDist.svg @@ -0,0 +1,6 @@ + diff --git a/visualpython/img/apps/apps_studentstTest.svg b/visualpython/img/apps/apps_studentstTest.svg new file mode 100644 index 00000000..402de255 --- /dev/null +++ b/visualpython/img/apps/apps_studentstTest.svg @@ -0,0 +1,6 @@ + diff --git a/visualpython/img/delete.svg b/visualpython/img/delete.svg index 7e88533a..9ad2d368 100644 --- a/visualpython/img/delete.svg +++ b/visualpython/img/delete.svg @@ -1,5 +1,5 @@ diff --git a/visualpython/img/delete_activated.svg b/visualpython/img/delete_activated.svg new file mode 100644 index 00000000..1662171e --- /dev/null +++ b/visualpython/img/delete_activated.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/delete_disabled.svg b/visualpython/img/delete_disabled.svg new file mode 100644 index 00000000..7e88533a --- /dev/null +++ b/visualpython/img/delete_disabled.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/import_disabled.svg b/visualpython/img/import_disabled.svg new file mode 100644 index 00000000..daba0d42 --- /dev/null +++ b/visualpython/img/import_disabled.svg @@ -0,0 +1,7 @@ + diff --git a/visualpython/img/item_disable.svg b/visualpython/img/item_disable.svg new file mode 100644 index 00000000..9f3970cc --- /dev/null +++ b/visualpython/img/item_disable.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_disable_hover.svg b/visualpython/img/item_disable_hover.svg new file mode 100644 index 00000000..6491b8c6 --- /dev/null +++ b/visualpython/img/item_disable_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_enable.svg b/visualpython/img/item_enable.svg new file mode 100644 index 00000000..ab75f8a5 --- /dev/null +++ b/visualpython/img/item_enable.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/item_enable_hover.svg b/visualpython/img/item_enable_hover.svg new file mode 100644 index 00000000..02c24e50 --- /dev/null +++ b/visualpython/img/item_enable_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/non_numeric.svg b/visualpython/img/non_numeric.svg new file mode 100644 index 00000000..a7d9e42b --- /dev/null +++ b/visualpython/img/non_numeric.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/numeric.svg b/visualpython/img/numeric.svg new file mode 100644 index 00000000..abedb779 --- /dev/null +++ b/visualpython/img/numeric.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/package_manager.svg b/visualpython/img/package_manager.svg new file mode 100644 index 00000000..75dee2f0 --- /dev/null +++ b/visualpython/img/package_manager.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/package_manager_hover.svg b/visualpython/img/package_manager_hover.svg new file mode 100644 index 00000000..47a3693c --- /dev/null +++ b/visualpython/img/package_manager_hover.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/radio_checked.svg b/visualpython/img/radio_checked.svg new file mode 100644 index 00000000..7de7eba1 --- /dev/null +++ b/visualpython/img/radio_checked.svg @@ -0,0 +1,4 @@ + diff --git a/visualpython/img/radio_unchecked.svg b/visualpython/img/radio_unchecked.svg new file mode 100644 index 00000000..af64b40a --- /dev/null +++ b/visualpython/img/radio_unchecked.svg @@ -0,0 +1,3 @@ + diff --git a/visualpython/img/snippets/duplicate.svg b/visualpython/img/snippets/duplicate.svg index ad8b11b8..8e65f311 100644 --- a/visualpython/img/snippets/duplicate.svg +++ b/visualpython/img/snippets/duplicate.svg @@ -1,3 +1,3 @@ diff --git a/visualpython/img/snippets/duplicate_disabled.svg b/visualpython/img/snippets/duplicate_disabled.svg new file mode 100644 index 00000000..8be1cd9f --- /dev/null +++ b/visualpython/img/snippets/duplicate_disabled.svg @@ -0,0 +1,3 @@ + diff --git a/visualpython/img/snippets/duplicated_activated.svg b/visualpython/img/snippets/duplicated_activated.svg new file mode 100644 index 00000000..d59ce072 --- /dev/null +++ b/visualpython/img/snippets/duplicated_activated.svg @@ -0,0 +1,3 @@ + diff --git a/visualpython/img/snippets/run.svg b/visualpython/img/snippets/run.svg index 8739260b..8fccce46 100644 --- a/visualpython/img/snippets/run.svg +++ b/visualpython/img/snippets/run.svg @@ -1,6 +1,13 @@ diff --git a/visualpython/img/snippets/run_activated.svg b/visualpython/img/snippets/run_activated.svg new file mode 100644 index 00000000..fc9c405e --- /dev/null +++ b/visualpython/img/snippets/run_activated.svg @@ -0,0 +1,13 @@ + diff --git a/visualpython/img/snippets/run_disabled.svg b/visualpython/img/snippets/run_disabled.svg new file mode 100644 index 00000000..0167ee8d --- /dev/null +++ b/visualpython/img/snippets/run_disabled.svg @@ -0,0 +1,13 @@ + diff --git a/visualpython/img/status_green.svg b/visualpython/img/status_green.svg new file mode 100644 index 00000000..13e43ec3 --- /dev/null +++ b/visualpython/img/status_green.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/status_orange.svg b/visualpython/img/status_orange.svg new file mode 100644 index 00000000..93b230c1 --- /dev/null +++ b/visualpython/img/status_orange.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/status_red.svg b/visualpython/img/status_red.svg new file mode 100644 index 00000000..6fe4a8bc --- /dev/null +++ b/visualpython/img/status_red.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/upgrade.svg b/visualpython/img/upgrade.svg new file mode 100644 index 00000000..d06ee058 --- /dev/null +++ b/visualpython/img/upgrade.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/upgrade_activated.svg b/visualpython/img/upgrade_activated.svg new file mode 100644 index 00000000..68adab28 --- /dev/null +++ b/visualpython/img/upgrade_activated.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/img/upgrade_disabled.svg b/visualpython/img/upgrade_disabled.svg new file mode 100644 index 00000000..f0afdc8e --- /dev/null +++ b/visualpython/img/upgrade_disabled.svg @@ -0,0 +1,5 @@ + diff --git a/visualpython/js/MainFrame.js b/visualpython/js/MainFrame.js index b7601fee..b16149b1 100644 --- a/visualpython/js/MainFrame.js +++ b/visualpython/js/MainFrame.js @@ -88,6 +88,11 @@ define([ top: colabHeaderHeight + 'px' }); }); + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + $('#vp_protectorShowWithoutKernel').click(function(evt) { + // just hide protector + vpConfig.hideProtector(); + }); } window.vpEvent = new com_Event(this); @@ -103,7 +108,7 @@ define([ // get visualpython minimum width // resizable setting // $('#vp_wrapper').resizable('disable'); - if (vpConfig.extensionType !== 'lab') { + if (vpConfig.extensionType !== 'lab' && vpConfig.extensionType !== 'lite') { $('#vp_wrapper').resizable({ // alsoResize: '#vp_menuFrame', helper: 'vp-wrapper-resizer', @@ -151,7 +156,7 @@ define([ top: colabHeaderHeight + 'px' }); this._resizeNotebook(vpWidth); - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: do nothing } } @@ -250,6 +255,19 @@ define([ { vp_note_display: vp_note_display, vp_note_width: vp_note_width }, { parent: this } ); + + // load protector for jupyterlite + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + $('#vp_wrapper').append(`

No kernel

You have to open the notebook or console to use Visual Python.

+ +

`) + } // consider height and width this._resizeWindow(); @@ -334,9 +352,9 @@ define([ this.boardFrame.hide(); newVpWidth = menuWidth + MENU_BOARD_SPACING; $('#vp_wrapper').width(newVpWidth); - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: set parent width and position, min-width - let target = $('#vp_wrapper').parent(); + let target = $('#vp_wrapper').closest('#jp-right-stack'); // compatible for notebook 7.x let prevWidth = target[0].getBoundingClientRect().width; let prevLeft = $(target).position().left; let widthDiff = newVpWidth - vpWidth; @@ -349,11 +367,23 @@ define([ vpLab.shell.collapseRight(); vpLab.shell.expandRight(); } - let relativeSizes = vpLab.shell._hsplitPanel.layout.relativeSizes(); - let absoluteSizes = vpLab.shell._hsplitPanel.layout.absoluteSizes(); - let newSize = absoluteSizes[1] - widthDiff; - let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); - vpLab.shell._hsplitPanel.layout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + if (vpLab.name === 'JupyterLab') { + let splitLayout = vpLab.shell._hsplitPanel.layout; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.absoluteSizes(); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } else { + // compatible for notebook 7.x + // vpLab.shell.layout.widgets[0] + let splitLayout = vpLab.shell.layout.widgets[0]; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.widgets.map(x => x.node.getBoundingClientRect().width); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } } else { $('#vp_wrapper').resizable({ minWidth: MENU_MIN_WIDTH + MENU_BOARD_SPACING }); } @@ -364,9 +394,9 @@ define([ this.boardFrame.show(); newVpWidth = vpWidth + BOARD_MIN_WIDTH + MENU_BOARD_SPACING; $('#vp_wrapper').width(newVpWidth); - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: set parent width and position, min-width - let target = $('#vp_wrapper').parent(); + let target = $('#vp_wrapper').closest('#jp-right-stack'); // compatible for notebook 7.x let prevWidth = target[0].getBoundingClientRect().width; let prevLeft = $(target).position().left; let widthDiff = newVpWidth - vpWidth; @@ -379,11 +409,23 @@ define([ vpLab.shell.collapseRight(); vpLab.shell.expandRight(); } - let relativeSizes = vpLab.shell._hsplitPanel.layout.relativeSizes(); - let absoluteSizes = vpLab.shell._hsplitPanel.layout.absoluteSizes(); - let newSize = absoluteSizes[1] - widthDiff; - let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); - vpLab.shell._hsplitPanel.layout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + if (vpLab.name === 'JupyterLab') { + let splitLayout = vpLab.shell._hsplitPanel.layout; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.absoluteSizes(); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } else { + // compatible for notebook 7.x + // vpLab.shell.layout.widgets[0] + let splitLayout = vpLab.shell.layout.widgets[0]; + let relativeSizes = splitLayout.relativeSizes(); + let absoluteSizes = splitLayout.widgets.map(x => x.node.getBoundingClientRect().width); + let newSize = absoluteSizes[1] - widthDiff; + let totalWidth = absoluteSizes.reduce((x, a) => x + a, 0); + splitLayout.setRelativeSizes([relativeSizes[0], newSize/totalWidth, newVpWidth/totalWidth]); + } } else { $('#vp_wrapper').resizable({ minWidth: VP_MIN_WIDTH }); } @@ -474,14 +516,14 @@ define([ this.boardFrame.showLoadingBar(); // create components // LAB: use require - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let parentBlock = null; let prevBlock = null; loadStateList.forEach(obj => { - let { blockType, menuId, menuState, menuConfig, argIdx, position, afterAction } = obj; + let { file, blockType, menuId, menuState, menuConfig, argIdx, position, afterAction } = obj; // get OptionComponent Object // LAB: relative path needed - let OptionComponent = require('./' + menuConfig.file); + let OptionComponent = require('./' + file); if (OptionComponent) { let taskState = menuState.taskState; let blockState = menuState.blockState; diff --git a/visualpython/js/board/BoardFrame.js b/visualpython/js/board/BoardFrame.js index e8e56b21..cf273e67 100644 --- a/visualpython/js/board/BoardFrame.js +++ b/visualpython/js/board/BoardFrame.js @@ -87,7 +87,7 @@ define([ get blockList() { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; @@ -102,7 +102,7 @@ define([ set blockList(val) { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; @@ -122,7 +122,7 @@ define([ addToBlockList(newVal, position=-1) { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; @@ -154,7 +154,7 @@ define([ removeFromBlockList(removeVal) { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; @@ -173,7 +173,7 @@ define([ getTitle() { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; @@ -189,7 +189,7 @@ define([ setTitle(newTitle) { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; @@ -479,7 +479,7 @@ define([ this.blockMenu = new BlockMenu(this); - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let that = this; vpLab.shell._currentChanged.connect(function(sender, value) { // if lab tab changed, reset title and reload board @@ -643,7 +643,7 @@ define([ let vpFilePath = filesPath[0].path; let vpFileName = filesPath[0].file; // read file - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: read file using python open vpKernel.readFile(vpFilePath).then(function(resultObj) { try { @@ -972,7 +972,7 @@ define([ moveBlock(startIdx, endIdx, parentBlock=null) { let sessionId = 'default'; // LAB: get session id - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { let panelId = vpKernel.getLabPanelId(); if (panelId) { sessionId = panelId; diff --git a/visualpython/js/com/com_Config.js b/visualpython/js/com/com_Config.js index 5b2ea9b4..9062699a 100644 --- a/visualpython/js/com/com_Config.js +++ b/visualpython/js/com/com_Config.js @@ -48,15 +48,16 @@ define([ /** * * @param {*} initialData - * @param {*} extensionType extension type: notebook/colab/lab + * @param {*} extensionType extension type: notebook/colab/lab/lite */ constructor(extensionType='notebook', initialData={}) { // initial mode + this._isReady = { 'default': false }; this.extensionType = extensionType; this.parentSelector = 'body'; if (extensionType === 'notebook') { this.parentSelector = '#site'; - } else if (extensionType === 'colab' || extensionType === 'lab') { + } else if (extensionType === 'colab' || extensionType === 'lab' || extensionType === 'lite') { // this.parentSelector = '.notebook-horizontal'; this.parentSelector = 'body'; } @@ -64,7 +65,7 @@ define([ this.data = { // Configuration 'vpcfg': { - + 'runType': 'run' }, // User defined code for Snippets 'vpudf': { @@ -74,7 +75,8 @@ define([ 'import matplotlib.pyplot as plt', '%matplotlib inline', 'import seaborn as sns', - 'import plotly.express as px' + 'import plotly.express as px', + 'import pyarrow as pa' ], 'matplotlib customizing': [ 'import matplotlib.pyplot as plt', @@ -131,7 +133,8 @@ define([ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' ] - } + }, + { library: 'pyarrow', alias:'pa' }, ] } @@ -195,6 +198,22 @@ define([ 'joblib': { code: 'import joblib', type: 'package' + }, + 'stats': { + code: 'from scipy import stats', + type: 'package' + }, + 'scipy': { + code: 'import scipy', + type: 'package' + }, + 'sm': { + code: 'import statsmodels.api as sm', + type: 'package' + }, + 'pyarrow': { + code: 'import pyarrow as pa', + type: 'package' } } @@ -203,6 +222,41 @@ define([ } + get isReady() { + let sessionId = 'default'; + if (this.extensionType === 'lab' || this.extensionType === 'lite') { + let panelId = vpKernel.getLabPanelId(); + if (panelId) { + sessionId = panelId; + } + } + if (sessionId in this._isReady) { + return this._isReady[sessionId]; + } + return false; + } + + set isReady(ready) { + let sessionId = 'default'; + if (this.extensionType === 'lab' || this.extensionType === 'lite') { + let panelId = vpKernel.getLabPanelId(); + if (panelId) { + sessionId = panelId; + } + } + this._isReady[sessionId] = ready; + } + + showProtector(title='No kernel', content='You have to open the notebook or console to use Visual Python.') { + $('#vp_protector .vp-protector-title').text(title); + $('#vp_protector .vp-protector-content').text(content); + $('#vp_protector').show(); + } + + hideProtector() { + $('#vp_protector').hide(); + } + /** * Read dejault config */ @@ -216,8 +270,7 @@ define([ vp_config_version: '1.0.0', vp_signature: 'VisualPython', vp_position: {}, - // CHROME: default to display vp - vp_section_display: true, + vp_section_display: false, vp_note_display: false, vp_menu_width: Config.MENU_MIN_WIDTH, vp_note_width: Config.BOARD_MIN_WIDTH @@ -251,7 +304,9 @@ define([ let packageAlias = { '_vp_np': 'np', '_vp_pd': 'pd', - '_vp_plt': 'plt' + '_vp_plt': 'plt', + '_vp_stats': 'stats', + '_vp_sm': 'sm' } for (let i = 0; i < tmpList.length; i += 2) { @@ -282,8 +337,12 @@ define([ * - automatically restart on jupyter kernel restart (loadVisualpython.js) */ readKernelFunction() { + let that = this; // CHROME: change method to load py files ($.get -> require) return new Promise(function(resolve, reject) { + // if (that.extensionType === 'lite') { + // that.showProtector('Kernel loading', 'Required inner function is loading now...'); + // } var libraryList = [ printCommand, fileNaviCommand, pandasCommand, variableCommand, visualizationCommand ]; @@ -300,7 +359,26 @@ define([ libraryList.forEach(libCode => { promiseList.push(vpKernel.execute(libCode, true)); }); - + + if (that.extensionType === 'lite') { + let preInstallCode = ''; + let preInstallPackList = [ + 'seaborn', + 'plotly', + 'scikit-learn', + 'scipy', + 'statsmodels' + ]; + preInstallPackList.forEach((packName, idx) => { + preInstallCode += '%pip install ' + packName + if (idx < preInstallPackList.length - 1) { + preInstallCode += '\n'; + } + }); + // pre-install packages + promiseList.push(vpKernel.execute(preInstallCode, true)); + } + // run all promises let failed = false; Promise.all(promiseList).then(function(resultObj) { @@ -310,6 +388,9 @@ define([ console.log('visualpython - failed to load library', resultObj); // TODO: show to restart kernel }).finally(function() { + // if (that.extensionType === 'lite') { + // that.hideProtector(); + // } if (!failed) { console.log('visualpython - loaded libraries', libraryList); resolve(true); @@ -450,8 +531,8 @@ define([ // not mounted reject('Colab Drive is not mounted!'); }) - } else if (that.extensionType === 'lab') { - // CHROME: edited to use .visualpython files + } else if (that.extensionType === 'lab' || that.extensionType === 'lite') { + // LAB: edited to use .visualpython files that._readFromLab('', configKey).then(function(result) { resolve(result); }).catch(function(err) { @@ -493,7 +574,7 @@ define([ that._checkMounted().then(function() { that._readFromColab(configKey).then(function(result) { let data = result; - if (data == undefined || data == {}) { + if (data == undefined || (data instanceof Object && Object.keys(data).length === 0)) { resolve(data); return; } @@ -501,7 +582,7 @@ define([ resolve(data); return; } - if (Object.keys(data).length > 0) { + if (data instanceof Object && Object.keys(data).length > 0) { resolve(data[dataKey]); return; } @@ -513,11 +594,11 @@ define([ // not mounted reject('Colab Drive is not mounted!'); }) - } else if (that.extensionType === 'lab') { + } else if (that.extensionType === 'lab' || that.extensionType === 'lite') { // LAB: use local .visualpython files that._readFromLab(configKey).then(function(result) { let data = result; - if (data == undefined || data == {}) { + if (data == undefined || (data instanceof Object && Object.keys(data).length === 0)) { resolve(data); return; } @@ -525,7 +606,7 @@ define([ resolve(data); return; } - if (Object.keys(data).length > 0) { + if (data instanceof Object && Object.keys(data).length > 0) { resolve(data[dataKey]); return; } @@ -561,7 +642,7 @@ define([ /** * Set configuration data (on server) * @param {Object} dataObj - * @param {String} configKey + * @param {String} configKey vpcfg / vpudf / vpimport / vppackman */ setData(dataObj, configKey='vpudf') { let that = this; @@ -591,7 +672,7 @@ define([ reject(); }); }); - } else if (that.extensionType === 'lab') { + } else if (that.extensionType === 'lab' || that.extensionType === 'lite') { // LAB: use .visualpython files that.getData('', configKey).then(function(data) { let newDataObj = {}; @@ -636,7 +717,7 @@ define([ }).catch(function(err) { reject(false); }) - } else if (that.extensionType === 'lab') { + } else if (that.extensionType === 'lab' || that.extensionType === 'lite') { // LAB: use .visualpython files that.getData('', configKey).then(function(data) { let dataObj = data; @@ -769,7 +850,7 @@ define([ throw new Error('Error', response); } }).then(function (data) { - resolve(data.info.version); + resolve(data.info.version, data.releases); }).catch(function(err) { let errMsg = err.message; if (errMsg.includes('Failed to fetch')) { @@ -787,22 +868,71 @@ define([ return Config.version; } + checkVersionTimestamp = function() { + let that = this; + // check version timestamp + let nowDate = new Date(); + this.getData('version_timestamp', 'vpcfg').then(function(data) { + let doCheckVersion = false; + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Checking its version timestamp... : ' + data); + if (data == undefined || (data instanceof Object && Object.keys(data).length === 0)) { + // no timestamp, check version + doCheckVersion = true; + } else if (data != '') { + let lastCheck = new Date(parseInt(data)); + let diffCheck_now = new Date(nowDate.getFullYear(), nowDate.getMonth() + 1, nowDate.getDate()); + let diffCheck_last = new Date(lastCheck.getFullYear(), lastCheck.getMonth() + 1, lastCheck.getDate()); + + let diff = Math.abs(diffCheck_now.getTime() - diffCheck_last.getTime()); + diff = Math.ceil(diff / (1000 * 3600 * 24)); + + if (diff >= 1) { + // if More than 1 day passed, check version + doCheckVersion = true; + } + } + + // check version and update version_timestamp + if (doCheckVersion == true) { + that.checkVpVersion(true); + } + + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + }) + } + checkVpVersion(background=false) { let that = this; let nowVersion = this.getVpInstalledVersion(); let packageName = 'visualpython'; - if (this.extensionType === 'lab') { + if (this.extensionType === 'lab' || this.extensionType === 'lite') { packageName = 'jupyterlab-visualpython'; } this.getPackageVersion(packageName).then(function(latestVersion) { - if (nowVersion === latestVersion) { + let showUpdater = false; + if (nowVersion !== latestVersion) { + let nowVerParts = nowVersion.split('.').map(x => ~~x); + let latVerParts = latestVersion.split('.').map(x => ~~x); + for (var i = 0; i < nowVerParts.length; i++) { + const a = nowVerParts[i]; + const b = latVerParts[i]; + if (a < b) { + showUpdater = true; + break; + } else if (a > b) { + break; + } + } + } + if (showUpdater === false) { // if it's already up to date // hide version update icon $('#vp_versionUpdater').hide(); - if (background) { + if (background === true) { ; } else { - let msg = com_util.formatString('Visual Python is up to date. ({0})', latestVersion); + let msg = com_util.formatString('Visual Python is up to date. ({0})', nowVersion); com_util.renderInfoModal(msg); } // update version_timestamp @@ -826,6 +956,8 @@ define([ switch (clickedBtnIdx) { case 0: // cancel + // update version_timestamp + that.setData({ 'version_timestamp': new Date().getTime() }, 'vpcfg'); break; case 1: // update @@ -859,6 +991,16 @@ define([ ]; com_interface.insertCell('markdown', info.join('\n')); com_interface.insertCell('code', '!pip install jupyterlab-visualpython --upgrade'); + } else if (that.extensionType === 'lite') { + // LITE: update lab extension on lite + let info = [ + '## Visual Python Upgrade', + 'NOTE: ', + '- Refresh your web browser to start a new version.', + '- Save VP Note before refreshing the page.' + ]; + com_interface.insertCell('markdown', info.join('\n')); + com_interface.insertCell('code', "%pip install jupyterlab-visualpython==" + latestVersion); } // update version_timestamp @@ -931,7 +1073,7 @@ define([ /** * Version */ - Config.version = "2.3.7"; + Config.version = "3.0.2"; /** * Type of mode @@ -982,11 +1124,11 @@ define([ Config.ML_DATA_DICT = { 'Data Preparation': [ /** Encoding */ - 'OneHotEncoder', 'LabelEncoder', 'OrdinalEncoder', 'TargetEncoder', 'SMOTE', + 'OneHotEncoder', 'LabelEncoder', 'OrdinalEncoder', 'TargetEncoder', /** Scaling */ 'StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer', 'FunctionTransformer', 'PolynomialFeatures', 'KBinsDiscretizer', /** ETC */ - 'ColumnTransformer' + 'SimpleImputer', 'SMOTE', 'ColumnTransformer' ], 'Regression': [ 'LinearRegression', 'Ridge', 'Lasso', 'ElasticNet', 'SVR', 'DecisionTreeRegressor', 'RandomForestRegressor', 'GradientBoostingRegressor', 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor', @@ -1002,6 +1144,9 @@ define([ ], 'Auto ML': [ 'AutoSklearnRegressor', 'AutoSklearnClassifier', 'TPOTRegressor', 'TPOTClassifier' + ], + 'ETC': [ + 'GridSearchCV' ] }; @@ -1011,7 +1156,8 @@ define([ ...Config.ML_DATA_DICT['Classification'], ...Config.ML_DATA_DICT['Clustering'], ...Config.ML_DATA_DICT['Dimension Reduction'], - ...Config.ML_DATA_DICT['Auto ML'] + ...Config.ML_DATA_DICT['Auto ML'], + ...Config.ML_DATA_DICT['ETC'] ]; return Config; diff --git a/visualpython/js/com/com_Const.js b/visualpython/js/com/com_Const.js index d3557ea6..fbd18788 100644 --- a/visualpython/js/com/com_Const.js +++ b/visualpython/js/com/com_Const.js @@ -19,7 +19,7 @@ define ([ class Constants { } Constants.TOOLBAR_BTN_INFO = { - HELP: "Visual Python 2.3.7" + HELP: "Visual Python 3.0.2" , ICON: "vp-main-icon" , ID: "vpBtnToggle" , NAME: "toggle-vp" diff --git a/visualpython/js/com/com_Event.js b/visualpython/js/com/com_Event.js index 3cc1aece..10b3e09b 100644 --- a/visualpython/js/com/com_Event.js +++ b/visualpython/js/com/com_Event.js @@ -207,8 +207,13 @@ define([], function() { that.keyManager.keyCheck.shiftKey = false; } if (evt.keyCode == that.keyManager.keyCode.escKey) { + // check if there is visible file navigation : FileNavigation + if ($('.vp-filenavigation-base:visible').length > 0) { + // close file navigation + $('.vp-filenavigation-base:visible').remove(); + } // check if there is visible data selector : DataSelector - if ($('.vp-dataselector-base:visible').length > 0) { + else if ($('.vp-dataselector-base:visible').length > 0) { // close data selector $('.vp-dataselector-base:visible').remove(); } diff --git a/visualpython/js/com/com_Kernel.js b/visualpython/js/com/com_Kernel.js index 02bad35c..addb7093 100644 --- a/visualpython/js/com/com_Kernel.js +++ b/visualpython/js/com/com_Kernel.js @@ -44,6 +44,9 @@ define([ }, 'vpimport': { + }, + 'vppackman': { + } } } @@ -239,7 +242,7 @@ define([ cell.runButton.click(); // set last focused cell colab.global.notebook.focusCell(lastFocusedCellId); - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: // { code, stdin, stop_on_error, silent, ... } var codeObj = { @@ -249,7 +252,6 @@ define([ } var kernelConnection = that.getLabKernel(); if (kernelConnection) { - var future = kernelConnection.requestExecute(codeObj); var onIOPub = function(msg) { const msgType = msg.header.msg_type; switch(msgType) { @@ -380,149 +382,18 @@ define([ }; return; } - future.onIOPub = onIOPub; - // future.onIOPub = (msg) => { - // const msgType = msg.header.msg_type; - // switch(msgType) { - // case 'status': - // // if(!isExpectingOutput){ - // // if(msg.content.execution_state === 'idle'){ - // // resolve(); - // // } - // // } - // return; - // case 'execute_input': - // // var content = msg.content; - // // resolve({ - // // result: content, - // // type: type, - // // msg: { - // // content: { - // // name: type, - // // data: { - // // [type]: content - // // } - // // } - // // } - // // }); - // return; - // case 'stream': - // var content = msg.content; - // var type = content.name; - // switch(type){ - // case 'stdout': - // var message = content.text; - // resolve({ - // result: message, - // type: type, - // msg: { - // content: { - // name: type, - // data: { - // [type]: message - // } - // } - // } - // }); - // break; - // case 'stderr': - // var message = content.text; - // reject({status: 'stderr', ename: 'stderr', evalue: message}); - // break; - // default: - // var message = '[jupyterLabTerminal]: Unknown stream type ' + type; - // reject({status: 'error', ename: 'Unknown stream type', evalue: message}); - // } - // break; - // case 'error': - // //stderr does not yield output for all errors - // // var message = msg.content.ename + '\n' + msg.content.evalue; - // // check if it has a problem on restarting vp inner function - // // ex) "name '_vp_print' is not defined" - // if (msg.content.ename === 'NameError' - // && msg.content.evalue.includes('_vp_') - // && msg.content.evalue.includes('is not defined')) { - // // restart vp - // vpConfig.readKernelFunction(); - // } - // reject({status: 'error', ename: msg.content.ename, evalue: msg.content.evalue}); - // break; - // case 'execute_result': - // var type = 'text'; - // if (msg.content) { - // try { - // if (msg.content['text']) { - // result = String(msg.content['text']); - // type = 'text'; - // } else if (msg.content.data) { - // if (msg.content.data['image/png']) { - // result = String(msg.content.data['image/png']); - // type = 'image/png'; - // } else if (msg.content.data['text/plain']) { - // result = String(msg.content.data['text/plain']); - // type = 'text/plain'; - // } else if (msg.content.data['text/html']) { - // result = String(msg.content.data['text/html']); - // type = 'text/html'; - // } - // } - // resolve({result: result, type: type, msg: msg}); - // } catch(ex) { - // reject(ex); - // } - // } else { - // resolve({result: result, type: type, msg: msg}); - // } - // break; - // case 'display_data': - // var type = 'text'; - // if (msg.content) { - // try { - // if (msg.content['text']) { - // result = String(msg.content['text']); - // type = 'text'; - // } else if (msg.content.data) { - // if (msg.content.data['image/png']) { - // result = String(msg.content.data['image/png']); - // type = 'image/png'; - // } else if (msg.content.data['text/plain']) { - // result = String(msg.content.data['text/plain']); - // type = 'text/plain'; - // } else if (msg.content.data['text/html']) { - // result = String(msg.content.data['text/html']); - // type = 'text/html'; - // } - // } - // resolve({result: result, type: type, msg: msg}); - // } catch(ex) { - // reject(ex); - // } - // } else { - // resolve({result: result, type: type, msg: msg}); - // } - // break; - // case 'update_display_data': - // var result = msg.content; - // resolve({ - // result: result, - // type: msgType, - // msg: { - // content: { - // name: msgType, - // data: { - // [msgType]: result - // } - // } - // } - // }); - // break; - // default: - // var message = '[jupyterLabTerminal]: Unknown message type ' + msgType; - // reject({status: 'error', ename: 'Unknown message type', evalue: message}); - - // }; - // return; - // }; + var future; + if (vpConfig.isReady === false) { + vpConfig.isReady = true; + vpConfig.hideProtector(); + vpConfig.readKernelFunction().then(function() { + future = kernelConnection.requestExecute(codeObj); + future.onIOPub = onIOPub; + }); + } else { + future = kernelConnection.requestExecute(codeObj); + future.onIOPub = onIOPub; + } } } @@ -602,14 +473,27 @@ define([ getLabNotebookPanel(){ var mainWidgets = this.app.shell.widgets('main'); var widget = mainWidgets.next(); - while(widget){ - if(widget.sessionContext){ + while (widget) { + if (widget.sessionContext) { var type = widget.sessionContext.type; if(type == 'notebook' || type == 'console'){ //other wigets might be of type DocumentWidget if (widget.isVisible){ return widget; } } + } else if (widget.value !== undefined && widget.done === false) { + // for upper lab 4 + let widgetObj = widget.value; + if (widgetObj.sessionContext) { + var type = widgetObj.sessionContext.type; + if(type == 'notebook' || type == 'console'){ //other wigets might be of type DocumentWidget + if (widgetObj.isVisible){ + return widgetObj; + } + } + } + } else if (widget.done === true) { + break; } widget = mainWidgets.next(); } @@ -623,6 +507,14 @@ define([ while (widget) { if (widget.sessionContext) { widgetList.push(widget); + } else if (widget.value !== undefined && widget.done === false) { + // for upper lab 4 + let widgetObj = widget.value; + if (widgetObj.sessionContext) { + widgetList.push(widgetObj); + } + } else if (widget.done === true) { + break; } widget = mainWidgets.next(); } @@ -718,10 +610,14 @@ define([ * @param {*} dataframe * @returns */ - getRowList(dataframe) { + getRowList(dataframe, start_idx) { var that = this; + if (typeof start_idx === 'undefined') { + start_idx = 0; + } + return new Promise(function(resolve, reject) { - that.execute(com_util.formatString('_vp_print(_vp_get_rows_list({0}))', dataframe)) + that.execute(com_util.formatString('_vp_print(_vp_get_rows_list({0}, {1}))', dataframe, start_idx)) .then(function(resultObj) { resolve(resultObj); }).catch(function(err) { @@ -829,6 +725,9 @@ define([ case 'vpimport': configFile = 'import'; break; + case 'vppackman': + configFile = 'packman'; + break; } var cmd = com_util.formatString("print(_vp_get_colab_vpcfg('{0}'))", configFile); return new Promise(function(resolve, reject) { @@ -861,6 +760,9 @@ define([ case 'vpimport': configFile = 'import'; break; + case 'vppackman': + configFile = 'packman'; + break; } // write file var sbfileSaveCmd = new com_String(); @@ -895,6 +797,9 @@ define([ case 'vpimport': configFile = 'import'; break; + case 'vppackman': + configFile = 'packman'; + break; } var cmd = com_util.formatString("print(_vp_get_lab_vpcfg('{0}'))", configFile); return new Promise(function(resolve, reject) { @@ -927,6 +832,9 @@ define([ case 'vpimport': configFile = 'import'; break; + case 'vppackman': + configFile = 'packman'; + break; } // write file var sbfileSaveCmd = new com_String(); @@ -1023,6 +931,23 @@ define([ }) } + //==================================================================== + // Package Manager + //==================================================================== + getPackageList(packList=[]) { + var that = this; + let code = com_util.formatString("_vp_print(_vp_check_package_list({0}))", JSON.stringify(packList)); + return new Promise(function(resolve, reject) { + that.execute(code).then(function(resultObj) { + // resolve + resolve(resultObj); + }).catch(function(err) { + // reject + reject(err); + }) + }); + } + //==================================================================== // Configuration api //==================================================================== diff --git a/visualpython/js/com/com_generatorV2.js b/visualpython/js/com/com_generatorV2.js index 28abaf8f..42e6c760 100644 --- a/visualpython/js/com/com_generatorV2.js +++ b/visualpython/js/com/com_generatorV2.js @@ -14,8 +14,9 @@ define([ 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/com/component/FileNavigation' -], function (com_util, com_makeDom, SuggestInput, VarSelector2, DataSelector, FileNavigation) { +], function (com_util, com_makeDom, SuggestInput, VarSelector2, DataSelector, MultiSelector, FileNavigation) { /** * show result after code executed */ @@ -39,6 +40,7 @@ define([ 'var_select': 'Select Variable', 'var_multi': 'Select N-Variables', 'col_select': 'Select Column', + 'col_multi': 'Select Columns', 'textarea': 'Input textarea', 'input_number': 'Input number', 'input_text': 'Input text', @@ -137,7 +139,7 @@ define([ package.options && package.options.forEach(function(o, i) { var obj = JSON.parse(JSON.stringify(o)); let newTag = vp_createTag(pageThis, obj, state); - if (obj.required) { + if (obj.required === true || obj.output === true) { tblInput.append(newTag); } else { tblOption.append(newTag); @@ -145,7 +147,11 @@ define([ }); // TODO: userOption + if (package.code.includes('${etc}')) { + + } + bindMultiSelector(pageThis); bindAutoComponentEvent(pageThis); } @@ -162,11 +168,16 @@ define([ var tblContent = $(''); - let { name, label, component, required } = obj; + let { name, label, component, required, output } = obj; let value = state[name]; - var requiredFontStyle = required == true? 'vp-orange-text' : ''; - var lblTag = $(`${label}`).attr({ + var requiredFontStyle = ''; + if (required === true) { + requiredFontStyle = 'vp-bold vp-orange-text'; + } else if (output === true) { + requiredFontStyle = 'vp-bold'; + } + var lblTag = $(`${label || com_util.optionToLabel(name)}`).attr({ 'for': name, 'class': requiredFontStyle, 'title': '(' + name + ')' @@ -307,7 +318,8 @@ define([ case 'option_select': var optSlct = $('').attr({ 'class':'vp-select option-select vp-state', - 'id':obj.name + 'id':obj.name, + 'title': (obj.help==undefined?'':obj.help), }); obj.options.forEach((opt, idx, arr) => { var label = (obj.options_label != undefined? obj.options_label[idx]:opt); @@ -360,9 +372,12 @@ define([ let dataSelector = new DataSelector({ pageThis: pageThis, id: obj.name, + type: obj.comp_type || 'data', allowDataType: obj.var_type, placeholder: obj.placeholder || 'Select data', value: value, + columnSelection: obj.columnSelection || 'multiple', // single / multiple + returnFrameType: obj.returnFrameType || '', // '' / DataFrame / Series required: obj.required === true }); content = $(dataSelector.toTagString()); @@ -374,7 +389,8 @@ define([ id: obj.name, class: 'vp-input vp-state', placeholder: obj.placeholder || 'Select data', - required: obj.required === true + required: obj.required === true, + title: (obj.help==undefined?'':obj.help), }); vp_generateVarSuggestInput(pageThis.wrapSelector(), obj); content = tag; @@ -385,7 +401,8 @@ define([ 'id': obj.name, 'class': 'vp-select var-multi vp-state', // multiple selection true - 'multiple': true + 'multiple': true, + 'title': (obj.help==undefined?'':obj.help), }); vp_generateVarSelect(tag, obj.var_type, obj.value); content = tag; @@ -394,8 +411,26 @@ define([ var tag = $('').attr({ 'type': 'text', 'id': obj.name, - 'class': 'vp-input vp-state' + 'class': 'vp-input vp-state', + 'title': (obj.help==undefined?'':obj.help), + }); + content = tag; + break; + case 'col_multi': + var tag1 = $('

').attr({ + 'id': obj.name + '_auto_comp', + 'class': 'vp-auto-multi-column', + 'data-id': obj.name, + 'data-target': obj.comp_target || 'i0', + 'style': 'height: 150px;', + 'title': (obj.help==undefined?'':obj.help), }); + var tag2 = $('').attr({ + type: 'hidden', + id: obj.name, + class: 'vp-input vp-state', + }) + var tag = $('

').append(tag1).append(tag2); content = tag; break; case 'textarea': @@ -702,6 +737,17 @@ define([ case 'dtype': value = $(pageThis.wrapSelector(parent + ' #'+obj.name)).val(); break; + case 'col_multi': + let colList = pageThis.autoGen[obj.name].getDataList(); + pageThis.state[obj.name] = colList.map(data => { return data.code }); + value = colList.map(data => { return data.code }).join(','); + if (colList.length == 0) { + value = ''; + } else if (colList.length > 0) { + value = '[' + value + ']'; + } + $(pageThis.wrapSelector('#'+obj.name)).val(value); + break; case 'file-open': case 'file-save': case 'table': @@ -733,7 +779,9 @@ define([ package.options && package.options.forEach(function(v, i) { var val = state[v.name]; if (val == undefined || val == '' || val == v.default) { - val = vp_getTagValue(pageThis, v, parent=parent); + if (pageThis) { + val = vp_getTagValue(pageThis, v, parent=parent); + } } var id = '${' + v.name + '}'; if (val == undefined || val.trim() == '') { @@ -754,7 +802,11 @@ define([ } // code completion 2 if (v.usePair == true) { - val = ', ' + v.name + '=' + val; + if (v.pairKey) { + val = ', ' + v.pairKey + '=' + val; + } else { + val = ', ' + v.name + '=' + val; + } } // code completion 3 if (v.component != undefined && v.componentCode != undefined) { @@ -794,9 +846,11 @@ define([ if (_VP_SHOW_RESULT && package.options) { var outputOptList = package.options.filter(x => x.output === true); var outputStr = ''; - outputOptList.forEach(opt => { - outputStr += (outputStr !== ''?', ':'') + vp_getTagValue(pageThis, opt); - }) + if (pageThis) { + outputOptList.forEach(opt => { + outputStr += (outputStr !== ''?', ':'') + vp_getTagValue(pageThis, opt); + }) + } if (outputStr != '') { code += '\n'+ outputStr; } @@ -820,7 +874,7 @@ define([ * @param {array/boolean} columnWithIndex boolean array or value to decide whether select tag has index option * Usage : * $(document).on('change', this.wrapSelector('#dataframe_tag_id'), function() { - * pdGen.vp_bindColumnSource(that, 'dataframe_tag_id', ['column_input_id'], 'select', [true, true, true]); + * pdGen.vp_bindColumnSource(that, 'dataframe_tag_id', ['column_input_id'], 'select', false, false); * }); */ var vp_bindColumnSource = function(pageThis, targetId, columnInputIdList, tagType="input", columnWithEmpty=false, columnWithIndex=false) { @@ -838,7 +892,7 @@ define([ suggestInputX.addClass('vp-input vp-state'); suggestInputX.setNormalFilter(false); suggestInputX.setValue(defaultValue); - $(selector + ' #' + columnInputId).replaceWith(function() { + $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return suggestInputX.toTagString(); }); } else { @@ -847,7 +901,7 @@ define([ 'id': columnInputId, 'class': 'vp-select vp-state' }); - $(selector + ' #' + columnInputId).replaceWith(function() { + $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return $(tag); }); } @@ -903,7 +957,7 @@ define([ suggestInputX.setSuggestList(function() { return list; }); //FIXME: suggestInputX.setNormalFilter(false); suggestInputX.setValue(defaultValue); - $(selector + ' #' + columnInputId).replaceWith(function() { + $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return suggestInputX.toTagString(); }); } else { @@ -912,7 +966,7 @@ define([ 'class': 'vp-select vp-state' }); // make tag - list.forEach(listVar => { + list.forEach((listVar, idx) => { var option = document.createElement('option'); $(option).attr({ 'value':listVar.value, @@ -929,6 +983,7 @@ define([ $(pageThis.wrapSelector('#' + columnInputId)).replaceWith(function() { return $(tag); }); + $(pageThis.wrapSelector('#' + columnInputId)).trigger('change'); } }).catch(function(err) { vpLog.display(VP_LOG_TYPE.ERROR, 'com_generator - bindColumnSource error ', err) @@ -1219,6 +1274,63 @@ define([ return $(''); } + var bindMultiSelector = function(pageThis) { + //==================================================================== + // for column multi selector + //==================================================================== + $(pageThis.wrapSelector('.vp-auto-multi-column')).each((idx, tag) => { + let compId = tag.id; + let id = $(tag).data('id'); + let targetId = $(tag).data('target'); + let colSelector = new MultiSelector( + pageThis.wrapSelector('#' + compId), + { + mode: 'columns', parent: (pageThis.state[targetId] || ''), selectedList: pageThis.state[compId], + change: function(type, list) { + let value = list.map(data => { return data.code }).join(','); + if (list.length == 0) { + value = ''; + } else if (list.length > 0) { + value = '[' + value + ']'; + } + pageThis.state[compId] = list.map(data => { return data.code }); + pageThis.state[id] = value; + $(pageThis.wrapSelector('#'+id)).val(value); + } + } + ); + pageThis.autoGen = { + [id]: colSelector, + ...pageThis.autoGen + }; + $(pageThis.wrapSelector('#' + targetId)).on('change', function() { + let targetVariable = $(this).val(); + let colSelector = new MultiSelector( + pageThis.wrapSelector('#' + compId), + { + mode: 'columns', parent: targetVariable, selectedList: pageThis.state[compId], + change: function(type, list) { + let value = list.map(data => { return data.code }).join(','); + if (list.length == 0) { + value = ''; + } else if (list.length > 0) { + value = '[' + value + ']'; + } + pageThis.state[compId] = list.map(data => { return data.code }); + pageThis.state[id] = value; + $(pageThis.wrapSelector('#'+id)).val(value); + } + }, + ); + pageThis.autoGen = { + [id]: colSelector, + ...pageThis.autoGen + }; + }); + + }); + } + var bindAutoComponentEvent = function(pageThis) { let selector = pageThis.wrapSelector(); // Auto-component selector @@ -1482,7 +1594,6 @@ define([ $(pageThis.wrapSelector('#'+id)).val(code); }); - //==================================================================== // Event for tabBlock //==================================================================== diff --git a/visualpython/js/com/com_interface.js b/visualpython/js/com/com_interface.js index 46dc4818..ae7800a4 100644 --- a/visualpython/js/com/com_interface.js +++ b/visualpython/js/com/com_interface.js @@ -104,24 +104,41 @@ define([ } // move to executed cell // CHROME: TODO: - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { var { NotebookActions } = require('@jupyterlab/notebook'); var { signalToPromise } = require('@jupyterlab/coreutils'); var notebookPanel = vpKernel.getLabNotebookPanel(); if (notebookPanel && notebookPanel.sessionContext){ var sessionContext = notebookPanel.sessionContext; let sessionType = sessionContext.type; + if (vpConfig.extensionType === 'lite') { + // check one more time for jupyter lite + // - it returns 'notebook' when it's console + if (notebookPanel.console !== undefined) { + sessionType = 'console'; + } + } if (sessionType === 'notebook') { var notebook = notebookPanel.content; var notebookModel = notebook.model; - var cellModel = notebookModel.contentFactory.createCell(type, {}); - cellModel.value.text = command; - - const newCellIndex = notebook.activeCellIndex + 1; + // LAB 4.x: notebookPanel.content.contentFactory.createCodeCell / createMarkdownCell + // var cellModel = notebookModel.contentFactory.createCell(type, {}); // 셀 추가 - notebookModel.cells.insert(newCellIndex, cellModel); - notebook.activeCellIndex = newCellIndex; + // LAB 4.x: insert cell api 변경 + // notebookModel.cells.insert(newCellIndex, cellModel); + // notebook.activeCellIndex = newCellIndex; + NotebookActions.insertBelow(notebook); + try { + NotebookActions.changeCellType(notebook, type); + } catch(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + } + // let activeCellIndex = notebook.activeCellIndex; + // let cell = notebookModel.cells.model.getCell(activeCellIndex); + // cell.setSource(command); + let activeCell = notebook.activeCell; + activeCell.model.sharedModel.setSource(command); if (exec == true) { try{ @@ -133,22 +150,24 @@ define([ // move to executed cell $(vpKernel.getLabNotebookPanel().content.activeCell.node)[0].scrollIntoView(true); } else if (sessionType === 'console') { + //TODO:LITE: console check needed var labConsole = notebookPanel.content; var widget = labConsole.widgets[0]; if (type === 'markdown') { // add # before the lines - command = '#' + command.split('\n').join('#'); + command = '#' + command.split('\n').join('\n#'); } - + + // LAB 4.x: console insert and run + widget.promptCell.model.sharedModel.setSource(command) // execute or not if (exec == true) { // console allow only code cell - var cellModel = widget.createCodeCell(); - cellModel.model.value.text = command; - widget.addCell(cellModel); - widget._execute(cellModel); - } else { - widget.promptCell.model.value.text = command; + // var cellModel = widget.createCodeCell(); + // cellModel.model.value.text = command; + // widget.addCell(cellModel); + // widget._execute(cellModel); + widget.execute() } widget.promptCell.inputArea.editor.focus(); } @@ -169,7 +188,7 @@ define([ */ var insertCells = function(type, commands, exec=true, sigText='') { - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { var { NotebookActions } = require('@jupyterlab/notebook'); var notebookPanel = vpKernel.getLabNotebookPanel(); } @@ -239,24 +258,40 @@ define([ }, 300); } } - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { if (notebookPanel && notebookPanel.sessionContext){ - var sessionContext = notebookPanel.sessionContext; + var sessionContext = notebookPanel.sessionContext; let sessionType = sessionContext.type; + if (vpConfig.extensionType === 'lite') { + // check one more time for jupyter lite + // - it returns 'notebook' when it's console + if (notebookPanel.console !== undefined) { + sessionType = 'console'; + } + } if (sessionType === 'notebook') { var notebook = notebookPanel.content; var notebookModel = notebook.model; var options = { }; - var cellModel = notebookModel.contentFactory.createCell(type, options); - cellModel.value.text = command; - - const newCellIndex = notebook.activeCellIndex + 1; + // LAB 4.x: notebookPanel.content.contentFactory.createCodeCell / createMarkdownCell + // var cellModel = notebookModel.contentFactory.createCell(type, options); // 셀 추가 - notebookModel.cells.insert(newCellIndex, cellModel); - notebook.activeCellIndex = newCellIndex; - - var cell = notebook.activeCell; + // LAB 4.x: insert cell api 변경 + // notebookModel.cells.insert(newCellIndex, cellModel); + // notebook.activeCellIndex = newCellIndex; + NotebookActions.insertBelow(notebook); + try { + NotebookActions.changeCellType(notebook, type); + } catch(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + } + // let activeCellIndex = notebook.activeCellIndex; + // let cell = notebookModel.cells.model.getCell(activeCellIndex); + // cell.setSource(command); + let activeCell = notebook.activeCell; + activeCell.model.sharedModel.setSource(command); + if (exec == true) { try{ NotebookActions.run(notebook, sessionContext); @@ -265,9 +300,27 @@ define([ } } } else if (sessionType === 'console') { - var console = notebookPanel.content; - var cellModel = console.contentFactory.createCell(type, {}); - cellModel.value.text = command; + // LAB 4.x: notebookPanel.console.contentFactory.createCodeCell + // var console = notebookPanel.content; + // var cellModel = console.contentFactory.createCell(type, {}); + var labConsole = notebookPanel.content; + var widget = labConsole.widgets[0]; + if (type === 'markdown') { + // add # before the lines + command = '#' + command.split('\n').join('\n#'); + } + + // LAB 4.x: console insert and run + widget.promptCell.model.sharedModel.setSource(command) + // execute or not + if (exec == true) { + // console allow only code cell + // var cellModel = widget.createCodeCell(); + // cellModel.model.value.text = command; + // widget.addCell(cellModel); + // widget._execute(cellModel); + widget.execute() + } } } else { // No session found @@ -282,12 +335,15 @@ define([ } else if (vpConfig.extensionType === 'colab') { // CHROME: TODO: - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: TODO: - let activeCell = notebookPanel.content.activeCell; - let activeCellTop = $(activeCell.node).position().top; - // scroll to current cell - $(vpKernel.getLabNotebookPanel().content.activeCell.node)[0].scrollIntoView(true); + let sessionType = notebookPanel.sessionContext.type; + if (sessionType === 'notebook') { + let activeCell = notebookPanel.content.activeCell; + let activeCellTop = $(activeCell.node).position().top; + // scroll to current cell + $(notebookPanel.content.activeCell.node)[0].scrollIntoView(true); + } } com_util.renderSuccessMessage('Your code is successfully generated.'); diff --git a/visualpython/js/com/com_util.js b/visualpython/js/com/com_util.js index 2afb8190..5f63ecf7 100644 --- a/visualpython/js/com/com_util.js +++ b/visualpython/js/com/com_util.js @@ -245,6 +245,31 @@ define([ return label; } + /** + * Pack the code with ignoring the warnings + * @param {*} code + */ + var ignoreWarning = function(code) { + var convertedCode = new com_String(); + convertedCode.appendLine('import warnings'); + convertedCode.appendLine('from IPython.display import display'); + convertedCode.appendLine('with warnings.catch_warnings():'); + convertedCode.appendLine(" warnings.simplefilter('ignore')"); + let codeLines = code.split('\n'); + codeLines.forEach((line, idx) => { + if (idx > 0) { + convertedCode.appendLine(); + } + if (codeLines.length == idx + 1) { + // last line + convertedCode.appendFormat(" display({0})", line); + } else { + convertedCode.appendFormat(" {0}", line); + } + }); + return convertedCode.toString(); + } + //============================================================================ // Cross-browser RegEx Split //============================================================================ @@ -377,6 +402,7 @@ define([ setIsAPIListRunCode: setIsAPIListRunCode, getIsAPIListRunCode: getIsAPIListRunCode, safeString: safeString, + ignoreWarning: ignoreWarning, regex_split: regex_split } diff --git a/visualpython/js/com/component/AlertModal.js b/visualpython/js/com/component/AlertModal.js index 6931eef2..813b7f9c 100644 --- a/visualpython/js/com/component/AlertModal.js +++ b/visualpython/js/com/component/AlertModal.js @@ -21,9 +21,14 @@ define([ /** * AlertModal + * com_util.renderAlertModal(title, detail); + * - title: string + * - detail: object + * - content: string + * - type: text / code */ class AlertModal extends Component { - constructor(title, detail='') { + constructor(title, detail={ content:'', type:'text' }) { super($('body'), { title: title, detail: detail }); } @@ -44,9 +49,16 @@ define([ // set title $(this.wrapSelector('.vp-alertModal-titleStr')).text(this.state.title); - // set detail as tooltip - if (this.state.detail) { - $(this.wrapSelector('.vp-alertModal-titleStr')).attr({ title: this.state.detail }); + // set detail + let { content='', type='text' } = this.state.detail; + if (content !== '') { + if (type === 'code') { + $(this.wrapSelector('.vp-alertModal-detailStr')).html('

' + content + '

') + } else { + $(this.wrapSelector('.vp-alertModal-detailStr')).text(content); + } + } else { + $(this.wrapSelector('.vp-alertModal-detailStr')).hide(); } } diff --git a/visualpython/js/com/component/DataSelector.js b/visualpython/js/com/component/DataSelector.js index 4da01526..1b64b9ee 100644 --- a/visualpython/js/com/component/DataSelector.js +++ b/visualpython/js/com/component/DataSelector.js @@ -50,14 +50,22 @@ define([ super._init(); this.prop = { - type: 'data', // selector type : data / column + type: 'data', // selector type : data / column / ml pageThis: null, // target's page object id: '', // target id value: null, // pre-defined value + withPopup: true, // with filter button to show simple subset popup finish: null, // callback after selection (value, dtype) select: null, // callback after selection from suggestInput (value, dtype) allowDataType: null, // list of allowed data types + dataCategory: null, // list of data category (use it for ml categories) + columnSelection: 'multiple', // single/multi : allowed column selection + returnFrameType: '', // DataFrame/Series : required data type for DataFrame -> Series/DataFrame operation + // if Series, only one column selection is allowed and returns Series + // if DataFrame, always returns DataFrame + // if Empty(=== ''), if one column selected, returns Series / others, returns DataFrame // additional options + boxClasses: '', classes: '', attrs: '', placeholder: 'Select variable', @@ -69,9 +77,23 @@ define([ // If null, define default allow data type if (!this.prop.allowDataType) { // default allow data types - this.prop.allowDataType = [ - 'DataFrame', 'Series', 'ndarray', 'list', 'dict' - ] + if (this.prop.type === 'ml') { + this.prop.allowDataType = vpConfig.getMLDataTypes(); + } else { + this.prop.allowDataType = [ + 'DataFrame', 'Series', 'ndarray', 'list', 'dict' + ] + } + } + if (!this.prop.dataCategory) { + if (this.prop.type === 'ml') { + this.prop.dataCategory = vpConfig.getMLCategories(); + } else { + this.prop.dataCategory = this.prop.allowDataType; + } + } + if (this.prop.returnFrameType === 'Series') { + this.prop.columnSelection = 'single'; // only single selection allowed } this.state = { @@ -84,6 +106,7 @@ define([ slicingEnd1: '', slicingStart2: '', slicingEnd2: '', + singleColumn: '', ndRowType: 'slicing', ndColType: 'slicing', useIndex: false, @@ -95,7 +118,7 @@ define([ } this._target = null; - if (this.prop.pageThis) { + if (this.prop.pageThis && this.prop.id !== '') { this._target = this.prop.pageThis.wrapSelector('#' + this.prop.id); } @@ -183,7 +206,7 @@ define([ autoFocus: true, minLength: 0, source: function (req, res) { - var srcList = varList; + var srcList = varList.filter(obj => that.prop.allowDataType.includes(obj.dtype)); var returlList = new Array(); for (var idx = 0; idx < srcList.length; idx++) { // srcList as object array @@ -228,6 +251,8 @@ define([ $(this).select(); $(this).autocomplete('search', $(this).val()); }).autocomplete('instance')._renderItem = function(ul, item) { + $(ul).addClass('vp-scrollbar-vertical'); + $(ul).css('max-height', '250px'); return $('

').attr('data-value', item.value) .append(`

${item.label} | ${item.dtype}

`) .appendTo(ul); @@ -278,7 +303,11 @@ define([ that.renderVariableBox(that._varList.filter(obj => !that.prop.allowDataType.includes(obj.dtype))); } else { // filter variable list - that.renderVariableBox(that._varList.filter(obj => obj.dtype == type)); + if (that.prop.type === 'ml') { + that.renderVariableBox(that._varList.filter(obj => vpConfig.getMLDataDict(type).includes(obj.dtype))); + } else { + that.renderVariableBox(that._varList.filter(obj => obj.dtype == type)); + } } }); @@ -358,20 +387,48 @@ define([ templateForTarget() { let value = this.prop.value; if (value == undefined) { - value = this.prop.pageThis.state[this.prop.id] || ''; + if (this.prop.id !== '') { + value = this.prop.pageThis.state[this.prop.id] || ''; + } else { + value = ''; + } } + if (this.prop.withPopup === true) { + return ` +

+ + + + + +

+ `; + } else { + return ` +

+ +

+ `; + } + } + + templateForSingleSelector() { return ` -

- - - - - +

+ Use Index +

+ Column +

- `; +

+ ` } templateForMultiSelector() { @@ -511,7 +568,7 @@ define([ let varTags = new com_String(); let types = [ 'All', - ...this.prop.allowDataType, + ...this.prop.dataCategory, 'Others' ]; // Add Data Types to filter @@ -532,10 +589,17 @@ define([ let that = this; let varTags = new com_String(); varTags = new com_String(); - varList && varList.forEach((obj, idx) => { - varTags.appendFormatLine('

{4}

' - , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label); - }); + if (this.prop.type === 'ml') { + varList && varList.forEach((obj, idx) => { + varTags.appendFormatLine('

{4} | {5}

' + , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label, obj.dtype); + }); + } else { + varList && varList.forEach((obj, idx) => { + varTags.appendFormatLine('

{4}

' + , 'vp-ds-var-item', (that.state.data == obj.value?'selected':''), obj.dtype, obj.dtype, obj.label); + }); + } $(this.wrapSelector('.vp-ds-variable-box')).html(varTags.toString()); // focus on selected item @@ -557,22 +621,93 @@ define([ switch (dataType) { case 'DataFrame': - // render option page - $(this.wrapSelector('.vp-ds-option-inner-box')).html(this.templateForMultiSelector()); // column selecting - this._columnSelector = new MultiSelector(this.wrapSelector('.vp-ds-df-multiselector'), - { mode: 'columns', parent: [data], selectedList: this.state.indexing, allowAdd: true } - ); - + if (this.prop.columnSelection === 'single') { + // render option page + $(this.wrapSelector('.vp-ds-option-inner-box')).html(this.templateForSingleSelector()); + // bind column source + // com_generator.vp_bindColumnSource(this, 'data', ['singleColumn'], 'select', false, false); + // vp_bindColumnSource(pageThis, targetId, columnInputIdList, tagType="input", columnWithEmpty=false, columnWithIndex=false) + const columnInputIdList = ['singleColumn']; + if (data === '') { + // reset with no source + columnInputIdList && columnInputIdList.forEach(columnInputId => { + let defaultValue = that.state[columnInputId]; + if (defaultValue === null || defaultValue === undefined) { + defaultValue = ''; + } + // option tags + var tag = $('').attr({ + 'id': columnInputId, + 'class': 'vp-select vp-state' + }); + $(that.wrapSelector('#' + columnInputId)).replaceWith(function() { + return $(tag); + }); + }); + return ; + } + // get result and show on detail box + vpKernel.getColumnList(data).then(function(resultObj) { + try { + let { result, type, msg } = resultObj; + var { list } = JSON.parse(result); + + // columns using suggestInput + columnInputIdList && columnInputIdList.forEach((columnInputId, idx) => { + let defaultValue = that.state[columnInputId]; + if (defaultValue === null || defaultValue === undefined) { + defaultValue = ''; + } + // create tag + var tag = $('').attr({ + 'id': columnInputId, + 'class': 'vp-select vp-state w150' + }); + // make tag + list.forEach((listVar, idx) => { + var option = document.createElement('option'); + $(option).attr({ + 'value':listVar.value, + 'text':listVar.label, + 'data-type':listVar.dtype + }); + // cell metadata test : defaultValue as selected + if (listVar.value === defaultValue) { + $(option).prop('selected', true); + } + option.append(document.createTextNode(listVar.label)); + $(tag).append(option); + }); + $(that.wrapSelector('#' + columnInputId)).replaceWith(function() { + return $(tag); + }); + $(that.wrapSelector('#' + columnInputId)).trigger('change'); + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, 'com_generator - bindColumnSource error ', err) + }); + } catch (e) { + vpLog.display(VP_LOG_TYPE.ERROR, 'com_generator - bindColumnSource: not supported data type. ', e); + } + }); + } else { + // render option page + $(this.wrapSelector('.vp-ds-option-inner-box')).html(this.templateForMultiSelector()); + this._columnSelector = new MultiSelector(this.wrapSelector('.vp-ds-df-multiselector'), + { mode: 'columns', parent: [data], selectedList: this.state.indexing, allowAdd: true } + ); + } // bind event $(this.wrapSelector('#useIndex')).on('change', function() { let checked = $(this).prop('checked'); that.state.useIndex = checked; if (checked === true) { $(that.wrapSelector('.vp-ds-df-multiselector')).hide(); + $(that.wrapSelector('.vp-ds-df-singleselector')).hide(); $(that.wrapSelector('.vp-ds-df-index-box')).show(); } else { $(that.wrapSelector('.vp-ds-df-multiselector')).show(); + $(that.wrapSelector('.vp-ds-df-singleselector')).show(); $(that.wrapSelector('.vp-ds-df-index-box')).hide(); } }); @@ -686,6 +821,7 @@ define([ let { data, dataType, useIndex, + singleColumn, slicingStart1, slicingEnd1, slicingStart2, slicingEnd2, ndRowType, ndColType @@ -700,21 +836,39 @@ define([ code.append('.index'); } else { // use column selector - if (this._columnSelector != null) { - let result = this._columnSelector.getDataList(); - this.state.indexing = result.map(obj => obj.code); // save state - let columnList = []; - result && result.forEach(obj => { - columnList.push(obj.code); - }); - if (columnList.length > 0) { - if (columnList.length == 1) { - // return as Series - code.appendFormat('[{0}]', columnList.join(', ')); - // change datatype to Series - this.state.returnDataType = 'Series'; - } else { - code.appendFormat('[[{0}]]', columnList.join(', ')); + if (this.prop.columnSelection === 'single') { + // single selector + if (this.prop.returnFrameType === 'DataFrame') { + // return as DataFrame + code.appendFormat('[[{0}]]', singleColumn); + } else { + // return as Series + code.appendFormat('[{0}]', singleColumn); + this.state.returnDataType = 'Series'; + } + } else { + // multiple selector + if (this._columnSelector != null) { + let result = this._columnSelector.getDataList(); + this.state.indexing = result.map(obj => obj.code); // save state + let columnList = []; + result && result.forEach(obj => { + columnList.push(obj.code); + }); + if (columnList.length > 0) { + if (columnList.length == 1) { + if (this.prop.returnFrameType === 'DataFrame') { + // return as DataFrame + code.appendFormat('[[{0}]]', columnList.join(', ')); + } else { + // return as Series + code.appendFormat('[{0}]', columnList.join(', ')); + // change datatype to Series + this.state.returnDataType = 'Series'; + } + } else { + code.appendFormat('[[{0}]]', columnList.join(', ')); + } } } } @@ -724,7 +878,7 @@ define([ case 'list': code.append(data); // start / end value - if ((slicingStart1 && slicingStart1 != '') || (slicingEnd1 && slicingEnd1 != '')) { + if ((slicingStart1 != null && slicingStart1 != '') || (slicingEnd1 != null && slicingEnd1 != '')) { code.appendFormat('[{0}:{1}]', slicingStart1, slicingEnd1); } break; @@ -739,7 +893,7 @@ define([ let colCode = ''; if (ndRowType == 'slicing') { // slicing start / end value - if ((slicingStart1 && slicingStart1 != '') || (slicingEnd1 && slicingEnd1 != '')) { + if ((slicingStart1 != null && slicingStart1 != '') || (slicingEnd1 != null && slicingEnd1 != '')) { rowCode = com_util.formatString('{0}:{1}', slicingStart1, slicingEnd1); } } else { @@ -760,7 +914,7 @@ define([ } if (ndColType == 'slicing') { // slicing start / end value - if ((slicingStart2 && slicingStart2 != '') || (slicingEnd2 && slicingEnd2 != '')) { + if ((slicingStart2 != null && slicingStart2 != '') || (slicingEnd2 != null && slicingEnd2 != '')) { colCode = com_util.formatString('{0}:{1}', slicingStart2, slicingEnd2); } } else { @@ -788,7 +942,7 @@ define([ } } else { // start / end value - if ((slicingStart1 && slicingStart1 != '') || (slicingEnd1 && slicingEnd1 != '')) { + if ((slicingStart1 != null && slicingStart1 != '') || (slicingEnd1 != null && slicingEnd1 != '')) { code.appendFormat('[{0}:{1}]', slicingStart1, slicingEnd1); } } diff --git a/visualpython/js/com/component/FileNavigation.js b/visualpython/js/com/component/FileNavigation.js index ad06869c..506ffc0a 100644 --- a/visualpython/js/com/component/FileNavigation.js +++ b/visualpython/js/com/component/FileNavigation.js @@ -84,6 +84,10 @@ define([ this.pathStackPointer = -1; this.pathStack = []; this.currentFileList = []; + this.selectedExt = ''; + if (this.state.extensions.length > 0) { + this.selectedExt = this.state.extensions[0]; + } this.pathState = { parentPath: '', @@ -120,7 +124,8 @@ define([ var pathType = $(this).attr('data-path'); var dirObj = { direction: NAVIGATION_DIRECTION_TYPE.TO, - destDir: '/' + destDir: '/', + useFunction: false } switch (pathType) { case 'desktop': @@ -142,6 +147,9 @@ define([ case 'drive': dirObj.destDir = "/content/drive/MyDrive"; break; + case 'lite-home': + dirObj.destDir = "/drive"; + break; case '/': default: dirObj.direction = NAVIGATION_DIRECTION_TYPE.TOP; @@ -203,6 +211,8 @@ define([ $(fileNaviBody).find('.fnp-sidebar-menu.colab').show(); } else if (vpConfig.extensionType === 'lab') { $(fileNaviBody).find('.fnp-sidebar-menu.lab').show(); + } else if (vpConfig.extensionType === 'lite') { + $(fileNaviBody).find('.fnp-sidebar-menu.lite').show(); } return fileNaviBody; } @@ -253,6 +263,7 @@ define([ if (allowExtensionList.length > 0 && !allowExtensionList.includes(extension)) { // TODO: alert //vpCommon.renderAlertModal('Not supported file type'); + vpLog.display(VP_LOG_TYPE.ERROR, 'Not supported file type: ' + extension); return; } @@ -273,6 +284,32 @@ define([ // clear body $(this.wrapSelector('.fileNavigationPage-body')).html(''); + /** + * Filter file/dir which included in this.state.extensions + */ + if (Array.isArray(this.state.extensions) && this.state.extensions.length > 0 && this.state.extensions.toString() !== '') { + fileList = fileList.filter((data, index) => { + if (index == 0) { + return true; + } + + if (data.type && data.type == 'dir') { + // if directory, just show + return true; + } else if (data.name) { + var extension = data.name.substring(data.name.lastIndexOf('.') + 1); + // if (that.state.extensions.includes(extension)) { + if (that.selectedExt === '' || extension === that.selectedExt) { + return true; + } else { + return false; + } + } else { + return false; + } + }); + } + // render file items let dirArr = []; let fileArr = []; @@ -404,12 +441,12 @@ define([ page.appendFormatLine('' , 'vp_fileNavigationInput', 'New File Name', this.state.fileName); page.appendFormatLine(''); page.appendFormatLine('', 'vp-filenavi-btn', 'select', 'Select'); @@ -423,6 +460,13 @@ define([ that.handleSelectFile(filePath, fileName); }); + // bind file extension change event + $(this.wrapSelector('#vp_fileNavigationExt')).on('change', function() { + let ext = $(this).val(); + that.selectedExt = ext; + + that.renderFileList(); + }); // bind save cancel event $(this.wrapSelector('.vp-filenavi-btn')).on('click', function() { let menu = $(this).data('menu'); @@ -470,10 +514,10 @@ define([ let that = this; /** Implement after rendering */ // if save mode - if (this.state.type == 'save') { - // render saving box - this.renderSaveBox(); - } + // if (this.state.type == 'save') { + // render saving box + this.renderSaveBox(); + // } // get current path this.getCurrentDirectory().then(function(currentPath) { @@ -520,20 +564,21 @@ define([ //============================================================================ // Set selection result //============================================================================ - if (this.state.type == 'save') { - // add as saving file - this.setSelectedFile(fileInput, pathInput); - } else { - // Manage result using finish function - let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented - let status = true; - let error = null; - vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); - this.state.finish(filesPath, status, error); + this.setSelectedFile(fileInput, pathInput); + // if (this.state.type == 'save') { + // // add as saving file + // this.setSelectedFile(fileInput, pathInput); + // } else { + // // Manage result using finish function + // let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented + // let status = true; + // let error = null; + // vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); + // this.state.finish(filesPath, status, error); - // remove and close file navigation - this.close(); - } + // // remove and close file navigation + // this.close(); + // } } getCurrentDirectory() { return vpKernel.getCurrentDirectory(); @@ -581,31 +626,6 @@ define([ return a - b; }); - /** - * Filter file/dir which included in this.state.extensions - */ - if (Array.isArray(that.state.extensions) && that.state.extensions.length > 0 && that.state.extensions.toString() !== '') { - filtered_varList = filtered_varList.filter((data, index) => { - if (index == 0) { - return true; - } - - if (data.type && data.type == 'dir') { - // if file, just show - return true; - } else if (data.name) { - var extension = data.name.substring(data.name.lastIndexOf('.') + 1); - if (that.state.extensions.includes(extension)) { - return true; - } else { - return false; - } - } else { - return false; - } - }); - } - vpLog.display(VP_LOG_TYPE.DEVELOP, 'FileNavigation - getFileList: ', filtered_varList); var { currentDirStr, currentRelativePathStr } = that.splitPathStrAndSetStack(dirObj, filtered_varList); @@ -626,7 +646,7 @@ define([ let { msg, result } = resultObj; // show error using alert - if (msg.content.evalue) { + if (msg.content && msg.content.evalue) { let resultStr = msg.content.evalue; //t.match(/\[Errno [0-9]+?\] (.*)/)[1] // get error message from traceback diff --git a/visualpython/js/com/component/HelpViewer.js b/visualpython/js/com/component/HelpViewer.js new file mode 100644 index 00000000..84ccfe44 --- /dev/null +++ b/visualpython/js/com/component/HelpViewer.js @@ -0,0 +1,221 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : HelpViewer.js + * Author : Black Logic + * Note : Component > HelpViewer + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 07. 13 + * Change Date : + */ +//============================================================================ +// [CLASS] HelpViewer +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/component/helpViewer.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/component/popupComponent'), // INTEGRATION: unified version of css loader + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/Component', + 'vp_base/js/com/component/LoadingSpinner' +], function(hvHtml, ppCss, com_util, com_Const, com_String, Component, LoadingSpinner) { + + /** + * HelpViewer + */ + class HelpViewer extends Component { + constructor() { + super($(vpConfig.parentSelector), {}, {}); + } + + _init() { + this.position = { + right: 10, top: 120 + }; + this.size = { + width: 500, + height: 500 + }; + } + + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + let that = this; + + $(that.wrapSelector('.vp-popup-maximize')).on('click', function() { + // save position + that.position = $(that.wrapSelector()).position(); + // save size + that.size = { + width: $(that.wrapSelector()).width(), + height: $(that.wrapSelector()).height() + } + // maximize popup + $(that.wrapSelector()).css({ + width: '100%', + height: '100%', + top: 0, + left: 0 + }); + // show / hide buttons + $(this).hide(); + $(that.wrapSelector('.vp-popup-return')).show(); + }); + + // Return operation + $(this.wrapSelector('.vp-popup-return')).on('click', function(evt) { + // return size + $(that.wrapSelector()).css({ + width: that.size.width + 'px', + height: that.size.height + 'px', + top: that.position.top, + left: that.position.left + }); + // show / hide buttons + $(this).hide(); + $(that.wrapSelector('.vp-popup-maximize')).show(); + }); + + $(that.wrapSelector('.vp-popup-close')).on('click', function() { + that.remove(); + }); + + $(that.wrapSelector('.vp-popup-button')).on('click', function() { + var btnType = $(this).data('type'); + switch(btnType) { + case 'cancel': + that.remove(); + break; + } + }); + + // Focus recognization + $(this.wrapSelector()).on('click', function() { + that.focus(); + }); + } + + _bindDraggable() { + var that = this; + let containment = 'body'; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + containment = '#main'; + } + $(this.wrapSelector()).draggable({ + handle: '.vp-popup-title', + containment: containment, + start: function(evt, ui) { + // check focused + $(that.eventTarget).trigger({ + type: 'focus_option_page', + component: that + }); + } + }); + } + + _bindResizable() { + let that = this; + $(this.wrapSelector()).resizable({ + handles: 'all', + start: function(evt, ui) { + // show / hide buttons + $(that.wrapSelector('.vp-popup-return')).hide(); + $(that.wrapSelector('.vp-popup-maximize')).show(); + } + }); + } + + template() { + this.$pageDom = $(hvHtml); + + return this.$pageDom; + } + + render() { + super.render(); + + // set detailed size + $(this.wrapSelector()).css({ + width: this.size.width + 'px', + height: this.size.height + 'px' + }); + + // position + $(this.wrapSelector()).css({ top: this.position.top, right: this.position.right }); + + this._bindDraggable(); + this._bindResizable(); + } + + wrapSelector(selector='') { + var sbSelector = new com_String(); + var cnt = arguments.length; + if (cnt < 2) { + // if there's no more arguments + sbSelector.appendFormat(".vp-popup-frame.{0} {1}", this.uuid, selector); + } else { + // if there's more arguments + sbSelector.appendFormat(".vp-popup-frame.{0}", this.uuid); + for (var idx = 0; idx < cnt; idx++) { + sbSelector.appendFormat(" {0}", arguments[idx]); + } + } + return sbSelector.toString(); + } + + open(content, useHelp=true, importCode='') { + this.show(); + + let that = this; + + let code = content; + if (useHelp === true) { + if (importCode !== '') { + code = importCode + '\n' + `print(help(${content}))`; + } else { + code = `print(help(${content}))`; + } + } + + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); + vpKernel.execute(code).then(function(resultObj) { + let { result } = resultObj; + + $(that.wrapSelector('#helpContent')).text(result); + + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + }).finally(function() { + loadingSpinner.remove(); + }); + + this.focus(); + } + + generateCode() { + return ''; + } + + show() { + $(this.wrapSelector()).show(); + } + + remove() { + $(this.wrapSelector()).remove(); + } + + focus() { + $('.vp-popup-frame').removeClass('vp-focused'); + $('.vp-popup-frame').css({ 'z-index': 1200 }); + $(this.wrapSelector()).addClass('vp-focused'); + $(this.wrapSelector()).css({ 'z-index': 1205 }); // move forward + } + + } + + return HelpViewer; +}); \ No newline at end of file diff --git a/visualpython/js/com/component/InstanceEditor.js b/visualpython/js/com/component/InstanceEditor.js index f687faa2..c3a030c0 100644 --- a/visualpython/js/com/component/InstanceEditor.js +++ b/visualpython/js/com/component/InstanceEditor.js @@ -349,11 +349,15 @@ define([ // add targetid as state if exists if (targetId) { popupState[targetId] = targetCode; + that.state[targetId] = targetCode; + } + if (that.optionPopup) { + that.optionPopup.remove(); + that.optionPopup = null; } that.optionPopup = new LibraryComponent(popupState, { pageThis: that, - useInputVariable: true, targetSelector: that.pageThis.wrapSelector('#' + that.targetId), finish: function(code) { @@ -554,16 +558,5 @@ define([ } } - - - - - - - - - - - return InstanceEditor; }) \ No newline at end of file diff --git a/visualpython/js/com/component/LibraryComponent.js b/visualpython/js/com/component/LibraryComponent.js index c7050460..e02c31f2 100644 --- a/visualpython/js/com/component/LibraryComponent.js +++ b/visualpython/js/com/component/LibraryComponent.js @@ -30,6 +30,7 @@ define([ /** Write codes executed before rendering */ this.config.dataview = false; this.config.sizeLevel = 1; + this.config.helpview = true; this.packageId = this.id; // deep copy package info @@ -46,6 +47,42 @@ define([ return; } this.config.checkModules = ['pd']; + + // set docs link + if (this.package.docs === undefined) { + let docsLink = 'https://pandas.pydata.org/docs/reference/api/pandas.'; + let docsMatchObj = this.package.code.match(/\= (.+)\.(.+)\(/); + if (docsMatchObj) { + let targetType = docsMatchObj[1]; // ${i0} or pd + let method = docsMatchObj[2]; + if (targetType === 'pd') { + docsLink += method + '.html'; + } else { + docsLink += 'DataFrame.' + method + '.html'; + } + this.config.docs = docsLink; + } else { + this.config.docs = 'https://pandas.pydata.org/docs/reference/index.html'; + } + } else { + this.config.docs = this.package.docs; + } + + // set helpview content + let helpMatchObj = this.package.code.match(/\= (.+)\.(.+)\(/); + if (helpMatchObj) { + let helpContent = ''; + let targetType = helpMatchObj[1]; // ${i0} or pd + let method = helpMatchObj[2]; + if (targetType === 'pd') { + helpContent += '_vp_pd.' + method; + } else { + helpContent += '_vp_pd.' + 'DataFrame.' + method; + } + this.config.helpInfo.content = helpContent; + } else { + this.config.helpview = false; + } vpLog.display(VP_LOG_TYPE.DEVELOP, 'loading state', this.state); } diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index d847043b..c2fab057 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -80,8 +80,8 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Perform modeling from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } ] }, 'predict': { @@ -90,7 +90,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -100,7 +100,7 @@ define([ code: '${pred_prob_allocate} = ${model}.predict_proba(${pred_prob_featureData})', description: 'Predict class probabilities for X.', options: [ - { name: 'pred_prob_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_test' }, + { name: 'pred_prob_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_prob_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -110,7 +110,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -125,7 +125,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -134,16 +134,21 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, 'transform': { - ...defaultActions['transform'], - description: 'Transform labels to normalized encoding.' + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -153,12 +158,75 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } } } + if (modelType == 'LabelEncoder') { + actions = { + ...actions, + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Fit Encoder/Scaler to X.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' } + ] + }, + 'fit_transform': { + name: 'fit_transform', + label: 'Fit and transform', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', + description: 'Fit Encoder/Scaler to X, then transform X.', + options: [ + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -181,7 +249,7 @@ define([ code: '${dec_allocate} = ${model}.decision_function(${dec_featureData})', description: 'Compute the decision function of X.', options: [ - { name: 'dec_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'dec_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'dec_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable' } ] } @@ -198,7 +266,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Fit and predict.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -215,7 +283,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Perform clustering from features, or distance matrix.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'fit_predict': { @@ -224,7 +292,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute clusters from a data or distance matrix and predict labels.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -238,7 +306,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Compute clustering.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } ] }, 'predict': { @@ -247,7 +315,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict the closest target data X belongs to.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -257,7 +325,7 @@ define([ code: '${fit_pred_allocate} = ${model}.fit_predict(${fit_pred_featureData})', description: 'Compute cluster centers and predict cluster index for each sample.', options: [ - { name: 'fit_pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] } @@ -271,7 +339,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Compute clustering and transform X to cluster-distance space.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -281,7 +349,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Transform X to a cluster-distance space.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -297,7 +365,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -306,7 +374,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -321,18 +389,18 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, 'fit_transform': { name: 'fit_transform', label: 'Fit and transform', - code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}, ${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'fit_trans_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -342,7 +410,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -352,7 +420,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -366,7 +434,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -375,7 +443,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -385,7 +453,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -395,12 +463,58 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}${fit_targetData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } @@ -415,8 +529,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: '', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, @@ -434,15 +548,67 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ - { name: 'importance_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X_train' }, - { name: 'importance_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y_train' }, - { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, + 'feature_importances': { + name: 'feature_importances', + label: 'Feature importances', + code: "${fi_allocate} = vp_create_feature_importances(${model}, ${fi_featureData}${sort})", + description: 'Allocate feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fi_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'df_i' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true } + ] + }, + 'plot_feature_importances': { + name: 'plot_feature_importances', + label: 'Plot feature importances', + code: "vp_plot_feature_importances(${model}, ${fi_featureData}${sort}${top_count})", + description: 'Draw feature_importances_', + options: [ + { name: 'fi_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true }, + ] } } switch (category) { @@ -517,6 +683,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] @@ -535,8 +714,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -548,7 +727,44 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'], + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'Coefficient': { name: 'coef_', label: 'Coefficient', @@ -568,6 +784,19 @@ define([ ] } } + let svcList = [ + 'DecisionTreeRegressor', + 'RandomForestRegressor', + 'GradientBoostingRegressor', + 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor' + ]; + if (svcList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } break; case 'Classification': infos = { @@ -582,8 +811,8 @@ define([ code: '${cvs_allocate} = cross_val_score(${model}, ${cvs_featureData}, ${cvs_targetData}${scoring}${cv})', description: 'Evaluate a score by cross-validation.', options: [ - { name: 'cvs_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'cvs_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'cvs_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'cvs_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'scoring', component: ['option_select'], usePair: true, type: 'text', options: [ '', @@ -595,7 +824,109 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'] + 'roc_curve': { + name: 'roc_curve', + label: 'ROC Curve', + import: 'from sklearn import metrics', + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.predict_proba(${roc_featureData})[:, 1])\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()", + description: '', + options: [ + { name: 'roc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'roc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'auc': { + name: 'auc', + label: 'AUC', + import: 'from sklearn import metrics', + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.predict_proba(${auc_featureData})[:, 1])', + description: '', + options: [ + { name: 'auc_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } + ] + }, + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, + } + + // feature importances + let clfList = [ + 'DecisionTreeClassifier', + 'RandomForestClassifier', + 'GradientBoostingClassifier', + 'XGBClassifier', + 'LGBMClassifier', + 'CatBoostClassifier', + ] + if (clfList.includes(modelType)) { + infos = { + ...infos, + 'feature_importances': defaultInfos['feature_importances'], + 'plot_feature_importances': defaultInfos['plot_feature_importances'] + } + } + + // use decision_function on ROC, AUC + let decisionFunctionTypes = [ + 'LogisticRegression', 'SVC', 'GradientBoostingClassifier' + ]; + if (decisionFunctionTypes.includes(modelType)) { + infos = { + ...infos, + 'roc_curve': { + ...infos['roc_curve'], + code: "fpr, tpr, thresholds = metrics.roc_curve(${roc_targetData}, ${model}.decision_function(${roc_featureData}))\ + \nplt.plot(fpr, tpr, label='ROC Curve')\ + \nplt.xlabel('Sensitivity')\ + \nplt.ylabel('Specificity')\ + \nplt.show()" + }, + 'auc': { + ...infos['auc'], + code: 'metrics.roc_auc_score(${auc_targetData}, ${model}.decision_function(${auc_featureData}))', + } + } } break; case 'Auto ML': @@ -611,16 +942,18 @@ define([ break; case 'Clustering': infos = { - // 'Size of clusters': { - // name: 'Size of clusters', - // code: "print(f'Size of clusters: {np.bincount(pred)}')", // FIXME: model.cluster_centers_ / use model info or hide it - // options: [] - // } + 'get_params': { + ...defaultInfos['get_params'] + } } if (modelType == 'KMeans') { infos = { ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Return the mean accuracy on the given test data and labels.' + }, 'cluster_centers_': { name: 'cluster_centers', label: 'Cluster centers', @@ -642,11 +975,21 @@ define([ code: "# import\nfrom scipy.cluster.hierarchy import dendrogram, ward\n\nlinkage_array = ward(${dendro_data})\ndendrogram(linkage_array, p=3, truncate_mode='level', no_labels=True)\nplt.show()", description: 'Draw a dendrogram', options: [ - { name: 'dendro_data', label: 'Data', component: ['var_select'], var_type: ['DataFrame'] } + { name: 'dendro_data', label: 'Data', component: ['data_select'], var_type: ['DataFrame'] } ] } } } + + if (modelType == 'GaussianMixture') { + infos = { + ...infos, + 'score': { + ...defaultInfos['score'], + description: 'Compute the per-sample average log-likelihood of the given data X.' + } + } + } break; case 'Dimension Reduction': if (modelType == 'LDA') { @@ -657,8 +1000,8 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData}, ${score_targetData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, - { name: 'score_targetData', label: 'Target Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'y' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'score_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } @@ -686,12 +1029,45 @@ define([ code: '${score_allocate} = ${model}.score(${score_featureData})', description: 'Return the average log-likelihood of all samples.', options: [ - { name: 'score_featureData', label: 'Feature Data', component: ['var_select'], var_type: ['DataFrame', 'Series'], value: 'X' }, + { name: 'score_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, { name: 'score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index ce1ef592..d9d3d9fa 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -45,6 +45,13 @@ define([ //======================================================================== // [CLASS] MultiSelector //======================================================================== + /** + * MultiSelector + * Usage + * this._columnSelector = new MultiSelector(this.wrapSelector('#multi-selector-id'), + { mode: 'columns', parent: [data], selectedList: this.state.indexing, allowAdd: true } + ); + */ class MultiSelector extends Component { /** @@ -62,42 +69,65 @@ define([ // configuration this.config = this.state; - var { mode, type, parent, dataList=[], selectedList=[], includeList=[], excludeList=[], allowAdd=false } = this.config; + var { + mode, type, parent, + dataList=[], selectedList=[], includeList=[], excludeList=[], + allowAdd=false, showDescription=true, + change=null + } = this.config; this.mode = mode; // variable / columns / index / ndarray0 / ndarray1 / methods / data(given data) this.parent = parent; + this.varType = type; // for mode:variable, variable type list to search this.selectedList = selectedList; this.includeList = includeList; this.excludeList = excludeList; - this.allowAdd = allowAdd; + this.allowAdd = allowAdd; // allow adding new item + this.showDescription = showDescription; // show description on the top of the box + + this.change = change; // function (type=('add'|'remove'), list=[]) this.dataList = dataList; // [ { value, code, type }, ... ] this.pointer = { start: -1, end: -1 }; + this.loadDataList(); + } + + render() { + ; + } + + loadDataList() { var that = this; - switch (mode) { + if (this.mode !== 'variable' && this.mode !== 'data') { + if (this.parent == null || this.parent === '' || (Array.isArray(this.parent) && this.parent.length == 0)) { + this._executeCallback([]); + return; + } + } + switch (this.mode) { case 'columns': - this._getColumnList(parent, function(dataList) { + this._getColumnList(this.parent, function(dataList) { that._executeCallback(dataList); }); break; case 'variable': - this._getVariableList(type, function(dataList) { + this._getVariableList(this.varType, function(dataList) { that._executeCallback(dataList); }); break; case 'index': - this._getRowList(parent, function(dataList) { + this._getRowList(this.parent, function(dataList) { that._executeCallback(dataList); }); break; case 'ndarray0': - this._getNdarray(parent, 0, function(dataList) { + this._getNdarray(this.parent, 0, function(dataList) { that._executeCallback(dataList); }); break; case 'ndarray1': - this._getNdarray(parent, 1, function(dataList) { + this._getNdarray(this.parent, 1, function(dataList) { that._executeCallback(dataList); }); break; @@ -107,10 +137,6 @@ define([ } } - // render() { - // ; - // } - _executeCallback(dataList) { if (this.includeList && this.includeList.length > 0) { dataList = dataList.filter(data => this.includeList.includes(data.code)); @@ -156,7 +182,8 @@ define([ ...x, value: x.label, code: x.value, - type: x.dtype + type: x.dtype, + isNumeric: x.is_numeric }; }); callback(colList); @@ -174,7 +201,8 @@ define([ ...x, value: x.label, code: x.value, - type: x.dtype + type: x.dtype, + isNumeric: x.is_numeric }; }); callback(list); @@ -220,7 +248,8 @@ define([ ndList.push({ value: i, code: i, - type: 'int' + type: 'int', + location: i }); } callback(ndList); @@ -235,9 +264,10 @@ define([ } load() { - $(this.frameSelector).html(this.render()); + $(this.frameSelector).html(this.templateForMultiSelector()); this.bindEvent(); this.bindDraggable(); + this._bindItemClickEvent(); } getDataList() { @@ -248,7 +278,7 @@ define([ var name = $(colTags[i]).data('name'); var type = $(colTags[i]).data('type'); var code = $(colTags[i]).data('code'); - if (code) { + if (code != null) { dataList.push({ name: name, type: type, code: code}); } } @@ -256,11 +286,13 @@ define([ return dataList; } - render() { + templateForMultiSelector() { var that = this; var tag = new com_String(); - tag.appendLine('Drag-and-drop columns to right to select.'); + if (this.showDescription === true) { + tag.appendLine('Drag-and-drop columns to right to select.'); + } tag.appendFormatLine('

', APP_SELECT_CONTAINER, this.uuid); // select - left tag.appendFormatLine('

', APP_SELECT_LEFT); @@ -275,6 +307,7 @@ define([ $(this.wrapSelector()).val(value); $(this.wrapSelector()).trigger('change'); }); + vpSearchSuggest.setAutoFocus(false); vpSearchSuggest.setNormalFilter(true); tag.appendLine(vpSearchSuggest.toTagString()); tag.appendFormatLine('') @@ -312,11 +345,13 @@ define([ tag.appendLine('

'); } tag.appendLine('

'); // APP_SELECT_RIGHT + tag.appendLine(''); tag.appendLine('

'); // APP_SELECT_CONTAINER return tag.toString(); } renderSelectionBox(dataList) { + let mode = this.mode; var tag = new com_String(); tag.appendFormatLine('

', APP_SELECT_BOX, 'left', APP_DROPPABLE, 'no-selection vp-scrollbar'); // get data and make draggable items @@ -328,15 +363,29 @@ define([ } else { info = ''; } + let iconStr = ''; + let infoStr = ''; + if (mode === 'columns') { + if (data.isNumeric === true) { + iconStr = ''; + } else { + iconStr = ''; + } + } else if (mode === 'variable') { + infoStr = ` | ${data.type}`; + } // render item box - tag.appendFormatLine('

{7}

' - , APP_SELECT_ITEM, APP_DRAGGABLE, data.location, data.value, data.type, data.code, info, data.value); + tag.appendFormat('

' + , APP_SELECT_ITEM, APP_DRAGGABLE, data.location, data.value, data.type, data.code, info); + tag.appendFormat('{0}{1}{2}', iconStr, data.value, infoStr); + tag.appendLine('

'); }); tag.appendLine('

'); // APP_SELECT_BOX return tag.toString(); } renderSelectedBox(dataList) { + let mode = this.mode; var tag = new com_String(); tag.appendFormatLine('

', APP_SELECT_BOX, 'right', APP_DROPPABLE, 'no-selection vp-scrollbar'); // get data and make draggable items @@ -348,9 +397,22 @@ define([ } else { info = ''; } + let iconStr = ''; + let infoStr = ''; + if (mode === 'columns') { + if (data.isNumeric === true) { + iconStr = ''; + } else { + iconStr = ''; + } + } else if (mode === 'variable') { + infoStr = ` | ${data.type}`; + } // render item box - tag.appendFormatLine('

{8}

' - , APP_SELECT_ITEM, APP_DRAGGABLE, 'added', data.location, data.value, data.type, data.code, info, data.value); + tag.appendFormat('

' + , APP_SELECT_ITEM, APP_DRAGGABLE, 'added', data.location, data.value, data.type, data.code, info); + tag.appendFormat('{0}{1}{2}', iconStr, data.value, infoStr); + tag.appendLine('

'); }); tag.appendLine('

'); // APP_SELECT_BOX return tag.toString(); @@ -378,62 +440,7 @@ define([ // draggable that.bindDraggable(); - }); - - // item indexing - $(this.wrapSelector('.' + APP_SELECT_ITEM)).on('click', function(event) { - var dataIdx = $(this).attr('data-idx'); - var idx = $(this).index(); - var added = $(this).hasClass('added'); // right side added item? - var selector = ''; - - // remove selection for select box on the other side - if (added) { - // remove selection for left side - $(that.wrapSelector('.' + APP_SELECT_ITEM + ':not(.added)')).removeClass('selected'); - // set selector - selector = '.added'; - } else { - // remove selection for right(added) side - $(that.wrapSelector('.' + APP_SELECT_ITEM + '.added')).removeClass('selected'); - // set selector - selector = ':not(.added)'; - } - - if (vpEvent.keyManager.keyCheck.ctrlKey) { - // multi-select - that.pointer = { start: idx, end: -1 }; - $(this).toggleClass('selected'); - } else if (vpEvent.keyManager.keyCheck.shiftKey) { - // slicing - var startIdx = that.pointer.start; - - if (startIdx == -1) { - // no selection - that.pointer = { start: idx, end: -1 }; - } else if (startIdx > idx) { - // add selection from idx to startIdx - var tags = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); - for (var i = idx; i <= startIdx; i++) { - $(tags[i]).addClass('selected'); - } - that.pointer = { start: startIdx, end: idx }; - } else if (startIdx <= idx) { - // add selection from startIdx to idx - var tags = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); - for (var i = startIdx; i <= idx; i++) { - $(tags[i]).addClass('selected'); - } - that.pointer = { start: startIdx, end: idx }; - } - } else { - // single-select - that.pointer = { start: idx, end: -1 }; - // un-select others - $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).removeClass('selected'); - // select this - $(this).addClass('selected'); - } + that._bindItemClickEvent(); }); // item indexing - add all @@ -444,6 +451,8 @@ define([ $(that.wrapSelector('.' + APP_SELECT_ITEM)).addClass('added'); $(that.wrapSelector('.' + APP_SELECT_ITEM + '.selected')).removeClass('selected'); that.pointer = { start: -1, end: -1 }; + + that.change && that.change('add', that.getDataList()); }); // item indexing - add @@ -456,6 +465,8 @@ define([ $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).addClass('added'); $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).removeClass('selected'); that.pointer = { start: -1, end: -1 }; + + that.change && that.change('add', that.getDataList()); }); // item indexing - del @@ -476,6 +487,8 @@ define([ selectedTag.removeClass('added'); selectedTag.removeClass('selected'); that.pointer = { start: -1, end: -1 }; + + that.change && that.change('remove', that.getDataList()); }); // item indexing - delete all @@ -493,12 +506,16 @@ define([ $(that.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('added'); $(that.wrapSelector('.' + APP_SELECT_ITEM + '.selected')).removeClass('selected'); that.pointer = { start: -1, end: -1 }; + + that.change && that.change('remove', that.getDataList()); }); // add new item $(this.wrapSelector('.vp-cs-add-item-btn')).on('click', function(event) { let newItemName = $(that.wrapSelector('.vp-cs-add-item-name')).val(); that._addNewItem(newItemName); + + that.change && that.change('add', that.getDataList()); }); // add new item (by pushing enter key) $(this.wrapSelector('.vp-cs-add-item-name')).on('keyup', function(event) { @@ -508,8 +525,86 @@ define([ if (keycode == 13) { // enter let newItemName = $(this).val(); that._addNewItem(newItemName); + + that.change && that.change('add', that.getDataList()); + } + }); + + // refresh + $(this.wrapSelector('.vp-cs-refresh')).on('click', function(event) { + that.loadDataList(); + }); + + this._bindItemClickEvent(); + } + + _bindItemClickEvent() { + let that = this; + // item indexing + $(this.wrapSelector('.' + APP_SELECT_ITEM)).off('click'); + $(this.wrapSelector('.' + APP_SELECT_ITEM)).on('click', function(event) { + var dataIdx = $(this).attr('data-idx'); + var idx = $(this).index(); + var added = $(this).hasClass('added'); // right side added item? + var selector = ''; + + // remove selection for select box on the other side + if (added) { + // remove selection for left side + $(that.wrapSelector('.' + APP_SELECT_ITEM + ':not(.added)')).removeClass('selected'); + // set selector + selector = '.added'; + } else { + // remove selection for right(added) side + $(that.wrapSelector('.' + APP_SELECT_ITEM + '.added')).removeClass('selected'); + // set selector + selector = ':not(.added)'; + } + + if (vpEvent.keyManager.keyCheck.ctrlKey) { + // multi-select + that.pointer = { start: idx, end: -1 }; + $(this).toggleClass('selected'); + } else if (vpEvent.keyManager.keyCheck.shiftKey) { + // slicing + var startIdx = that.pointer.start; + + if (startIdx == -1) { + // no selection + that.pointer = { start: idx, end: -1 }; + } else if (startIdx > idx) { + // add selection from idx to startIdx + var tags = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); + for (var i = idx; i <= startIdx; i++) { + $(tags[i]).addClass('selected'); + } + that.pointer = { start: startIdx, end: idx }; + } else if (startIdx <= idx) { + // add selection from startIdx to idx + var tags = $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)); + for (var i = startIdx; i <= idx; i++) { + $(tags[i]).addClass('selected'); + } + that.pointer = { start: startIdx, end: idx }; + } + } else { + // single-select + that.pointer = { start: idx, end: -1 }; + // un-select others + $(that.wrapSelector('.' + APP_SELECT_ITEM + selector)).removeClass('selected'); + // select this + $(this).addClass('selected'); } }); + + // item deleting (manually added item only) + $(this.wrapSelector('.vp-cs-del-item')).off('click'); + $(this.wrapSelector('.vp-cs-del-item')).on('click', function(event) { + $(this).closest('.' + APP_SELECT_ITEM).remove(); + that.pointer = { start: -1, end: -1 }; + + that.change && that.change('remove', that.getDataList()); + }); } _addNewItem(newItemName) { @@ -551,6 +646,7 @@ define([ let newItemIndex = this.dataList.length; var targetTag = $(`

${newItemName} +

`); $(targetTag).appendTo( $(this.wrapSelector('.' + APP_SELECT_BOX + '.right')) @@ -560,6 +656,8 @@ define([ $(this.wrapSelector('.' + APP_SELECT_ITEM)).removeClass('selected'); // clear item input $(this.wrapSelector('.vp-cs-add-item-name')).val(''); + // bind click event + this._bindItemClickEvent(); // bind draggable this.bindDraggable(); } @@ -613,6 +711,7 @@ define([ if ($(this).hasClass('right')) { // add $(dropGroup).addClass('added'); + that.change && that.change('add', that.getDataList()); } else { // del $(dropGroup).removeClass('added'); @@ -620,6 +719,7 @@ define([ $(droppedOn).find('.' + APP_SELECT_ITEM).sort(function(a, b) { return ($(b).data('idx')) < ($(a).data('idx')) ? 1 : -1; }).appendTo( $(droppedOn) ); + that.change && that.change('remove', that.getDataList()); } // remove selection $(droppableQuery).find('.selected').removeClass('selected'); diff --git a/visualpython/js/com/component/NumpyComponent.js b/visualpython/js/com/component/NumpyComponent.js index a8b4b7e7..d4bb217f 100644 --- a/visualpython/js/com/component/NumpyComponent.js +++ b/visualpython/js/com/component/NumpyComponent.js @@ -53,6 +53,16 @@ define([ } this.config.checkModules = ['np']; + if (this.package.docs === undefined) { + if (packageName == 'numpy') { + this.config.docs = 'https://numpy.org/doc/'; + } else if (packageName == 'python') { + this.config.docs = 'https://docs.python.org/3.11/library/index.html'; + } + } else { + this.config.docs = this.package.docs; + } + vpLog.display(VP_LOG_TYPE.DEVELOP, 'loading state', this.state); } diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js new file mode 100644 index 00000000..a808315e --- /dev/null +++ b/visualpython/js/com/component/PackageManager.js @@ -0,0 +1,479 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : PackageManager.js + * Author : Black Logic + * Note : Component > PackageManager + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 06. 14 + * Change Date : + */ +//============================================================================ +// [CLASS] PackageManager +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/component/packageManager.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/component/packageManager'), // INTEGRATION: unified version of css loader + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/FileNavigation', + 'vp_base/js/com/component/LoadingSpinner' +], function(ifHtml, ifCss, com_util, com_Const, com_String, com_interface, SuggestInput, PopupComponent, FileNavigation, LoadingSpinner) { + + /** + * PackageManager + */ + class PackageManager extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.name = 'Package Manager'; + this.config.codeview = false; + this.config.dataview = false; + this.config.runButton = false; + this.config.sizeLevel = 1; + + this.state = { + selected: '', + popupType: '', + ...this.state + } + + this.packageLib = {}; + this.packageLibTemplate = { + 'numpy': { pipName: 'numpy' }, + 'pandas': { pipName: 'pandas' }, + 'pyarrow': { pipName: 'pyarrow' }, + 'matplotlib': { pipName: 'matplotlib' }, + 'seaborn': { pipName: 'seaborn' }, + 'plotly': { pipName: 'plotly' }, + 'wordcloud': { pipName: 'wordcloud' }, + 'sklearn': { pipName: 'scikit-learn' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, + 'scipy': { pipName: 'scipy' }, + 'statsmodels': { pipName: 'statsmodels' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, + 'pingouin': { pipName: 'pingouin' }, + 'category_encoders': { pipName: 'category_encoders' }, + 'imblearn': { pipName: 'imblearn' }, + 'xgboost': { pipName: 'xgboost' }, + 'lightgbm': { pipName: 'lightgbm' }, + 'catboost': { pipName: 'catboost' }, + 'autosklearn': { pipName: 'auto-sklearn' }, + 'tpot': { pipName: 'tpot' }, + 'pymupdf': { pipName: 'pymupdf' }, + 'sweetviz': { pipName: 'sweetviz' }, + } + + if (vpConfig.extensionType === 'lite') { + this.packageLibTemplate = { + 'numpy': { pipName: 'numpy' }, + 'pandas': { pipName: 'pandas' }, + 'matplotlib': { pipName: 'matplotlib' }, + 'seaborn': { pipName: 'seaborn' }, + 'plotly': { pipName: 'plotly' }, + 'sklearn': { pipName: 'scikit-learn' }, + 'scikit_posthocs': { pipName: 'scikit-posthocs' }, + 'scipy': { pipName: 'scipy' }, + 'statsmodels': { pipName: 'statsmodels' }, + 'factor_analyzer': { pipName: 'factor-analyzer' }, + 'category_encoders': { pipName: 'category_encoders' }, + 'imblearn': { pipName: 'imblearn' }, + 'xgboost': { pipName: 'xgboost' }, + 'lightgbm': { pipName: 'lightgbm' }, + 'catboost': { pipName: 'catboost' }, + 'autosklearn': { pipName: 'auto-sklearn' }, + 'sweetviz': { pipName: 'sweetviz' }, + } + } + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('.vp-pm-search')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + let that = this; + + // search item + $(document).on('change', this.wrapSelector('.vp-pm-search'), function(evt) { + var value = $(this).val(); + if (value != '') { + $(that.wrapSelector('.vp-pm-item')).hide(); + $(that.wrapSelector('.vp-pm-item')).filter(function() { + let key = $(this).data('key'); + return key.search(value.toLowerCase()) >= 0; + }).show(); + } else { + $(that.wrapSelector('.vp-pm-item')).show(); + } + }); + + // sort menu popup + $(this.wrapSelector('.vp-pm-sort')).on('click', function(evt) { + evt.stopPropagation(); + $(that.wrapSelector('.vp-pm-sort-menu-box')).toggle(); + }); + + // sort item + $(this.wrapSelector('.vp-pm-sort-menu-item')).on('click', function() { + var menu = $(this).data('menu'); + if (menu === 'registered') { + // sort by name + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var keyA = parseInt($(a).data('seq')); + var keyB = parseInt($(b).data('seq')); + return keyA > keyB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'name') { + // sort by name + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var keyA = $(a).data('key'); + var keyB = $(b).data('key'); + return keyA > keyB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'installed') { + // sort by date + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var insA = $(a).data('installed'); + var insB = $(b).data('installed'); + if (insA === insB) { + var keyA = $(a).data('key'); + var keyB = $(b).data('key'); + return keyA > keyB ? 1 : -1 + } + return insA < insB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } else if (menu === 'uninstalled') { + $(that.wrapSelector('.vp-pm-item')).sort(function(a, b) { + var insA = $(a).data('installed'); + var insB = $(b).data('installed'); + if (insA === insB) { + var keyA = $(a).data('key'); + var keyB = $(b).data('key'); + return keyA > keyB ? 1 : -1 + } + return insA > insB ? 1 : -1 + }).appendTo($(that.wrapSelector('.vp-pm-table'))) + } + $(that.wrapSelector('.vp-pm-sort-menu-box')).hide(); + }); + + // reload package list + $(this.wrapSelector('.vp-pm-func-reload')).on('click', function() { + // reset search keyword + $(that.wrapSelector('.vp-pm-search')).val(''); + + // load package list + that.loadPackageList(); + }); + + // add package + $(this.wrapSelector('.vp-pm-add')).on('click', function() { + that.openOptionPopup('add'); + }); + } + + bindItemEvent() { + let that = this; + + // item menu click + $(this.wrapSelector('.vp-pm-item-menu-item')).off('click'); + $(this.wrapSelector('.vp-pm-item-menu-item:not(.disabled)')).on('click', function(evt) { + var menu = $(this).data('menu'); + var item = $(this).closest('.vp-pm-item'); + var key = $(item).data('key'); + if (menu === 'install') { + that.state.selected = key; + that.openOptionPopup('install'); + } else if (menu === 'uninstall') { + var pipName = that.packageLib[key].pipName; + var code = com_util.formatString("!pip uninstall -y {0}", pipName); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip uninstall {0}", pipName); + } + // DEPRECATED: no longer save to block as default + // create block and run it + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); + } else if (menu === 'upgrade') { + var pipName = that.packageLib[key].pipName; + var code = com_util.formatString("!pip install --upgrade {0}", pipName); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip install {0}", pipName); + } + // DEPRECATED: no longer save to block as default + // create block and run it + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + that.loadPackageList(); + } else if (menu === 'delete') { + $(item).remove(); + delete that.packageLib[key]; + vpConfig.removeData('packageList', 'vppackman').then(function() { + vpConfig.setData({ 'packageList': that.packageLib }, 'vppackman'); + that.loadPackageList(); + }); + } + evt.stopPropagation(); + }); + } + + templateForBody() { + return ifHtml; + } + + templateForAddPage() { + return `

+ Package name + + Pip name + +

`; + } + + templateForInstallPage() { + return `

+ Version selection +

+ Latest version +

+ Specified version + +

`; + } + + openOptionPopup(type) { + let that = this; + let title = ''; + let size = { width: 400, height: 250 }; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); + + this.state.popupType = type; + switch (type) { + case 'add': + title = 'Add new package to manage' + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForAddPage()); + break; + case 'install': + title = 'Install package' + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForInstallPage()); + + $(this.wrapSelector('.vp-inner-popup-body input[name="ver_select"]')).on('change', function() { + let checkedType = $(this).val(); + if (checkedType === 'specified') { + $(that.wrapSelector('.vp-inner-popup-version')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-version')).prop('disabled', true); + } + }); + break; + } + + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); + + // show popup box + this.openInnerPopup(title); + } + + handleInnerOk() { + switch (this.state.popupType) { + case 'add': + var packName = $(this.wrapSelector('.vp-inner-popup-package')).val(); + var pipName = $(this.wrapSelector('.vp-inner-popup-pip')).val(); + if (pipName === '') { + pipName = packName; + } + this.packageLib[packName] = { pipName: pipName }; + vpConfig.setData({ 'packageList': this.packageLib }, 'vppackman'); + + // load package list + this.loadPackageList(); + break; + case 'install': + let versionType = $(this.wrapSelector('.vp-inner-popup-body input[name="ver_select"]:checked')).val(); + var pipName = this.packageLib[this.state.selected].pipName; + var code = com_util.formatString("!pip install {0}", pipName); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip install {0}", pipName); + } + if (versionType === 'specified') { + // specified version + let version = $(this.wrapSelector('.vp-inner-popup-version')).val(); + if (version && version !== '') { + code = com_util.formatString("!pip install {0}=={1}", pipName, version); + if (vpConfig.extensionType === 'lite') { + code = com_util.formatString("%pip install {0}=={1}", pipName, version); + } + } else { + $(this.wrapSelector('.vp-inner-popup-version')).focus(); + return false; + } + } + // DEPRECATED: no longer save to block as default + // create block and run it + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Package Manager'); + + // load package list + this.loadPackageList(); + break; + } + + this.closeInnerPopup(); + } + + render() { + super.render(); + + let that = this; + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); + vpConfig.getData('', 'vppackman').then(function(savedData) { + // Reset abnormal data + if (savedData == undefined || savedData.packageList === undefined) { + savedData = { packageList: JSON.parse(JSON.stringify(that.packageLibTemplate)) }; + vpConfig.setData({ savedData }, 'vppackman'); + } + + that.packageLib = { + ...savedData.packageList + }; + loadingSpinner.remove(); + // load package list + that.loadPackageList(); + }).catch(function(err) { + vpLog.display(VP_LOG_TYPE.ERROR, err); + + that.packageLib = { + ...that.packageLibTemplate + }; + loadingSpinner.remove(); + // load package list + that.loadPackageList(); + }); + } + + /** + * + * @param {String} key + * @param {Object} info installed, version, path + * @param {number} index sequence of initial package list + * @returns + */ + renderPackageItem(key, info, index) { + var item = new com_String(); + item.appendFormatLine('

', 'vp-pm-item', key, info.installed===true?'1':'0', index); + item.appendFormatLine('

', 'vp-pm-item-header', (info.path?info.path:'')); + item.appendFormatLine('{0}', key); + if (info.installed === true) { + item.appendFormatLine('{0}', info.version); + } else { + item.appendLine('Not installed'); + } + item.appendFormatLine('

', 'vp-pm-item-menu'); + // install + item.appendFormatLine('

' + , 'vp-pm-item-menu-item', 'install', 'Install'); + if (vpConfig.extensionType !== 'lite') { + // upgrade + item.appendFormatLine('

' + , 'vp-pm-item-menu-item', (info.installed===true?'':'disabled'), 'upgrade', 'Upgrade'); + // uninstall + item.appendFormatLine('

' + , 'vp-pm-item-menu-item', (info.installed===true?'':'disabled'), 'uninstall', 'Uninstall'); + } + item.appendLine('

'); // end of vp-pm-item-menu + item.appendLine('

'); // end of vp-pm-item-header + // delete button + item.appendLine(''); + item.appendLine('

'); // end of vp-pm-item + return item.toString(); + } + + generateCode() { + return ''; + } + + loadPackageList() { + var that = this; + // import importlib + // _vp_pack = importlib.import_module('numpy') + // print(_vp_pack.__version__) + + // clear table except head + $(this.wrapSelector('.vp-pm-table')).html(''); + + let packageList = Object.keys(this.packageLib); + + // set auto search + let searchInput = new SuggestInput(); + searchInput.addClass('vp-pm-search vp-input'); + searchInput.setPlaceholder("Search"); + searchInput.setSuggestList(function () { return packageList; }); + $(this.wrapSelector('.vp-pm-search')).replaceWith(searchInput.toTagString()); + + let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-popup-body'))); + vpKernel.getPackageList(packageList).then(function(resultObj) { + let { result } = resultObj; + let packageInfo = JSON.parse(result); + + // load code list + var innerFuncCode = new com_String(); + Object.keys(packageInfo).forEach((key, idx) => { + let info = packageInfo[key]; // installed, version, path + if (info) { + var item = that.renderPackageItem(key, info, idx); + innerFuncCode.append(item); + } + }); + $(that.wrapSelector('.vp-pm-table')).html(innerFuncCode.toString()); + + // bind item menu event + that.bindItemEvent(); + }).catch(function() { + + }).finally(function() { + loadingSpinner.remove(); + }); + } + + } + + return PackageManager; +}); \ No newline at end of file diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 67971c26..60b37260 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -15,7 +15,7 @@ // CHROME: notebook/js/codemirror-ipython (function(mod) { if (typeof exports == "object" && typeof module == "object"){ // CommonJS - if (vpExtType === 'lab') { + if (vpExtType === 'lab' || vpExtType === 'lite') { mod(require("codemirror/lib/codemirror"), require("codemirror/mode/python/python") ); @@ -62,6 +62,7 @@ define([ '../com_interface', './Component', './DataSelector', + './HelpViewer', /** codemirror */ 'codemirror/lib/codemirror', @@ -70,7 +71,7 @@ define([ 'codemirror/addon/display/autorefresh', // 'notebook/js/codemirror-ipython' // LAB: disabled to avoid error FIXME: ], function(popupComponentHtml, popupComponentCss - , com_util, com_Const, com_String, com_interface, Component, DataSelector, codemirror + , com_util, com_Const, com_String, com_interface, Component, DataSelector, HelpViewer, codemirror ) { 'use strict'; @@ -81,7 +82,7 @@ define([ * Component */ class PopupComponent extends Component { - constructor(state={ config: { id: 'popup', name: 'Popup title', path: 'path/file' }}, prop={}) { + constructor(state={ config: { id: 'popup', name: 'Popup title', path: 'path/file', category: '' }}, prop={}) { // CHROME: FIXME: #site -> .notebook-vertical // super($('#site'), state, prop); super($(vpConfig.parentSelector), state, prop); @@ -98,7 +99,7 @@ define([ this.category = category; this.config = { - sizeLevel: 0, // 0: 400x400 / 1: 500x500 / 2: 600x500 / 3: 750x500 + sizeLevel: 0, // 0: 400x400 / 1: 500x500 / 2: 600x500 / 3: 750x500 / 4: executeMode: 'code', // cell execute mode // show header bar buttons installButton: false, // install button (#popupInstall) // FIXME: after creating packagemanager, deprecate it @@ -107,14 +108,26 @@ define([ // show view box codeview: true, dataview: true, + helpview: false, // show footer runButton: true, footer: true, + // run type : run / run-save / add + runType: 'run', + // position and size position: { right: 10, top: 120 }, size: { width: 400, height: 550 }, + // additional modes and infos saveOnly: false, // apply mode + resizable: true, + autoScroll: true, checkModules: [], // module aliases or function names docs: 'https://visual-python.gitbook.io/docs/getting-started/welcome-to-visual-python', + helpInfo: { + content: '', // method to show using help() ex) pd.DataFrame + useHelp: true, // custom text to show on help viewer + importCode: '' // import code for help + }, ...restConfig }; @@ -145,8 +158,9 @@ define([ } this.cmCodeview = null; - this.cmCodeList = []; + + this.helpViewer = undefined; } wrapSelector(selector='') { @@ -288,8 +302,31 @@ define([ } } + addCheckModules(module) { + if (this.config.checkModules.includes(module)) { + return ; + } + this.config.checkModules.push(module); + } + _bindEvent() { var that = this; + // Popup click / focus event + $(this.wrapSelector()).on('click focus', function(evt) { + // Close on blur + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-codeview-box') + && $(that.wrapSelector('.vp-popup-codeview-box')).find(evt.target).length == 0) { + that.closeView('code'); + } + } + if ($(that.wrapSelector('.vp-popup-button')).find(evt.target).length == 0) { + if (!$(evt.target).hasClass('vp-popup-dataview-box') + && $(that.wrapSelector('.vp-popup-dataview-box')).find(evt.target).length == 0) { + that.closeView('data'); + } + } + }); // Close popup event $(this.wrapSelector('.vp-popup-close')).on('click', function(evt) { if (that.getTaskType() === 'task') { @@ -307,10 +344,8 @@ define([ }); // Toggle operation (minimize) $(this.wrapSelector('.vp-popup-toggle')).on('click', function(evt) { - $(that.eventTarget).trigger({ - type: 'close_option_page', - component: that - }); + evt.stopPropagation(); + that.hide(); }); // Maximize operation $(this.wrapSelector('.vp-popup-maximize')).on('click', function(evt) { @@ -351,6 +386,9 @@ define([ // add install codes var codes = that.generateInstallCode(); codes && codes.forEach(code => { + if (vpConfig.extensionType === 'lite') { + code = code.replace('!', '%'); + } com_interface.insertCell('code', code, true, that.getSigText()); }); }); @@ -390,6 +428,11 @@ define([ that._saveSingleState($(this)[0]); }); + // click input box with selection + $(document).on('focus', this.wrapSelector('input'), function() { + $(this).select(); + }); + // Click buttons $(this.wrapSelector('.vp-popup-button')).on('click', function(evt) { var btnType = $(this).data('type'); @@ -410,6 +453,14 @@ define([ } evt.stopPropagation(); break; + case 'help': + if ($(that.wrapSelector('.vp-popup-dataview-box')).is(':hidden')) { + that.openView('help'); + } else { + that.closeView('help'); + } + evt.stopPropagation(); + break; case 'cancel': if (that.getTaskType() === 'task') { $(that.eventTarget).trigger({ @@ -425,33 +476,75 @@ define([ } break; case 'run': - let result = that.run(); + var result = that.run(); if (result) { - that.save(); + // that.save(); + that.close(); } break; case 'show-detail': $(that.wrapSelector('.vp-popup-run-detailbox')).show(); + // set run button + vpConfig.getData('runType', 'vpcfg').then(function(data) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); + that._setDefaultRunType(data); + }); evt.stopPropagation(); break; case 'save': that.save(); break; + case 'run-save': + var result = that.run(); + if (result) { + that.save(); + } + break; + case 'add': + var result = that.run(false); + if (result) { + // that.save(); + that.close(); + } + break; } }); + // Click detail radio + $(this.wrapSelector('.vp-popup-run-type')).on('click', function(evt) { + let runType = $(that.wrapSelector('.vp-popup-run-type[name=runType]:checked')).val(); + // save as vpConfig + vpConfig.setData({runType: runType}, 'vpcfg'); + that._setDefaultRunType(runType); + }); // Click detail buttons - $(this.wrapSelector('.vp-popup-detail-button')).on('click', function(evt) { + $(this.wrapSelector('.vp-popup-detail-action-button')).on('click', function(evt) { var btnType = $(this).data('type'); + vpConfig.setData({runType: btnType}, 'vpcfg'); + that._setDefaultRunType(btnType); // run, run-save, add switch(btnType) { - case 'apply': - that.save(); + // case 'apply': + // that.save(); + // break; + case 'run': + var result = that.run(); + if (result) { + // that.save(); + that.close(); + } break; - case 'add': - let result = that.run(false); + case 'run-save': + var result = that.run(); if (result) { that.save(); } break; + case 'add': + var result = that.run(false); + if (result) { + // that.save(); + that.close(); + } + break; } }); // Close event for inner popup @@ -471,6 +564,14 @@ define([ } }); + // blur on code, dataview + $(this.wrapSelector('.vp-popup-codeview-box')).on('hide', function() { + that.closeView('code'); + }); + $(this.wrapSelector('.vp-popup-dataview-box')).on('hide', function() { + that.closeView('data'); + }); + // focus on data selector input $(this.wrapSelector('.vp-data-selector')).on('focus', function(evt) { @@ -484,12 +585,13 @@ define([ _unbindEvent() { $(document).off('change', this.wrapSelector('.vp-state')); + $(document).off('focus', this.wrapSelector('input')); } _bindDraggable() { var that = this; let containment = 'body'; - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { containment = '#main'; } $(this.wrapSelector()).draggable({ @@ -536,7 +638,11 @@ define([ this.$pageDom = $(popupComponentHtml.replaceAll('${vp_base}', com_Const.BASE_PATH)); // set title // this.$pageDom.find('.vp-popup-title').text(this.category + ' > ' + this.name); - this.$pageDom.find('.vp-popup-title').html(`${this.name}`); + if (this.category && this.category !== '') { + this.$pageDom.find('.vp-popup-title').html(`${this.name}`); + } else { + this.$pageDom.find('.vp-popup-title').html(`${this.name}`); + } // set body let bodyTemplate = this.templateForBody(); // CHROME: check url keyword and replace it @@ -562,8 +668,8 @@ define([ let { installButton, importButton, packageButton, - codeview, dataview, runButton, footer, - sizeLevel, position, docs + codeview, dataview, helpview, runButton, footer, + sizeLevel, position, autoScroll, docs } = this.config; // apply link to docs @@ -598,6 +704,9 @@ define([ if (!dataview) { $(this.wrapSelector('.vp-popup-button[data-type="data"]')).hide(); } + if (!helpview) { + $(this.wrapSelector('.vp-popup-button[data-type="help"]')).hide(); + } // run button if (!runButton) { @@ -611,7 +720,15 @@ define([ $(this.wrapSelector('.vp-popup-body')).css({ 'height': 'calc(100% - 30px)' }) + } else { + // set run button + let that = this; + vpConfig.getData('runType', 'vpcfg').then(function(data) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); + that._setDefaultRunType(data); + }); } + // popup-frame size switch (sizeLevel) { @@ -625,10 +742,17 @@ define([ this.config.size = { width: 760, height: 550 }; break; case 4: + this.config.size = { width: 840, height: 550 }; + break; + case 5: this.config.size = { width: 1064, height: 550 }; break; } + if (autoScroll) { + $(this.wrapSelector('.vp-popup-body')).addClass('vp-scrollbar'); + } + // set detailed size $(this.wrapSelector()).css({ width: this.config.size.width + 'px', @@ -644,7 +768,9 @@ define([ } this._bindDraggable(); - this._bindResizable(); + if (this.config.resizable) { + this._bindResizable(); + } } templateForInnerPopup() { @@ -704,7 +830,37 @@ define([ } loadState() { - /** Implementation needed */ + vpLog.display(VP_LOG_TYPE.DEVELOP, this.state); + + let that = this; + Object.keys(this.state).forEach(key => { + if (key && key !== '' && key !== 'config') { + let tag = $(that.wrapSelector('#' + key) + ', ' + that.wrapSelector('input[name="' + key + '"]')); + let tagName = $(tag).prop('tagName'); + let savedValue = that.state[key]; + switch(tagName) { + case 'INPUT': + let inputType = $(tag).prop('type'); + if (inputType === 'text' || inputType === 'number' || inputType === 'hidden') { + $(tag).val(savedValue); + break; + } + if (inputType === 'checkbox') { + $(tag).prop('checked', savedValue); + break; + } + if (inputType === 'radio') { + $(tag).filter(`[value="${savedValue}"]`).prop('checked', true); + } + break; + case 'TEXTAREA': + case 'SELECT': + default: + $(tag).val(savedValue); + break; + } + } + }); } saveState() { @@ -716,9 +872,27 @@ define([ vpLog.display(VP_LOG_TYPE.DEVELOP, 'savedState', that.state); } - _saveSingleState(tag) { + _setDefaultRunType(runType) { + if (runType == undefined || runType == null || runType === '') { + runType = 'run'; // default = run + } + this.config.runType = runType; + $(this.wrapSelector(`.vp-popup-run-type[value="${runType}"]`)).prop('checked', true); + $(this.wrapSelector('.vp-popup-run-button')).attr('data-type', runType); + let runTitle = 'Run code'; // when runType is 'run' + switch (runType) { + case 'run-save': + runTitle = 'Save to block & Run code'; + break; + case 'add': + runTitle = 'Add code to cell'; + break; + } + $(this.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + } + + _getTagValue(tag) { let id = tag.id; - let customKey = $(tag).data('key'); let tagName = $(tag).prop('tagName'); // returns with UpperCase let newValue = ''; switch(tagName) { @@ -726,6 +900,15 @@ define([ let inputType = $(tag).prop('type'); if (inputType == 'checkbox') { newValue = $(tag).prop('checked'); + } else if (inputType == 'radio') { + let radioGroup = $(tag).prop('name'); + let checked = $(tag).prop('checked'); + if (checked === true) { + id = radioGroup; + newValue = $(tag).val(); + } else { + return ; + } } else { // inputType == 'text' || inputType == 'number' || inputType == 'hidden' || inputType == 'color' || inputType == 'range' newValue = $(tag).val(); @@ -740,12 +923,21 @@ define([ } break; } + return newValue; + } + + _saveSingleState(tag) { + let id = tag.id; + let customKey = $(tag).data('key'); + let newValue = this._getTagValue(tag); // if custom key is available, use it if (customKey && customKey != '') { // allow custom key until level 2 let customKeys = customKey.split('.'); - if (customKeys.length == 2) { + if (customKeys.length === 3) { + this.state[customKeys[0]][customKeys[1]][customKeys[2]] = newValue; + } else if (customKeys.length === 2) { this.state[customKeys[0]][customKeys[1]] = newValue; } else { this.state[customKey] = newValue; @@ -780,7 +972,7 @@ define([ */ checkRequiredOption() { let requiredFilled = true; - let requiredTags = $(this.wrapSelector('input[required=true]') + ',' + this.wrapSelector('input[required=required]')); + let requiredTags = $(this.wrapSelector('input[required=true]:visible') + ',' + this.wrapSelector('input[required=required]:visible')); vpLog.display(VP_LOG_TYPE.DEVELOP, 'checkRequiredOption', this, requiredTags); @@ -796,9 +988,23 @@ define([ } } + if (requiredFilled) { + requiredFilled = this.checkBeforeRun(); + } + return requiredFilled; } + /** + * Check options and do some operation before run + * @returns {boolean} check options and returns true or false (false will stop code running) + */ + checkBeforeRun() { + /** Implementation needed */ + + return true; + } + checkAndRunModules(execute=true, background=false) { let sigText = this.getSigText(); @@ -812,9 +1018,9 @@ define([ let checkedList = JSON.parse(result); let executeList = []; checkedList && checkedList.forEach((mod, idx) => { - if (mod == false) { + if (mod === false) { let modInfo = vpConfig.getModuleCode(checkModules[idx]); - if (modInfo) { + if (modInfo && modInfo?.code !== '') { executeList.push(modInfo.code); } } @@ -857,35 +1063,62 @@ define([ return true; } + _beforeOpen() { + /** Implementation needed - Generated on clicking Install Package button */ + } + /** * Open popup * - show popup * - focus popup * - bind codemirror */ - open() { + open(targetFrame=undefined) { vpLog.display(VP_LOG_TYPE.DEVELOP, 'open popup', this); this.loadState(); - this.show(); - - // set popup position if its top-left side is outside of view - let pos = $(this.wrapSelector()).position(); - if (pos) { - if (pos.top < 0) { - $(this.wrapSelector()).css({ top: 0 }); - } - if (pos.left < 0) { - $(this.wrapSelector()).css({ left: 0 }); + this._beforeOpen(); + if (targetFrame !== undefined) { + // hide popup frame + $(this.wrapSelector('.vp-popup-header')).hide(); + $(this.wrapSelector('.vp-popup-footer')).hide(); + // set width and height to 100% + $(this.wrapSelector()).css({ + width: '100%', + height: '100%', + 'min-height': 'unset', + display: 'block', + position: 'initial', + border: '0px' + }); + $(this.wrapSelector('.vp-popup-body')).css({ + padding: '0px' + }); + // show on targetFrame + $(this.wrapSelector()).appendTo($(targetFrame)); + } else { + this.show(); + + // set popup position if its top-left side is outside of view + let pos = $(this.wrapSelector()).position(); + if (pos) { + if (pos.top < 0) { + $(this.wrapSelector()).css({ top: 0 }); + } + if (pos.left < 0) { + $(this.wrapSelector()).css({ left: 0 }); + } } } this._bindCodemirror(); - $(this.eventTarget).trigger({ - type: 'focus_option_page', - component: this - }); + if (targetFrame == undefined) { + $(this.eventTarget).trigger({ + type: 'focus_option_page', + component: this + }); + } } setSaveOnlyMode() { @@ -902,13 +1135,21 @@ define([ close() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'close popup', this); this.saveState(); - this.hide(); + if (this.getTaskType() === 'task') { + $(this.eventTarget).trigger({ + type: 'remove_option_page', + component: this + }); + } else { + // if it's block, just hide it + this.hide(); + } } save() { if (this.prop.finish && typeof this.prop.finish == 'function') { var code = this.generateCode(); - this.prop.finish(code); + this.prop.finish(code, this.state); } $(this.eventTarget).trigger({ type: 'apply_option_page', @@ -919,7 +1160,9 @@ define([ remove() { vpLog.display(VP_LOG_TYPE.DEVELOP, 'remove popup', this); + this._unbindResizable(); this._unbindEvent(); + this.closeHelpView(); $(this.wrapSelector()).remove(); } @@ -936,6 +1179,10 @@ define([ blur() { $(this.wrapSelector()).removeClass('vp-focused'); + // blur on its block + if (this.taskItem) { + this.taskItem.blurItem(); + } } show() { @@ -943,11 +1190,28 @@ define([ $(this.wrapSelector()).show(); } + showInstallButton() { + $(this.wrapSelector('#popupInstall')).show(); + } + + showImportButton() { + $(this.wrapSelector('#popupImport')).show(); + } + hide() { + this.closeHelpView(); this.taskItem && this.taskItem.blurItem(); $(this.wrapSelector()).hide(); } + hideInstallButton() { + $(this.wrapSelector('#popupInstall')).hide(); + } + + hideImportButton() { + $(this.wrapSelector('#popupImport')).hide(); + } + isHidden() { return !$(this.wrapSelector()).is(':visible'); } @@ -974,7 +1238,7 @@ define([ * @param {*} viewType code / data */ openView(viewType) { - if (viewType == 'code') { + if (viewType === 'code') { this.saveState(); var code = this.generateCode(); let codeText = ''; @@ -990,19 +1254,72 @@ define([ setTimeout(function() { that.cmCodeview.refresh(); }, 1); - $(this.wrapSelector('.vp-popup-dataview-box')).hide(); - } else { + this.closeView('data'); + $(this.wrapSelector('.vp-popup-codeview-box')).show(); + } else if (viewType === 'data') { this.renderDataView(); - $(this.wrapSelector('.vp-popup-codeview-box')).hide(); + this.closeView('code'); + $(this.wrapSelector('.vp-popup-dataview-box')).show(); + } else if (viewType === 'help') { + this.closeView('code'); + this.closeView('data'); + this.openHelpView(); } - - $(this.wrapSelector('.vp-popup-'+viewType+'view-box')).show(); } closeView(viewType) { + if (viewType === 'code') { + // reset codeview + if (this.cmCodeview) { + this.cmCodeview.setValue(''); + this.cmCodeview.save(); + + var that = this; + setTimeout(function() { + that.cmCodeview.refresh(); + }, 1); + } + } else if (viewType === 'data') { + // reset dataview + $(this.wrapSelector('.vp-popup-dataview-box')).html(''); + } + // hide view $(this.wrapSelector('.vp-popup-'+viewType+'view-box')).hide(); } + /** + * Open HelpViewer + * - only one helpviewer at once + */ + openHelpView() { + this.closeHelpView(); + this.helpViewer = new HelpViewer(); + this.helpViewer.open(this.config.helpInfo.content, this.config.helpInfo.useHelp, this.config.helpInfo.importCode); + } + + /** + * Close helpViewer + */ + closeHelpView() { + if (this.helpViewer !== undefined) { + this.helpViewer.remove(); + } + } + + /** + * Set HelpViewer content + * @param {string} content + * @param {boolean} useHelp + * @param {string} importCode + */ + setHelpContent(content, useHelp=true, importCode='') { + this.config.helpInfo = { + content: content, + useHelp: useHelp, + importCode: importCode + }; + } + /** * Open inner popup box */ @@ -1011,7 +1328,7 @@ define([ $(this.wrapSelector('.vp-inner-popup-box')).show(); // focus on first input - $(this.wrapSelector('.vp-inner-popup-box input:not(:disabled):visible:first')).focus(); + $(this.wrapSelector('.vp-inner-popup-box input:not(:readonly):not(:disabled):visible:first')).focus(); // disable Jupyter key com_interface.disableOtherShortcut(); } diff --git a/visualpython/js/com/component/SuggestInput.js b/visualpython/js/com/component/SuggestInput.js index ef9aec35..2c7fa96e 100644 --- a/visualpython/js/com/component/SuggestInput.js +++ b/visualpython/js/com/component/SuggestInput.js @@ -17,6 +17,7 @@ define([ this._placeholder = "Select variable"; this._compID = ""; this._additionalClass = ""; + this._autoFocus = true; this._normalFilter = true; this._suggestList = new Array(); this._selectEvent = undefined; @@ -48,6 +49,13 @@ define([ setComponentID(compID = "") { this._compID = compID; } + /** + * set auto focus on enter + * @param {boolean} autoFocus + */ + setAutoFocus(autoFocus = true) { + this._autoFocus = autoFocus; + } /** * normal filter usage * @param {String} normalFilter @@ -118,7 +126,7 @@ define([ $(com_util.formatString(".{0}", that.uuid)).removeClass('suggest-input-uninit').addClass('suggest-input'); $(com_util.formatString(".{0}", that.uuid)).autocomplete({ - autoFocus: true, + autoFocus: that._autoFocus, minLength: minLength, source: function (req, res) { var srcList = typeof that._suggestList == "function" ? that._suggestList() : that._suggestList; @@ -160,6 +168,8 @@ define([ $(this).select(); $(com_util.formatString(".{0}", that.uuid)).autocomplete('search', $(com_util.formatString(".{0}", that.uuid)).val()); }).autocomplete('instance')._renderItem = function(ul, item) { + $(ul).addClass('vp-scrollbar-vertical'); + $(ul).css('max-height', '250px'); if (item.dtype != undefined) { return $('

').attr('data-value', item.value) .append(`

${item.label} | ${item.dtype}

`) diff --git a/visualpython/js/loadVisualpython.js b/visualpython/js/loadVisualpython.js index 5d401bb7..b06ffb62 100644 --- a/visualpython/js/loadVisualpython.js +++ b/visualpython/js/loadVisualpython.js @@ -99,7 +99,7 @@ define([ cfg = $.extend(true, {}, vpConfig.defaultConfig, cfg); vpFrame = new MainFrame(); - if (vpConfig.extensionType !== 'lab') { + if (vpConfig.extensionType !== 'lab' && vpConfig.extensionType !== 'lite') { vpFrame.loadMainFrame(); } @@ -192,6 +192,9 @@ define([ } else if (window.vpExtType === 'lab') { // LAB: added extType as 'lab' window.vpConfig = new com_Config('lab'); + } else if (window.vpExtType === 'lite') { + // LITE: added extType as 'lite' + window.vpConfig = new com_Config('lite'); } else { window.vpConfig = new com_Config(); } @@ -199,45 +202,13 @@ define([ /** * visualpython kernel */ - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { window.vpKernel = new com_Kernel(vpLab); } else { window.vpKernel = new com_Kernel(); } } - var _checkVersion = function() { - // check version timestamp - let nowDate = new Date(); - vpConfig.getData('version_timestamp', 'vpcfg').then(function(data) { - let doCheckVersion = false; - if (data == undefined) { - // no timestamp, check version - doCheckVersion = true; - } else if (data != '') { - let lastCheck = new Date(parseInt(data)); - let diffCheck_now = new Date(nowDate.getFullYear(), nowDate.getMonth() + 1, nowDate.getDate()); - let diffCheck_last = new Date(lastCheck.getFullYear(), lastCheck.getMonth() + 1, lastCheck.getDate()); - - let diff = Math.abs(diffCheck_now.getTime() - diffCheck_last.getTime()); - diff = Math.ceil(diff / (1000 * 3600 * 24)); - - if (diff >= 1) { - // if More than 1 day passed, check version - doCheckVersion = true; - } - } - - // check version and update version_timestamp - if (doCheckVersion == true) { - vpConfig.checkVpVersion(true); - } - - }).catch(function(err) { - vpLog.display(VP_LOG_TYPE.ERROR, err); - }) - } - //======================================================================== // External call function //======================================================================== @@ -290,9 +261,10 @@ define([ _addToolBarVpButton(); } _loadVpResource(cfg); - _checkVersion(); - - if (cfg.vp_section_display && vpFrame) { + vpConfig.checkVersionTimestamp(); + + if ((cfg.vp_section_display && vpFrame) + || vpConfig.extensionType === 'colab') { // CHROME: default to display vp vpFrame.openVp(); } @@ -301,6 +273,7 @@ define([ events.on('kernel_ready.Kernel', function (evt, info) { vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); // read vp functions + vpConfig.isReady = true; vpConfig.readKernelFunction(); }); } else if (vpConfig.extensionType === 'colab') { @@ -308,26 +281,44 @@ define([ colab.global.notebook.kernel.listen('connected', function(x) { vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); // read vp functions + vpConfig.isReady = true; vpConfig.readKernelFunction(); }); - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: if widget is ready or changed, ready for lab kernel connected, and restart vp vpLab.shell._currentChanged.connect(function(s1, value) { - var { newValue } = value; + // var { newValue } = value; + // LAB 4.x: get currentWidget + var newValue = s1.currentWidget; + vpLog.display(VP_LOG_TYPE.DEVELOP, 'jupyterlab shell currently changed', s1, newValue); // kernel restart for notebook and console if (newValue && newValue.sessionContext) { + vpConfig.hideProtector(); if (newValue.sessionContext.isReady) { - vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); - vpConfig.readKernelFunction(); + if (vpConfig.isReady === false) { + vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); + vpConfig.isReady = true; + vpConfig.readKernelFunction(); + vpConfig.checkVersionTimestamp(); + } } newValue.sessionContext._connectionStatusChanged.connect(function(s2, status) { if (status === 'connected') { - vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); - vpConfig.readKernelFunction(); + if (vpConfig.isReady === false) { + vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); + vpConfig.isReady = true; + vpConfig.readKernelFunction(); + vpConfig.checkVersionTimestamp(); + } } }); } else { vpLog.display(VP_LOG_TYPE.LOG, 'No widget detected...'); + vpConfig.isReady = false; + vpConfig.showProtector(); + if (vpConfig.extensionType === 'lite') { + vpLab.serviceManager.sessions.runningChanged.connect(handleRunningChanged); + } } }); } @@ -335,6 +326,22 @@ define([ return vpFrame; } + var handleRunningChanged = function(a, b) { + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Current widget:', vpLab.shell.currentWidget); + if (vpLab.shell.currentWidget) { + vpLab.shell.currentWidget.sessionContext.statusChanged.connect(function(currentWidget, status) { + if (status === 'idle' && vpConfig.isReady === false) { + vpLog.display(VP_LOG_TYPE.LOG, 'vp operations for kernel ready...'); + vpConfig.isReady = true; + vpConfig.hideProtector(); + vpConfig.readKernelFunction(); + vpConfig.checkVersionTimestamp(); + } + }); + vpLab.serviceManager.sessions.runningChanged.disconnect(handleRunningChanged); + } + } + return { initVisualpython: initVisualpython, readConfig: readConfig }; }); /* function, define */ diff --git a/visualpython/js/m_apps/Bind.js b/visualpython/js/m_apps/Bind.js index 4778245c..8d5e42af 100644 --- a/visualpython/js/m_apps/Bind.js +++ b/visualpython/js/m_apps/Bind.js @@ -31,6 +31,7 @@ define([ /** Write codes executed before rendering */ this.config.sizeLevel = 2; this.config.checkModules = ['pd']; + this.config.helpview = true; this.howList = [ { label: 'Inner', value: 'inner', desc: 'Inner join' }, @@ -127,6 +128,8 @@ define([ // clear user option $(that.wrapSelector('#vp_bdUserOption')).val(''); that.state.userOption = ''; + + that.setHelpContent('_vp_pd.' + type); }); //==================================================================== @@ -336,7 +339,7 @@ define([ renderDataPage(renderedText, isHtml = true) { var tag = new com_String(); - tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area + tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area if (isHtml) { tag.appendLine(renderedText); } else { @@ -352,7 +355,7 @@ define([ // if not, get output of all data in selected pandasObject vpKernel.execute(code).then(function(resultObj) { let { msg } = resultObj; - if (msg.content.data) { + if (msg.content && msg.content.data) { var htmlText = String(msg.content.data["text/html"]); var codeText = String(msg.content.data["text/plain"]); if (htmlText != 'undefined') { @@ -365,20 +368,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } @@ -392,6 +395,8 @@ define([ $(this.wrapSelector('#vp_bdWithoutColumn')).hide(); } + this.setHelpContent('_vp_pd.' + this.state.type); + this.loadVariableList(); } @@ -566,6 +571,7 @@ define([ } loadState() { + super.loadState(); var { type, concat, merge, userOption, allocateTo, resetIndex, withoutColumn } = this.state; diff --git a/visualpython/js/m_apps/File.js b/visualpython/js/m_apps/File.js index be00f7ad..7e6bd848 100644 --- a/visualpython/js/m_apps/File.js +++ b/visualpython/js/m_apps/File.js @@ -34,14 +34,17 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 1; + this.config.sizeLevel = 2; this.config.checkModules = ['pd']; this.fileExtensions = { - 'csv': 'csv', - 'excel': 'xlsx', - 'json': 'json', - 'pickle': '' + 'csv': ['csv', 'tsv', 'txt'], + 'excel': ['xlsx', 'xls'], + 'json': ['json'], + 'pickle': [], + 'sas': [], // xport or sas7bdat + 'spss': [], + 'parquet': ['parquet'] } this.package = { @@ -58,10 +61,15 @@ define([ // // this.dataPath = com_Const.DATA_PATH + "sample_csv/"; // this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/data/sample_csv/'; // } - this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; + if (vpConfig.extensionType === 'lite') { + this.dataPath = '/drive/data/'; + } else { + this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; + } this.state = { - fileExtension: 'csv', + fileType: 'csv', + fileExtension: ['csv'], selectedFile: '', selectedPath: '', vp_fileioType: 'Read', @@ -78,10 +86,13 @@ define([ this.fileState = { 'Read': { fileTypeId: { - 'csv': 'pd004', - 'excel': 'pd123', - 'json': 'pd076', - 'pickle': 'pd079' + 'csv': 'pd_readCsv', + 'excel': 'pd_readExcel', + 'json': 'pd_readJson', + 'pickle': 'pd_readPickle', + 'sas': 'pd_readSas', + 'spss': 'pd_readSpss', + 'parquet':'pd_readParquet' }, selectedType: 'csv', package: null, @@ -92,10 +103,11 @@ define([ }, 'Write': { fileTypeId: { - 'csv': 'pd005', - 'excel': 'pd124', - 'json': 'pd077', - 'pickle': 'pd078' + 'csv': 'pd_toCsv', + 'excel': 'pd_toExcel', + 'json': 'pd_toJson', + 'pickle': 'pd_toPickle', + 'parquet':'pd_toParquet' }, selectedType: 'csv', package: null, @@ -168,13 +180,19 @@ define([ "data_select" ], "output": true, - "required": true + "required": true, + "value": 'vp_df' } ] } } } + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#fileReadAs')); + } + _bindEvent() { super._bindEvent(); /** Implement binding events */ @@ -184,22 +202,70 @@ define([ that.state['vp_fileioType'] = pageType; $(that.wrapSelector('.vp-fileio-box')).hide(); $(that.wrapSelector('#vp_file' + pageType)).show(); - + + if (pageType === 'Read' && that.fileState[pageType].selectedType === 'spss') { + // show install button + that.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); + } else { + that.hideInstallButton(); + $(that.wrapSelector('.vp-spss-note')).remove(); + } + + //set fileExtensions that.fileResultState = { ...that.fileState[pageType].fileResultState }; }); + + // fileReadAs change Event, Use PyArrow + $(document).on('change', this.wrapSelector('#fileReadAs'), function() { + let isChecked = $(this).prop('checked'); + var fileioType = that.state.vp_fileioType; + var prefix = '#vp_file' + fileioType + ' '; + var selectedType = that.fileState[fileioType]['selectedType']; + var fileioTypePrefix = fileioType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } + let fileId = that.fileState[fileioType].fileTypeId[selectedType]; + + if (isChecked) { // pyArrow + fileId = "pa_" + fileioTypePrefix + selectedType[0].toUpperCase() + selectedType.slice(1); + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pa_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); + } else { // pandas + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + if (that.state.fileType != 'parquet'){ // parquet has no options area + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').show(); + } + } + + let pdLib = pandasLibrary.PANDAS_FUNCTION; + let thisPkg = JSON.parse(JSON.stringify(pdLib[fileId])); + + that.fileState[fileioType].package = thisPkg; + }); + } _bindEventByType(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + + var fileioTypePrefix = pageType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } // select file type $(this.wrapSelector(prefix + '#fileType')).change(function() { - var value = $(this).val(); - that.fileState[pageType].selectedType = value; + var fileType = $(this).val(); + that.fileState[pageType].selectedType = fileType; // reload that.renderPage(pageType); @@ -214,9 +280,14 @@ define([ type = 'open'; } + let extensionList = []; + if (that.state.fileExtension && that.state.fileExtension.length > 0) { + extensionList = that.state.fileExtension; + } + let fileNavi = new FileNavigation({ type: type, - extensions: [ that.state.fileExtension ], + extensions: extensionList, finish: function(filesPath, status, error) { let {file, path} = filesPath[0]; that.state.selectedFile = file; @@ -286,13 +357,21 @@ define([ templateForBody() { /** Implement generating template */ - return fileHtml; + let page = $(fileHtml); + + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'vp_sampleReturn', placeholder: 'Variable name', value: 'vp_df' + // }); + // $(page).find('#vp_sampleReturn').replaceWith(allocateSelector.toTagString()); + + return page; } renderPage(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + // clear $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).html(''); $(this.wrapSelector(prefix + '#vp_optionBox table tbody')).html(''); @@ -308,12 +387,19 @@ define([ this.fileResultState = { ...this.fileState[pageType].fileResultState }; + + if (selectedType == 'pickle' || selectedType == 'parquet') { + // hide additional option box + $(this.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); + } else { + $(this.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').show(); + } if (pageType == 'Write') { if (selectedType == 'json') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path_or_buf'); } - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path'); } } @@ -323,11 +409,21 @@ define([ // pdGen.vp_showInterfaceOnPage(this.wrapSelector('#vp_file' + pageType), thisPkg); pdGen.vp_showInterfaceOnPage(this, thisPkg, this.state, parent=('#vp_file' + pageType)); + // pyarrow can r/w parquet, csv and only read json. + if ((pageType == 'Read' && selectedType == 'json') || selectedType == 'parquet'|| selectedType == 'csv') { + // add checkbox 'Use PyArrow', next to File Type + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' Use PyArrow')) + ); + } else { + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' ')) + ); + } + // prepend file type selector - $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( - $('').append($(`File Type`)) - .append($('')) - ); var fileTypeList = Object.keys(fileTypeObj); fileTypeList.forEach(type => { $(this.wrapSelector(prefix + '#fileType')).append( @@ -338,7 +434,7 @@ define([ // prepend user option let hasAllocateTo = $(this.wrapSelector(prefix + '#o0')).length > 0; if (hasAllocateTo) { - $(this.wrapSelector(prefix + '#o0')).closest('tr').before( + $(this.wrapSelector(prefix + '#o0')).closest('tr').after( $('').append($(`User Option`)) .append($('')) ) @@ -348,9 +444,9 @@ define([ .append($('')) ) } - + $(this.wrapSelector(prefix + '#fileType')).val(selectedType); - + // add file navigation button if (pageType == 'Write') { if (selectedType == 'json') { @@ -358,7 +454,7 @@ define([ com_util.formatString('

' , 'vp-file-browser-button') ); - } else if (selectedType == 'pickle') { + } else if (selectedType == 'pickle' || selectedType == 'parquet') { $(prefix + '#path').parent().html( com_util.formatString('

' , 'vp-file-browser-button') @@ -377,6 +473,18 @@ define([ , 'vp-file-browser-button') ); } + + if (pageType === 'Read' && selectedType === 'spss') { + // show install button + this.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(this.wrapSelector('#fileType')).closest('tr')); + } else { + this.hideInstallButton(); + } // encoding suggest input $(this.wrapSelector('#encoding')).replaceWith(function() { @@ -389,8 +497,18 @@ define([ suggestInput.setPlaceholder('encoding option'); return suggestInput.toTagString(); }); - - + + // separator suggest input + $(this.wrapSelector('#sep')).replaceWith(function() { + // separator list : + var sepList = [',', '|', '\\t', '\\n', ':', ';', '-', '_', '&', '/', '\\']; + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('sep'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return sepList; }); + suggestInput.setPlaceholder('Input separator'); + return suggestInput.toTagString(); + }); } render() { @@ -412,6 +530,10 @@ define([ } + generateInstallCode() { + return [ '!pip install pyreadstat' ]; + } + generateCode() { var pageType = $(this.wrapSelector('#vp_fileioType')).val(); var sbCode = new com_String; @@ -449,7 +571,6 @@ define([ var result = pdGen.vp_codeGenerator(this, thisPkg, this.state, userOption.toString(), parent='#vp_fileWrite'); sbCode.append(result); } - return sbCode.toString(); } diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index bdb71e4f..4e91083d 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -21,8 +21,9 @@ define([ 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', 'vp_base/js/m_apps/Subset' -], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, Subset) { +], function(frameHtml, frameCss, com_Const, com_String, com_util, PopupComponent, SuggestInput, DataSelector, MultiSelector, Subset) { /** * Frame @@ -30,14 +31,14 @@ define([ class Frame extends PopupComponent { _init() { super._init(); - this.config.sizeLevel = 3; + this.config.sizeLevel = 5; this.config.checkModules = ['pd']; // state this.state = { originObj: '', tempObj: '_vp', - returnObj: '_vp', + returnObj: 'vp_df', inplace: false, menu: '', menuItem: '', @@ -61,12 +62,9 @@ define([ } // numpy.dtype or python type - this.astypeList = [ - 'datetime64', - 'int', 'int32', 'int64', - 'float', 'float64', - 'object', 'category', - 'bool', 'str' + this.astypeList = [ + 'object', 'int64', 'float64', 'bool', + 'datetime64[ns]', 'timedelta[ns]', 'category' ]; // { @@ -89,7 +87,8 @@ define([ { id: 'add_row', label: 'Add row', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.ADD_ROW }, { id: 'delete', label: 'Delete', selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP }, { id: 'rename', label: 'Rename', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RENAME }, - { id: 'asType', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'as_type', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'to_datetime', label: 'To Datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, { id: 'replace', label: 'Replace', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.REPLACE }, { id: 'discretize', label: 'Discretize', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, numeric_only: true, menuType: FRAME_EDIT_TYPE.DISCRETIZE } ] @@ -116,10 +115,10 @@ define([ id: 'encoding', label: 'Encoding', axis: FRAME_AXIS.COLUMN, - selection: FRAME_SELECT_TYPE.SINGLE, + selection: FRAME_SELECT_TYPE.MULTI, child: [ - { id: 'label_encoding', label: 'Label encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.SINGLE, menuType: FRAME_EDIT_TYPE.LABEL_ENCODING }, - { id: 'one_hot_encoding', label: 'Onehot encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.SINGLE, menuType: FRAME_EDIT_TYPE.ONE_HOT_ENCODING }, + { id: 'label_encoding', label: 'Label encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.LABEL_ENCODING }, + { id: 'one_hot_encoding', label: 'Onehot encoding', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.ONE_HOT_ENCODING }, ] }, { @@ -129,7 +128,8 @@ define([ child: [ { id: 'fillna', label: 'Fill NA', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.FILL_NA }, { id: 'dropna', label: 'Drop NA', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.DROP_NA }, - { id: 'drop_outlier', label: 'Drop outlier', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.SINGLE, menuType: FRAME_EDIT_TYPE.DROP_OUT }, + { id: 'fill_outlier', label: 'Fill outlier', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.FILL_OUT }, + { id: 'drop_outlier', label: 'Drop outlier', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP_OUT }, { id: 'drop_duplicates', label: 'Drop duplicates', axis: FRAME_AXIS.COLUMN, selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.DROP_DUP }, ] }, @@ -178,7 +178,7 @@ define([ // initialize state values that.state.originObj = origin; that.state.tempObj = '_vp'; - that.state.returnObj = that.state.tempObj; + that.state.returnObj = 'vp_df'; if (that.state.inplace === true) { that.state.returnObj = origin; } @@ -230,7 +230,7 @@ define([ // check/uncheck inplace $(this.wrapSelector('#inplace')).on('change', function() { let checked = $(this).prop('checked'); - let returnVariable = '_vp'; + let returnVariable = 'vp_df'; if (checked === true) { returnVariable = that.state.originObj; } @@ -242,7 +242,7 @@ define([ that.setPreview(that.getCurrentCode()); }); - // menu on column (Deprecated on v2.3.6 - Temporarily Show on v.2.3.7) + // menu on column (Deprecated on v2.3.6 - Temporarily Show on v.2.4.4) $(document).on('contextmenu', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_COLUMN), function(event) { event.preventDefault(); @@ -272,7 +272,7 @@ define([ that.showMenu(thisPos.left, thisPos.top + thisRect.height); }); - // menu on row (Deprecated on v2.3.6 - Temporarily Show on v.2.3.7) + // menu on row (Deprecated on v2.3.6 - Temporarily Show on v.2.4.4) $(document).on('contextmenu', this.wrapSelector('.' + VP_FE_TABLE + ' .' + VP_FE_TABLE_ROW), function(event) { event.preventDefault(); var idx = $(that.wrapSelector('.' + VP_FE_TABLE_ROW)).index(this); // 0 ~ n @@ -597,7 +597,7 @@ define([ that.loadCode(that.getTypeCode(FRAME_EDIT_TYPE.SHOW), true); }); - // click toolbar item (Deprecated on v2.3.6 - Temporarily Show on v.2.3.7) + // click toolbar item (Deprecated on v2.3.6 - Temporarily Show on v.2.4.4) $(document).on('click', this.wrapSelector('.vp-fe-toolbar-item'), function(evt) { evt.stopPropagation(); var itemType = $(this).data('type'); @@ -619,6 +619,7 @@ define([ case FRAME_EDIT_TYPE.RENAME: case FRAME_EDIT_TYPE.REPLACE: case FRAME_EDIT_TYPE.AS_TYPE: + case FRAME_EDIT_TYPE.TO_DATETIME: case FRAME_EDIT_TYPE.DISCRETIZE: case FRAME_EDIT_TYPE.DATA_SHIFT: case FRAME_EDIT_TYPE.SORT_INDEX: @@ -626,8 +627,10 @@ define([ case FRAME_EDIT_TYPE.FILL_NA: case FRAME_EDIT_TYPE.DROP_NA: case FRAME_EDIT_TYPE.DROP_DUP: + case FRAME_EDIT_TYPE.FILL_OUT: case FRAME_EDIT_TYPE.DROP_OUT: - case FRAME_EDIT_TYPE.DROP: // check one more time + case FRAME_EDIT_TYPE.DROP: + // open inner popup that.openInputPopup(editType); break; default: @@ -710,22 +713,30 @@ define([ return; } } else if (type === FRAME_EDIT_TYPE.REPLACE) { - if (content.replacetype === 'condition' && content.value === '') { - $(this.wrapSelector('.vp-inner-popup-input3')).focus(); - return; - } - } else if (type === FRAME_EDIT_TYPE.REPLACE) { - if (content.input === '') { - $(this.wrapSelector('.vp-inner-popup-input')).focus(); - return; - } + ; } else if (type === FRAME_EDIT_TYPE.FILL_NA) { if (content.method === 'value' && content.value === '') { $(this.wrapSelector('.vp-inner-popup-value')).focus(); return; } + } else if (type === FRAME_EDIT_TYPE.FILL_OUT) { + if (content.filltype === 'value' && content.fillvalue === '') { + $(this.wrapSelector('.vp-inner-popup-fillvalue')).focus(); + return; + } + } else if (type === FRAME_EDIT_TYPE.TO_DATETIME) { + if (content.newcol === '') { + $(this.wrapSelector('.vp-inner-popup-todt-new-col')).focus(); + } } - if (type == FRAME_EDIT_TYPE.DROP_OUT) { + // run check modules for outliers and load codes + if (type === FRAME_EDIT_TYPE.FILL_OUT) { + this.config.checkModules = ['pd', 'np', 'vp_fill_outlier']; + let that = this; + this.checkAndRunModules(true).then(function() { + that.loadCode(that.getTypeCode(that.state.popup.type, content)); + }); + } else if (type === FRAME_EDIT_TYPE.DROP_OUT) { this.config.checkModules = ['pd', 'np', 'vp_drop_outlier']; let that = this; this.checkAndRunModules(true).then(function() { @@ -768,38 +779,258 @@ define([ var that = this; if (menuType === FRAME_EDIT_TYPE.ADD_COL - || menuType === FRAME_EDIT_TYPE.ADD_ROW) { - ///// add page - // 1. add type - $(this.wrapSelector('.vp-inner-popup-addtype')).on('change', function() { - var tab = $(this).val(); - $(that.wrapSelector('.vp-inner-popup-tab')).hide(); - $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); + || menuType === FRAME_EDIT_TYPE.ADD_ROW + || menuType === FRAME_EDIT_TYPE.REPLACE) { + // Add page + if (menuType === FRAME_EDIT_TYPE.ADD_COL + || menuType === FRAME_EDIT_TYPE.ADD_ROW) { + ///// add page + // 1. add type + $(this.wrapSelector('.vp-inner-popup-addtype')).on('change', function() { + var tab = $(this).val(); + $(that.wrapSelector('.vp-inner-popup-tab')).hide(); + $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); + + if (tab === 'apply') { + let colName = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); + $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colName); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + } + }); + + // 2-1. hide column selection box + $(this.wrapSelector('.vp-inner-popup-var1box .vp-vs-data-type')).on('change', function() { + var type = $(this).val(); + if (type == 'DataFrame') { + $(that.wrapSelector('.vp-inner-popup-var1col')).show(); + } else { + $(that.wrapSelector('.vp-inner-popup-var1col')).hide(); + } + }); + + $(this.wrapSelector('.vp-inner-popup-var2box .vp-vs-data-type')).on('change', function() { + var type = $(this).val(); + if (type == 'DataFrame') { + $(that.wrapSelector('.vp-inner-popup-var2col')).show(); + } else { + $(that.wrapSelector('.vp-inner-popup-var2col')).hide(); + } + }); + + $(document).off('change', this.wrapSelector('.vp-inner-popup-vartype')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-vartype'), function() { + var type = $(this).val(); + $(this).closest('tr').find('.vp-inner-popup-vartype-box').hide(); + $(this).closest('tr').find('.vp-inner-popup-vartype-box.' + type).show(); + }); + } + + // Replace page + if (menuType === FRAME_EDIT_TYPE.REPLACE) { + $(this.wrapSelector('.vp-inner-popup-replacetype')).on('change', function() { + var tab = $(this).val(); + $(that.wrapSelector('.vp-inner-popup-tab')).hide(); + $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); + + if (tab === 'apply') { + let colName = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); + $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colName); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + } + }); + } + + // Add & Replace page + // condition add + $(document).off('click', this.wrapSelector('.vp-inner-popup-add-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-add-cond'), function (event) { + that.handleConditionAdd(); }); - - // 2-1. hide column selection box - $(this.wrapSelector('.vp-inner-popup-var1box .vp-vs-data-type')).on('change', function() { - var type = $(this).val(); - if (type == 'DataFrame') { - $(that.wrapSelector('.vp-inner-popup-var1col')).show(); + + // condition delete + $(document).off('click', this.wrapSelector('.vp-inner-popup-del-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-del-cond'), function (event) { + event.stopPropagation(); + + // clear previous one + $(this).closest('tr').remove(); + $(that.wrapSelector('.vp-inner-popup-oper-connect:last')).hide(); + }); + + // change column selection for condition page + $(document).off('change', this.wrapSelector('.vp-inner-popup-col-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-col-list'), function () { + var thisTag = $(this); + var varName = that.state.tempObj; + var colName = $(this).find('option:selected').attr('data-code'); + var colDtype = $(this).find('option:selected').attr('data-dtype'); + + var operTag = $(this).closest('td').find('.vp-inner-popup-oper-list'); + var condTag = $(this).closest('td').find('.vp-inner-popup-condition'); + + if (colName == '.index') { + // index + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', false); + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(''); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput([], ''); + }); + that.generateCode(); } else { - $(that.wrapSelector('.vp-inner-popup-var1col')).hide(); + // get result and load column list + vpKernel.getColumnCategory(varName, colName).then(function (resultObj) { + let { result } = resultObj; + try { + var category = JSON.parse(result); + if (category && category.length > 0 && colDtype == 'object') { + // if it's categorical column and its dtype is object, check 'Text' as default + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', true); + } else { + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', false); + } + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput(category, colDtype); + }); + } catch { + $(thisTag).closest('td').find('.vp-inner-popup-cond-use-text').prop('checked', false); + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput([], colDtype); + }); + } + }); } }); - - $(this.wrapSelector('.vp-inner-popup-var2box .vp-vs-data-type')).on('change', function() { - var type = $(this).val(); - if (type == 'DataFrame') { - $(that.wrapSelector('.vp-inner-popup-var2col')).show(); + + // change operator selection + $(document).off('change', this.wrapSelector('.vp-inner-popup-oper-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-oper-list'), function () { + var oper = $(this).val(); + var condTag = $(this).closest('td').find('.vp-inner-popup-condition'); + var useTextTag = $(this).closest('td').find('.vp-inner-popup-cond-use-text'); + // var colDtype = $(this).closest('td').find('.vp-col-list option:selected').attr('data-dtype'); + + // if operator is isnull(), notnull(), disable condition input + if (oper == 'isnull()' || oper == 'notnull()') { + $(condTag).prop('disabled', true); + $(useTextTag).prop('disabled', true); + } else { + $(condTag).prop('disabled', false); + $(useTextTag).prop('disabled', false); + } + }); + + // apply page + // 4. apply + $(this.wrapSelector('.vp-inner-popup-apply-column')).on('change', function() { + // TODO: change apply-condition (save value) + let colLabel = $(that.wrapSelector('.vp-inner-popup-apply-column option:selected')).text(); + $(that.wrapSelector('.vp-inner-popup-apply-target-name')).text(colLabel); + + var varName = that.state.tempObj; + let colName = $(this).val(); + var colDtype = $(this).find('option:selected').attr('data-dtype'); + var operTag = $(that.wrapSelector('.vp-inner-popup-apply-oper-list')); + var condTag = $(that.wrapSelector('.vp-inner-popup-apply-condition')); + // get result and load column list + vpKernel.getColumnCategory(varName, colName).then(function (resultObj) { + let { result } = resultObj; + try { + var category = JSON.parse(result); + if (category && category.length > 0 && colDtype == 'object') { + // if it's categorical column and its dtype is object, check 'Text' as default + $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', true); + } else { + $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', false); + } + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype, 'vp-inner-popup-apply-oper-list', $(this).val()); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput(category, colDtype, 'vp-inner-popup-apply-condition', $(this).val()); + }); + } catch { + $(that.wrapSelector('.vp-inner-popup-apply-cond-usetext')).prop('checked', false); + $(operTag).replaceWith(function () { + return that.templateForConditionOperator(colDtype, 'vp-inner-popup-apply-oper-list'); + }); + $(condTag).replaceWith(function () { + return that.templateForConditionCondInput([], colDtype, 'vp-inner-popup-apply-condition'); + }); + } + }); + }); + + // change operator selection + $(document).off('change', this.wrapSelector('.vp-inner-popup-apply-oper-list')); + $(document).on('change', this.wrapSelector('.vp-inner-popup-apply-oper-list'), function () { + var oper = $(this).val(); + var condTag = $(this).closest('td').find('.vp-inner-popup-apply-condition'); + var useTextTag = $(this).closest('td').find('.vp-inner-popup-apply-cond-usetext'); + + // if operator is isnull(), notnull(), disable condition input + if (oper == 'isnull()' || oper == 'notnull()') { + $(condTag).prop('disabled', true); + $(useTextTag).prop('disabled', true); } else { - $(that.wrapSelector('.vp-inner-popup-var2col')).hide(); + $(condTag).prop('disabled', false); + $(useTextTag).prop('disabled', false); } }); - } else if (menuType === FRAME_EDIT_TYPE.REPLACE) { - $(this.wrapSelector('.vp-inner-popup-replacetype')).on('change', function() { - var tab = $(this).val(); - $(that.wrapSelector('.vp-inner-popup-tab')).hide(); - $(that.wrapSelector('.vp-inner-popup-tab.' + tab)).show(); + + $(this.wrapSelector('.vp-inner-popup-toggle-else')).on('click', function() { + // toggle else on/off + let elseOn = $(this).attr('data-else'); // on / off + if (elseOn === 'off') { + // off -> on + $(this).attr('data-else', 'on'); + $(this).text('Else off'); + $(that.wrapSelector('.vp-inner-popup-apply-else-value')).prop('disabled', false); + $(that.wrapSelector('.vp-inner-popup-apply-else-usetext')).prop('disabled', false); + } else { + // on -> off + $(this).attr('data-else', 'off'); + $(this).text('Else on'); + $(that.wrapSelector('.vp-inner-popup-apply-else-value')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-apply-else-usetext')).prop('disabled', true); + } + }); + + $(this.wrapSelector('.vp-inner-popup-add-case')).on('click', function() { + // add case + $(this).parent().find('.vp-inner-popup-apply-case-box').append($(that.templateForApplyCase())); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + }); + + $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-add-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-apply-add-cond'), function() { + // add condition + $(this).parent().find('.vp-inner-popup-apply-cond-box').append($(that.templateForApplyCondition())); + // show operator except last operator + $(this).parent().find('.vp-inner-popup-apply-oper-connect:not(:last)').show(); + $(that.wrapSelector('.vp-inner-popup-apply-column')).trigger('change'); + }); + + $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-del-cond')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-apply-del-cond'), function() { + let condBox = $(this).closest('.vp-inner-popup-apply-cond-box'); + // delete apply cond + $(this).parent().remove(); + // hide last operator + $(condBox).find('.vp-inner-popup-apply-oper-connect:last').hide(); + }); + + $(document).off('click', this.wrapSelector('.vp-inner-popup-apply-del-case')); + $(document).on('click', this.wrapSelector('.vp-inner-popup-apply-del-case'), function() { + // delete apply case + $(this).parent().remove(); }); } else if (menuType === FRAME_EDIT_TYPE.DISCRETIZE) { // change bins @@ -862,6 +1093,106 @@ define([ let tag = $(this).closest('.vp-inner-popup-sortby-item'); tag.insertAfter(tag.next()); }); + } else if (menuType === FRAME_EDIT_TYPE.FILL_OUT) { + $(this.wrapSelector('.vp-inner-popup-filltype')).on('change', function() { + let filltype = $(this).val(); + if (filltype === 'value') { + $(that.wrapSelector('.vp-inner-popup-fillvalue')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-fillvalue')).prop('disabled', true); + } + }); + } else if (menuType === FRAME_EDIT_TYPE.DROP_NA) { + $(this.wrapSelector('.vp-inner-popup-how')).on('change', function() { + let val = $(this).val(); + if (val === '') { + $(that.wrapSelector('.vp-inner-popup-thresh')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-thresh')).prop('disabled', true); + } + }); + } else if (menuType === FRAME_EDIT_TYPE.FILL_NA) { + // bind event on method + $(this.wrapSelector('.vp-inner-popup-method')).on('change', function() { + let changedVal = $(this).val(); + if (changedVal === 'value') { + // show value row + $(that.wrapSelector('.vp-inner-popup-value-row')).show(); + $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); + } else if (changedVal === 'ffill' || changedVal === 'bfill') { + // show method fill row + $(that.wrapSelector('.vp-inner-popup-value-row')).hide(); + $(that.wrapSelector('.vp-inner-popup-fill-row')).show(); + } else { + // hide all + $(that.wrapSelector('.vp-inner-popup-value-row')).hide(); + $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); + } + }); + } else if (menuType === FRAME_EDIT_TYPE.TO_DATETIME) { + // bind event for selecting format + $(this.wrapSelector('.vp-inner-popup-todt-format')).on('change', function() { + let format = $(this).val(); + if (format === 'auto') { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(''); + } + + if (format === 'typing') { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', true); + } + }); + + // Add column set event + $(this.wrapSelector('.vp-inner-popup-todt-addcol')).on('click', function() { + let dateTypeList = [ // df[col].dt[{dateType}] + { label: 'Year', value: 'year' }, + { label: 'Month', value: 'month' }, + { label: 'Day', value: 'day' }, + { label: 'Date', value: 'date' }, + { label: 'DayOfWeek', value: 'dayofweek' }, + { label: 'DayOfYear', value: 'dayofyear' }, + { label: 'DaysInMonth', value: 'daysinmonth' }, + { label: 'Quarter', value: 'quarter' }, + { label: 'Time', value: 'time' }, + { label: 'Hour', value: 'hour' }, + { label: 'Minute', value: 'minute' }, + { label: 'Second', value: 'second' }, + { label: 'Nanosecond', value: 'nanosecond' }, + ]; + let dateTypeOptionTag = new com_String(); + dateTypeList.forEach(opt => { + dateTypeOptionTag.appendFormat('', opt.value, opt.label); + }); + + let addColItemTag = $(``); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-content')).append(addColItemTag); + $(addColItemTag)[0].scrollIntoView(); + + // bind event for selecting date type option + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).off('change'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).on('change', function() { + let dateTypeVal = $(this).val(); + $(this).parent().find('.vp-inner-popup-todt-addcol-colname').val(dateTypeVal); + }); + + // bind event for deleting + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).off('click'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).on('click', function() { + // delete item + $(this).closest('.vp-inner-popup-todt-addcol-item').remove(); + }); + }); } } @@ -873,28 +1204,29 @@ define([ $(that.wrapSelector('.vp-inner-popup-isedgechanged')).val("false"); let code = new com_String(); - code.appendFormatLine("_out, _bins = pd.cut({0}[{1}], bins={2}, right={3}, labels=False, retbins=True)" + code.appendFormatLine("_out, _bins = pd.cut({0}[{1}], bins={2}, right={3}, retbins=True)" , this.state.tempObj, this.state.selected[0].code, binsCount, right?'True':'False'); - code.append("_vp_print({'labels': _out.unique(), 'edges': list(_bins)})"); + code.append("_vp_print({'edges': list(_bins)})"); vpKernel.execute(code.toString()).then(function(resultObj) { let { result } = resultObj; - let { labels, edges } = JSON.parse(result); + let { edges } = JSON.parse(result); + let labelLength = edges.length - 1; let edgeTbody = new com_String(); - labels && labels.forEach((label, idx) => { + for (let idx = 0; idx < labelLength; idx++ ) { let leftDisabled = 'disabled'; let rightDisabled = ''; - if (idx === (labels.length - 1)) { + if (idx === (labelLength - 1)) { rightDisabled = 'disabled'; } edgeTbody.append(''); - edgeTbody.appendFormatLine('', idx, label); + edgeTbody.appendFormatLine('', idx, idx); edgeTbody.appendLine(':'); edgeTbody.appendFormatLine('', idx, edges[idx], leftDisabled); edgeTbody.appendLine('~'); edgeTbody.appendFormatLine('', idx + 1, edges[idx+1], rightDisabled); edgeTbody.append(''); - }); + } $(that.wrapSelector('.vp-inner-popup-range-table tbody')).html(edgeTbody.toString()); // label change event @@ -917,17 +1249,18 @@ define([ }); }).catch(function(errObj) { - // TODO: + vpLog.display(VP_LOG_TYPE.ERROR, errObj); }); } templateForBody() { let page = $(frameHtml); - let allocateSelector = new DataSelector({ - pageThis: this, id: 'vp_feReturn', placeholder: 'Variable name', required: true, value: '_vp' - }); - $(page).find('#vp_feReturn').replaceWith(allocateSelector.toTagString()); + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'vp_feReturn', placeholder: 'Variable name', required: true, value: 'vp_df' + // }); + // $(page).find('#vp_feReturn').replaceWith(allocateSelector.toTagString()); return page; } @@ -1185,7 +1518,10 @@ define([ if (type === 'column') { content.appendLine('Add type'); content.appendFormatLine(''); } @@ -1194,28 +1530,119 @@ define([ content.appendLine('

'); - // tab 1. variable - content.appendFormatLine('

', 'vp-inner-popup-tab', 'variable'); - content.appendLine(''); + // tab 1. calculate + content.appendFormatLine('

', 'vp-inner-popup-tab', 'calculate'); + content.appendLine('

'); content.appendLine(''); - content.appendLine(''); - content.appendFormatLine('', 'vp-inner-popup-vartype'); + content.appendLine(''); + content.appendLine(''); content.appendFormatLine('', 'vp-inner-popup-addvalue'); content.appendLine('

Variable	', 'vp-inner-popup-value', 0); + content.appendFormatLine('	'); + content.appendFormatLine(' ', 'vp-inner-popup-vartype-box variable'); + content.appendFormatLine('', 'vp-inner-popup-value', 0); content.appendFormatLine('{1}', 'vp-inner-popup-istext','Text'); + content.appendLine(' '); + content.appendFormatLine(' ', 'vp-inner-popup-vartype-box column'); + content.appendFormatLine(''); + content.appendLine(' '); // content.appendFormatLine('', 'vp-inner-popup-delete-value'); - content.appendLine('

'); content.appendLine('

'); // end of vp-inner-popup-tab value - // tab 2. apply + // tab 2. statistics + content.appendFormatLine(''); + + // tab 3. replace + content.appendFormatLine(''); + + // tab 4. condition + // replace page - 2. condition + content.appendFormatLine(''); + + // tab 5. apply content.appendFormatLine(''); // end of vp-inner-popup-addpage @@ -1228,11 +1655,51 @@ define([ renderAddValueBox(idx) { // add dataselector let valueSelector = new DataSelector({ - pageThis: this, id: 'vp_addValue' + idx, classes: 'vp-inner-popup-value', placeholder: 'Type value' + pageThis: this, classes: 'vp-inner-popup-value', placeholder: 'Type value', withPopup: false }); $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-value:nth(' + idx + ')').replaceWith(valueSelector.toTagString()); } + templateForApplyCase() { + return ``; + } + + templateForApplyCondition() { + let colLabel = $(this.wrapSelector('.vp-inner-popup-apply-column option:selected'))?.text(); + return ``; + } + renderCalculator(idx) { let content = new com_String(); content.appendFormatLine('', 'vp-inner-popup-apply-column'); + // columnList && columnList.forEach(col => { + // selectTag.appendFormatLine('', col.code, col.label); + // }); columnList && columnList.forEach(col => { - selectTag.appendFormatLine('', col.code, col.label); - }); + selectTag.appendFormatLine('', + col.code, col.type, col.code, col.label); + }); selectTag.appendLine(''); return selectTag.toString(); } @@ -1507,20 +1978,22 @@ define([ content.appendLine('

'); content.appendLine(''); content.appendFormatLine('', '', 'Column'); - var target = this.state.selected.map(col => col.label).join(','); - content.appendFormatLine(''); content.appendLine(''); content.appendFormatLine(''); content.appendLine('

{1}	', 'vp-inner-popup-input1', target); + var target = this.state.selected[0]; + content.appendFormatLine('	' + , 'vp-inner-popup-input1', target.label, target.code); content.appendLine('
Replace type

'); content.appendLine('

'); // end of vp-inner-popup-header content.appendLine('

'); - // replace page - 1. value - content.appendFormatLine('

', 'vp-inner-popup-tab value'); + // replace page - 1. replace + content.appendFormatLine('

', 'vp-inner-popup-tab replace'); content.appendFormatLine('{1}', 'vp-inner-popup-use-regex', 'Use Regular Expression'); content.appendLine('

'); content.appendFormatLine('

', 'vp-inner-popup-replace-table'); @@ -1536,20 +2009,51 @@ define([ content.appendLine('

'); // replace page - 2. condition content.appendFormatLine(''); // end of vp-inner-popup-tab condition - content.appendLine('Variable'); - content.appendFormatLine('', 'vp-inner-popup-input3'); - content.appendFormatLine('{1}', 'vp-inner-popup-istext3','Text'); - content.appendLine(''); - content.appendLine('

'); + // replace page - 3. apply + content.appendFormatLine(''); // end of vp-inner-popup-tab apply content.appendLine('

'); // end of vp-inner-popup-addpage @@ -1576,6 +2080,92 @@ define([ return content.toString(); } + handleConditionAdd() { + var conditonBox = $(this.renderReplaceCondition()); + + // hide last connect operator + conditonBox.find('.vp-inner-popup-oper-connect').hide(); + + // show connect operator right before last one + $(this.wrapSelector('.vp-inner-popup-oper-connect:last')).show(); + conditonBox.insertBefore(this.wrapSelector('.vp-inner-popup-condition-tbl tr:last')); + } + + renderReplaceCondition() { + var content = new com_String(); + content.appendLine(''); + // delete condition button + content.appendLine(''); + // condition line + content.appendLine('

'); + // - 1. column list + content.appendFormatLine(''); + // - 2. operator + content.appendLine(this.templateForConditionOperator('')); + // - 3. oper-value + content.appendLine(''); + // use text + content.appendFormatLine('{3}', + 'vp-inner-popup-condition-use-text', 'vp-inner-popup-cond-use-text', 'Uncheck it if you want to use variable or numeric values.', 'Text'); + content.appendLine('

'); + content.appendLine('

'); + content.appendLine(''); + content.appendLine('

'); + content.appendLine(''); + return content.toString(); + } + + templateForConditionOperator(dtype='object', className='vp-inner-popup-oper-list', prevValue='') { + var content = new com_String(); + content.appendFormatLine(''); + return content.toString(); + } + + templateForConditionCondInput(category, dtype='object', className='vp-inner-popup-condition', prevValue='') { + var vpCondSuggest = new SuggestInput(); + vpCondSuggest.addClass('vp-input m ' + className); + if (prevValue !== '') { + vpCondSuggest.setValue(prevValue); + } + if (category && category.length > 0) { + vpCondSuggest.setPlaceholder((dtype=='object'?'Categorical':dtype) + " dtype"); + vpCondSuggest.setSuggestList(function () { return category; }); + vpCondSuggest.setSelectEvent(function (value) { + $(this.wrapSelector()).val(value); + $(this.wrapSelector()).trigger('change'); + }); + vpCondSuggest.setNormalFilter(false); + } else { + vpCondSuggest.setPlaceholder(dtype==''?'Value':(dtype + " dtype")); + } + return vpCondSuggest.toTagString(); + } + renderAsType() { var astypeList = this.astypeList; var content = new com_String(); @@ -1585,7 +2175,12 @@ define([ content.appendFormatLine('{0}{1}{3}' , 'Column', 'Data type', 'vp-orange-text', 'New data type'); content.appendLine(''); - this.state.selected.forEach((col, idx) => { + let selectedList = this.state.selected; + if (selectedList.length === 0) { + // select all + selectedList = this.state.columnList; + } + selectedList.forEach((col, idx) => { content.appendLine(''); content.appendFormatLine('{1}', col.label, col.label); content.appendFormatLine('', col.type); @@ -1605,6 +2200,74 @@ define([ return content.toString(); } + renderToDatetime() { + var content = new com_String(); + let formatList = [ + { label: 'Auto', value: 'auto' }, + { label: '%Y/%m/%d', value: '%Y/%m/%d' }, + { label: '%Y-%m-%d', value: '%Y-%m-%d' }, + { label: '%y/%m/%d', value: '%y/%m/%d' }, + { label: '%y-%m-%d', value: '%y-%m-%d' }, + { label: '%d/%m/%Y', value: '%d/%m/%Y' }, + { label: '%d-%m-%Y', value: '%d-%m-%Y' }, + { label: '%d/%m/%y', value: '%d/%m/%y' }, + { label: '%d-%m-%y', value: '%d-%m-%y' }, + { label: 'Typing', value: 'typing' } + ]; + let formatOptionTag = new com_String(); + formatList.forEach(opt => { + formatOptionTag.appendFormat('', opt.value, opt.label); + }); + + content.appendFormat(` + `, this.state.selected[0].label, formatOptionTag.toString(), this.state.selected[0].label); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + renderFillNAPage() { var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-fillna-page'); @@ -1616,6 +2279,12 @@ define([ content.appendFormatLine('', "value", "Value"); content.appendFormatLine('', "ffill", "Forward fill"); content.appendFormatLine('', "bfill", "Back fill"); + let statsMethodList = ['mean', 'median', 'mode', 'min', 'max']; + content.appendLine(''); + statsMethodList.forEach(opt => { + content.appendFormatLine('', opt, opt); + }) + content.appendLine(''); content.appendLine(''); content.appendLine(''); content.appendFormatLine('', 'vp-inner-popup-value-row'); @@ -1649,10 +2318,10 @@ define([ Threshold - + Ignore index + + + + + + + Value + +

`; + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content); + return content.toString(); + } + + renderDropOutPage() { var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-dropout-page'); @@ -1707,27 +2396,45 @@ define([ let size = { width: width, height: height }; let that = this; - $(this.wrapSelector('.vp-inner-popup-body')).html(''); + $(this.wrapSelector('.vp-inner-popup-body')).empty(); switch (parseInt(type)) { case FRAME_EDIT_TYPE.ADD_COL: title = 'Add column'; - size = { width: 450, height: 450 }; + size = { width: 550, height: 480 }; content = this.renderAddPage('column'); this.renderAddValueBox(0); // bind event for adding values to calculate $(this.wrapSelector('.vp-inner-popup-addvalue')).on('click', function() { let valueCount = $(that.wrapSelector('.vp-inner-popup-value')).length; + let columnOptStr = ''; + that.state.columnList.forEach(col => { + columnOptStr += ``; + }); $(` ${that.renderCalculator(valueCount)} - Variable + + + + + + + - - Text `).insertBefore($(that.wrapSelector('.vp-inner-popup-addvalue-row'))); @@ -1742,6 +2449,11 @@ define([ }); }); + // bind multiselector + this.statsSelector = new MultiSelector(this.wrapSelector('.vp-inner-popup-stats-col-list'), + { mode: 'columns', parent: this.state.tempObj, showDescription: false } + ); + // bind codemirror for apply textarea this.applyCm = this.initCodemirror({ key: 'vp-inner-popup-apply-lambda', @@ -1844,83 +2556,74 @@ define([ break; case FRAME_EDIT_TYPE.REPLACE: title = 'Replace'; - // content = this.renderReplacePage(); content = this.renderReplacePage(); - - // bind codemirror - this.subsetCm = this.initCodemirror({ - key: 'vp-inner-popup-subset', - selector: this.wrapSelector('.vp-inner-popup-subset'), - type: 'readonly' - }); - // set subset - let contentState = that.getPopupContent(type); - this.subsetEditor = new Subset({ - pandasObject: this.state.tempObj, - selectedColumns: [ com_util.convertToStr(contentState.name, contentState.nameastext) ], - config: { name: 'Subset' } }, - { - useInputVariable: true, - useInputColumns: true, - targetSelector: this.wrapSelector('.vp-inner-popup-subset'), - pageThis: this, - allowSubsetTypes: ['iloc', 'loc'], - beforeOpen: function(subsetThis) { - let contentState = that.getPopupContent(type); - let name = com_util.convertToStr(contentState.name, contentState.nameastext); - subsetThis.state.selectedColumns = [ name ]; - }, - finish: function(code) { - that.subsetCm.setValue(code); - that.subsetCm.save(); - setTimeout(function () { - that.subsetCm.refresh(); - }, 1); - } - }); + size = { width: 550, height: 480 }; + + // // bind codemirror + // this.subsetCm = this.initCodemirror({ + // key: 'vp-inner-popup-subset', + // selector: this.wrapSelector('.vp-inner-popup-subset'), + // type: 'readonly' + // }); + // // set subset + // let contentState = that.getPopupContent(type); + // this.subsetEditor = new Subset({ + // pandasObject: this.state.tempObj, + // selectedColumns: [ com_util.convertToStr(contentState.name, contentState.nameastext) ], + // config: { name: 'Subset' } }, + // { + // useInputVariable: true, + // useInputColumns: true, + // targetSelector: this.wrapSelector('.vp-inner-popup-subset'), + // pageThis: this, + // allowSubsetTypes: ['iloc', 'loc'], + // beforeOpen: function(subsetThis) { + // let contentState = that.getPopupContent(type); + // let name = com_util.convertToStr(contentState.name, contentState.nameastext); + // subsetThis.state.selectedColumns = [ name ]; + // }, + // finish: function(code) { + // that.subsetCm.setValue(code); + // that.subsetCm.save(); + // setTimeout(function () { + // that.subsetCm.refresh(); + // }, 1); + // } + // }); // initial code - var code = this.subsetEditor.generateCode(); - this.subsetCm.setValue(code); - this.subsetCm.save(); - setTimeout(function () { - that.subsetCm.refresh(); - }, 1); - - // data selector - // vp-inner-popup-input3 + // var code = this.subsetEditor.generateCode(); + // this.subsetCm.setValue(code); + // this.subsetCm.save(); + // setTimeout(function () { + // that.subsetCm.refresh(); + // }, 1); + // set dataselector let replaceVarSelector = new DataSelector({ - pageThis: this, id: 'vp_replaceVariable', classes: 'vp-inner-popup-input3', placeholder: 'Type or select variable' + pageThis: this, id: 'vp_replaceVariable', classes: 'vp-inner-popup-value', placeholder: 'Type or select variable' + , withPopup: false }); - $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-input3').replaceWith(replaceVarSelector.toTagString()); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-value').replaceWith(replaceVarSelector.toTagString()); break; case FRAME_EDIT_TYPE.AS_TYPE: title = 'Convert type'; content = this.renderAsType(); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + title = 'Convert to Datetime'; + size = { width: 500, height: 450 }; + content = this.renderToDatetime(); + break; case FRAME_EDIT_TYPE.FILL_NA: title = 'Fill NA'; content = this.renderFillNAPage(); // set dataselector let valueSelector = new DataSelector({ - pageThis: this, id: 'vp_fillValue', classes: 'vp-inner-popup-value', placeholder: 'Type or select value' + pageThis: this, classes: 'vp-inner-popup-value', placeholder: 'Type or select value', + withPopup: false }); $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-value').replaceWith(valueSelector.toTagString()); - - // bind event on method - $(this.wrapSelector('.vp-inner-popup-method')).on('change', function() { - let changedVal = $(this).val(); - if (changedVal === 'value') { - // show value row - $(that.wrapSelector('.vp-inner-popup-value-row')).show(); - $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); - } else { - // show method fill row - $(that.wrapSelector('.vp-inner-popup-value-row')).hide(); - $(that.wrapSelector('.vp-inner-popup-fill-row')).show(); - } - }); break; case FRAME_EDIT_TYPE.DROP_NA: title = 'Drop NA'; @@ -1930,6 +2633,18 @@ define([ title = 'Drop duplicates'; content = this.renderDropDupPage(); break; + case FRAME_EDIT_TYPE.FILL_OUT: + title = 'Fill outlier'; + content = this.renderFillOutPage(); + + // set dataselector + let fillvalueSelector = new DataSelector({ + pageThis: this, classes: 'vp-inner-popup-fillvalue', placeholder: 'Type or select value', + withPopup: false + }); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-fillvalue').replaceWith(fillvalueSelector.toTagString()); + $(this.wrapSelector('.vp-inner-popup-body')).find('.vp-inner-popup-fillvalue').prop('disabled', true); + break; case FRAME_EDIT_TYPE.DROP_OUT: title = 'Drop outlier'; size = { width: 400, height: 200 }; @@ -2005,15 +2720,22 @@ define([ } var tab = $(this.wrapSelector('.vp-inner-popup-addtype')).val(); if (type === FRAME_EDIT_TYPE.ADD_ROW) { - tab = 'variable'; + tab = 'calculate'; } content['addtype'] = tab; - if (tab == 'variable') { + if (tab == 'calculate') { let values = []; let opers = []; - $(this.wrapSelector('.vp-inner-popup-tab.variable tr.vp-inner-popup-value-row')).each((idx, tag) => { - let valueastext = $(tag).find('.vp-inner-popup-istext').prop('checked'); - values.push(com_util.convertToStr($(tag).find('.vp-inner-popup-value').val(), valueastext)); + $(this.wrapSelector('.vp-inner-popup-tab.calculate tr.vp-inner-popup-value-row')).each((idx, tag) => { + let varType = $(tag).find('.vp-inner-popup-vartype').val(); + if (varType === 'variable') { + let valueastext = $(tag).find('.vp-inner-popup-istext').prop('checked'); + values.push(com_util.convertToStr($(tag).find('.vp-inner-popup-value').val(), valueastext)); + } else { + // columns + let column = $(tag).find('.vp-inner-popup-var-col-list').val(); + values.push(com_util.formatString("{0}[{1}]", this.state.tempObj, column)); + } let oper = $(that.wrapSelector('.vp-inner-popup-oper:nth(' + idx + ')')).val(); if (oper && oper !== '') { opers.push(oper); @@ -2021,20 +2743,110 @@ define([ }); content['values'] = values; content['opers'] = opers; + } else if (tab == 'statistics') { + var colList = this.statsSelector.getDataList(); + content['columns'] = colList.map(x => x.code); + content['method'] = $(this.wrapSelector('.vp-inner-popup-stats-method')).val(); + } else if (tab == 'replace') { + content['target'] = $(this.wrapSelector('.vp-inner-popup-value-col-list option:selected')).data('code'); + var useregex = $(this.wrapSelector('.vp-inner-popup-use-regex')).prop('checked'); + content['useregex'] = useregex; + content['list'] = []; + for (var i=0; i <= this.state.popup.replace.index; i++) { + var origin = $(this.wrapSelector('.vp-inner-popup-origin' + i)).val(); + var origintext = $(this.wrapSelector('.vp-inner-popup-origin-istext'+i)).prop('checked'); + var replace = $(this.wrapSelector('.vp-inner-popup-replace' + i)).val(); + var replacetext = $(this.wrapSelector('.vp-inner-popup-replace-istext'+i)).prop('checked'); + if (origin && replace) { + content['list'].push({ + origin: origin, + origintext: origintext, + replace: replace, + replacetext: replacetext + }); + } + } } else if (tab == 'apply') { - content['column'] = $(this.wrapSelector('.vp-inner-popup-apply-column')).val(); - content['apply'] = $(this.wrapSelector('.vp-inner-popup-apply-lambda')).val(); + content['target'] = $(this.wrapSelector('.vp-inner-popup-apply-column')).val(); + let caseList = []; + $(this.wrapSelector('.vp-inner-popup-apply-case-item')).each((idx, caseTag) => { + let condList = []; + let replaceValue = $(caseTag).find('.vp-inner-popup-apply-case-val').val(); + let replaceValText = $(caseTag).find('.vp-inner-popup-apply-case-usetext').prop('checked'); + + let operTag = $(caseTag).find('.vp-inner-popup-apply-oper-list'); + let condTag = $(caseTag).find('.vp-inner-popup-apply-condition'); + let condTextTag = $(caseTag).find('.vp-inner-popup-apply-cond-usetext'); + let operConnTag = $(caseTag).find('.vp-inner-popup-apply-oper-connect'); + for (let i=0; i { + let condList = []; + let replaceValue = $(caseTag).find('.vp-inner-popup-apply-case-val').val(); + let replaceValText = $(caseTag).find('.vp-inner-popup-apply-case-usetext').prop('checked'); + + let operTag = $(caseTag).find('.vp-inner-popup-apply-oper-list'); + let condTag = $(caseTag).find('.vp-inner-popup-apply-condition'); + let condTextTag = $(caseTag).find('.vp-inner-popup-apply-cond-usetext'); + let operConnTag = $(caseTag).find('.vp-inner-popup-apply-oper-connect'); + for (let i=0; i { + let selectedList = this.state.selected; + if (selectedList.length === 0) { + // select all + selectedList = this.state.columnList; + } + selectedList.forEach((col, idx) => { var value = $(this.wrapSelector('.vp-inner-popup-astype'+idx)).val(); - content[idx] = { - label: col.code, - value: value - }; + if (value !== undefined && value !== '') { + content[idx] = { + label: col.code, + value: value + }; + } }); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + content['format'] = $(this.wrapSelector('.vp-inner-popup-todt-format')).val(); + content['format_typing'] = $(this.wrapSelector('.vp-inner-popup-todt-format-typing')).val(); + content['dayfirst'] = $(this.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(); + content['newcol'] = $(this.wrapSelector('.vp-inner-popup-todt-new-col')).val(); + var colList = []; + var addcolItemTags = $(this.wrapSelector('.vp-inner-popup-todt-addcol-item')); + addcolItemTags && addcolItemTags.each((idx, tag) => { + let colName = $(tag).find('.vp-inner-popup-todt-addcol-colname').val(); + let dateType = $(tag).find('.vp-inner-popup-todt-addcol-type').val(); + if (colName !== '' && dateType !== '') { + colList.push({ colName: colName, dateType: dateType }); + } + }); + content['collist'] = colList; + break; case FRAME_EDIT_TYPE.DISCRETIZE: content['input'] = $(this.wrapSelector('.vp-inner-popup-input')).val(); content['inputastext'] = $(this.wrapSelector('.vp-inner-popup-inputastext')).prop('checked'); @@ -2141,6 +3046,10 @@ define([ content['thresh'] = $(this.wrapSelector('.vp-inner-popup-thresh')).val(); content['ignore_index'] = $(this.wrapSelector('.vp-inner-popup-ignoreindex')).val(); break; + case FRAME_EDIT_TYPE.FILL_OUT: + content['filltype'] = $(this.wrapSelector('.vp-inner-popup-filltype')).val(); + content['fillvalue'] = $(this.wrapSelector('.vp-inner-popup-fillvalue')).val(); + break; case FRAME_EDIT_TYPE.DROP_DUP: content['keep'] = $(this.wrapSelector('.vp-inner-popup-how')).val(); content['ignore_index'] = $(this.wrapSelector('.vp-inner-popup-ignoreindex')).val(); @@ -2226,10 +3135,10 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(that.wrapSelector('.' + VP_FE_INFO_CONTENT)).replaceWith(function() { return that.renderInfoPage(errorContent); }); @@ -2237,10 +3146,10 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(that.wrapSelector('.' + VP_FE_INFO_CONTENT)).replaceWith(function() { return that.renderInfoPage(errorContent); }); @@ -2294,22 +3203,26 @@ define([ } break; case FRAME_EDIT_TYPE.DROP_NA: - var locObj = ''; + var dropNAOptions = []; if (axis == FRAME_AXIS.ROW) { - code.appendFormat("{0}.loc[[{1}],:].dropna(axis=0", tempObj, selectedName); + dropNAOptions.push("axis=1"); } else { - code.appendFormat("{0}.loc[:,[{1}]].dropna(axis=1", tempObj, selectedName); + dropNAOptions.push("axis=0"); + } + if (selectedName && selectedName !== '') { + dropNAOptions.push(com_util.formatString("subset=[{0}]", selectedName)); } if (content.how && content.how !== '') { - code.appendFormat(", how='{0}'", content.how); + dropNAOptions.push(com_util.formatString("how='{0}'", content.how)); } if (content.thresh && content.thresh !== '') { - code.appendFormat(", thresh={0}", content.thresh); + dropNAOptions.push(com_util.formatString("thresh={0}", content.thresh)); } if (content.ignore_index && content.ignore_index !== '') { - code.appendFormat(", ignore_index={0}", content.ignore_index); + dropNAOptions.push(com_util.formatString("ignore_index={0}", content.ignore_index)); } - code.append(", inplace=True)"); + dropNAOptions.push("inplace=True"); + code.appendFormat("{0}.dropna({1})", tempObj, dropNAOptions.join(', ')); break; case FRAME_EDIT_TYPE.DROP_DUP: let dropDupOptions = []; @@ -2322,23 +3235,37 @@ define([ if (content.ignore_index && content.ignore_index !== '') { dropDupOptions.push(com_util.formatString("ignore_index={0}", content.ignore_index)); } - dropDupOptions.push(com_util.formatString("inplace=True")); + dropDupOptions.push("inplace=True"); code.appendFormat("{0}.drop_duplicates({1})", tempObj, dropDupOptions.join(', ')); break; + case FRAME_EDIT_TYPE.FILL_OUT: + if (axis == FRAME_AXIS.COLUMN) { + code.appendFormat("{0} = vp_fill_outlier({1}, [{2}], fill_type='{3}'", tempObj, tempObj, selectedName, content.filltype); + if (content.filltype === 'value') { + code.appendFormat(", fill_value_lst={0}", content.fillvalue); + } + code.append(")"); + } + break; case FRAME_EDIT_TYPE.DROP_OUT: if (axis == FRAME_AXIS.COLUMN) { - code.appendFormat("{0} = vp_drop_outlier({1}, {2})", tempObj, tempObj, selectedName); + code.appendFormat("{0} = vp_drop_outlier({1}, [{2}])", tempObj, tempObj, selectedName); } break; case FRAME_EDIT_TYPE.LABEL_ENCODING: if (axis == FRAME_AXIS.COLUMN) { - let encodedColName = this.state.selected.map(col=> { + let encodedColNameList = this.state.selected.map(col=> { if (col.code !== col.label) { - return com_util.formatString("'{0}'", col.label + '_label'); + return { 'origin': com_util.formatString("'{0}'", col.label), 'encoded': com_util.formatString("'{0}'", col.label + '_label') }; + } + return { 'origin': col.label, 'encoded': col.label + '_label' }; + }); + encodedColNameList.forEach((encodedColObj, idx) => { + if (idx > 0) { + code.appendLine(); } - return col.label + '_label' - }).join(','); - code.appendFormat("{0}[{1}] = pd.Categorical({2}[{3}]).codes", tempObj, encodedColName, tempObj, selectedName); + code.appendFormat("{0}[{1}] = pd.Categorical({2}[{3}]).codes", tempObj, encodedColObj['encoded'], tempObj, encodedColObj['origin']); + }); } break; case FRAME_EDIT_TYPE.ONE_HOT_ENCODING: @@ -2382,7 +3309,7 @@ define([ return ''; } var tab = content.addtype; - if (tab == 'variable') { + if (tab == 'calculate') { let values = []; content['values'] && content['values'].forEach((val, idx) => { if (idx > 0) { @@ -2390,9 +3317,111 @@ define([ } values.push(val); }); - code.appendFormat("{0}[{1}] = {2}", tempObj, content.name, values.join(' ')); + let valueStr = values.join(' '); + if (valueStr === "" || valueStr === "''") { + code.appendFormat("{0}[{1}] = np.nan", tempObj, content.name); + } else { + code.appendFormat("{0}[{1}] = {2}", tempObj, content.name, valueStr); + } + } else if (tab == 'statistics') { + code.appendFormat("{0}[{1}] = ", tempObj, content.name); + code.append(tempObj); + if (content['columns'].length > 0) { + code.appendFormat("[[{0}]]", content['columns'].join(',')); + } + code.appendFormat(".apply(lambda x: x.{0}(numeric_only=True), axis=1)", content.method); + // code.appendFormat(".{0}(axis=1)", content.method); // FIXME: for pandas version upper 2.0.0 + } else if (tab == 'replace') { + var replaceStr = new com_String(); + var targetName = content['target']; + var useRegex = content['useregex']; + content['list'].forEach((obj, idx) => { + if (idx == 0) { + replaceStr.appendFormat("{0}: {1}" + , com_util.convertToStr(obj.origin, obj.origintext, useRegex) + , com_util.convertToStr(obj.replace, obj.replacetext, useRegex)); + } else { + replaceStr.appendFormat(", {0}: {1}" + , com_util.convertToStr(obj.origin, obj.origintext, useRegex) + , com_util.convertToStr(obj.replace, obj.replacetext, useRegex)); + } + }); + code.appendFormat("{0}[{1}] = {2}[[{3}]].replace({{4}}", tempObj, content.name, tempObj, targetName, replaceStr); + if (useRegex) { + code.append(', regex=True'); + } + code.append(')'); } else if (tab == 'apply') { - code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); + // code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); + let lambdaCode = 'lambda x: '; + content['caseList'].forEach(obj => { + // value if (cond list) else + let caseCode = obj.value + ' '; + let condCode = ''; + obj.condList.forEach((condObj, idx) => { + let { oper, cond, connector } = condObj; + if (oper === 'isnull()') { + condCode += `(pd.isnull(x))`; + } else if (oper === 'notnull()') { + condCode += `(pd.notnull(x))`; + } else if (oper === 'contains') { + condCode += `(${cond} in x)`; + } else if (oper === 'not contains') { + condCode += `(${cond} not in x)`; + } else if (oper === 'starts with') { + condCode += `(x.startswith(${cond}))`; + } else if (oper === 'ends with') { + condCode += `(x.endswith(${cond}))`; + } else { + condCode += `(x ${oper} ${cond})`; + } + if (connector !== undefined) { + condCode += ` ${connector} `; + } + }); + caseCode += 'if ' + condCode + ' else '; + lambdaCode += caseCode; + }); + lambdaCode += content['else']; + code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.target, lambdaCode); + } else if (tab === 'condition') { + code.appendFormat("{0}.loc[", tempObj); + content['list'].forEach((obj, idx) => { + let { colName, oper, cond, condAsText, connector } = obj; + code.append('('); + + let colValue = tempObj; + if (colName && colName != '') { + if (colName == '.index') { + colValue += colName; + } else { + colValue += com_util.formatString('[{0}]', colName); + } + } + let condValue = com_util.convertToStr(cond, condAsText); + if (oper == 'contains') { + code.appendFormat('{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'not contains') { + code.appendFormat('~{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'starts with') { + code.appendFormat('{0}.str.startswith({1})', colValue, condValue); + } else if (oper == 'ends with') { + code.appendFormat('{0}.str.endswith({1})', colValue, condValue); + } else if (oper == 'isnull()' || oper == 'notnull()') { + code.appendFormat('{0}.{1}', colValue, oper); + } else { + code.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); + } + code.append(')'); + if (idx < (content['list'].length - 1)) { + code.append(connector); + } + }); + var value = com_util.convertToStr(content.value, content.valueastext); + if (value === '') { + value = 'np.nan'; + } + code.appendFormat(", {0}] = {1}", content.name, value); } break; case FRAME_EDIT_TYPE.ADD_ROW: @@ -2411,14 +3440,9 @@ define([ code.appendFormat("{0}.loc[{1}] = {2}", tempObj, content.name, values.join(' ')); break; case FRAME_EDIT_TYPE.REPLACE: - // if no name entered - if (content.name == '') { - return ''; - } - var name = com_util.convertToStr(content.name, content.nameastext); - name = selectedName; + var name = content.name; var tab = content.replacetype; - if (tab === 'value') { + if (tab === 'replace') { var replaceStr = new com_String(); var useRegex = content['useregex']; content['list'].forEach((obj, idx) => { @@ -2441,8 +3465,80 @@ define([ } code.append(')'); } else if (tab === 'condition') { + code.appendFormat("{0}.loc[", tempObj); + var condCode = new com_String(); + content['list'].forEach((obj, idx) => { + let { colName, oper, cond, condAsText, connector } = obj; + condCode.append('('); + + let colValue = tempObj; + if (colName && colName != '') { + if (colName == '.index') { + colValue += colName; + } else { + colValue += com_util.formatString('[{0}]', colName); + } + } + let condValue = com_util.convertToStr(cond, condAsText); + if (oper == 'contains') { + condCode.appendFormat('{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'not contains') { + condCode.appendFormat('~{0}.str.contains({1})', colValue, condValue); + } else if (oper == 'starts with') { + condCode.appendFormat('{0}.str.startswith({1})', colValue, condValue); + } else if (oper == 'ends with') { + condCode.appendFormat('{0}.str.endswith({1})', colValue, condValue); + } else if (oper == 'isnull()' || oper == 'notnull()') { + condCode.appendFormat('{0}.{1}', colValue, oper); + } else { + condCode.appendFormat('{0}{1}{2}', colValue, oper != ''?(' ' + oper):'', condValue != ''?(' ' + condValue):''); + } + condCode.append(')'); + if (idx < (content['list'].length - 1)) { + condCode.append(connector); + } + }); + if (condCode.toString() === '') { + condCode.append(':'); + } var value = com_util.convertToStr(content.value, content.valueastext); - code.appendFormat("{0} = {1}", content.subset, value); + if (value === '') { + value = 'np.nan'; + } + code.appendFormat("{0}, {1}] = {2}", condCode.toString(), content.name, value); + } else if (tab == 'apply') { + // code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.column, content.apply); + let lambdaCode = 'lambda x: '; + content['caseList'].forEach(obj => { + // value if (cond list) else + let caseCode = obj.value + ' '; + let condCode = ''; + obj.condList.forEach((condObj, idx) => { + let { oper, cond, connector } = condObj; + if (oper === 'isnull()') { + condCode += `pd.isnull(x)`; + } else if (oper === 'notnull()') { + condCode += `pd.notnull(x)`; + } else if (oper === 'contains') { + condCode += `(${cond} in x)`; + } else if (oper === 'not contains') { + condCode += `(${cond} not in x)`; + } else if (oper === 'starts with') { + condCode += `(x.startswith(${cond}))`; + } else if (oper === 'ends with') { + condCode += `(x.endswith(${cond}))`; + } else { + condCode += `(x ${oper} ${cond})`; + } + if (connector !== undefined) { + condCode += ` ${connector} `; + } + }); + caseCode += 'if ' + condCode + ' else '; + lambdaCode += caseCode; + }); + lambdaCode += content['else']; + code.appendFormat("{0}[{1}] = {2}[{3}].apply({4})", tempObj, content.name, tempObj, content.target, lambdaCode); } break; case FRAME_EDIT_TYPE.AS_TYPE: @@ -2456,6 +3552,31 @@ define([ }); code.appendFormat("{0} = {1}.astype({{2}})", tempObj, tempObj, astypeStr.toString()); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + code.appendFormat("{0}['{1}'] = pd.to_datetime({2}[{3}]", tempObj, content['newcol'], tempObj, selectedName); + let optionList = []; + if (content['format'] === 'auto') { + if (content['dayfirst'] !== '') { + optionList.push(`dayfirst=${content['dayfirst']}`); + } + } else if (content['format'] === 'typing') { + if (content['format_typing'] !== '') { + optionList.push(`format='${content['format_typing']}'`); + } + } else { + optionList.push(`format='${content['format']}'`); + } + if (optionList.length > 0) { + code.appendFormat(', {0}', optionList.join(', ')); + } + code.append(')'); + if (content['collist'].length > 0) { + content['collist'].forEach(obj => { + code.appendLine(); + code.appendFormat("{0}['{1}'] = {2}['{3}'].dt.{4}", tempObj, obj.colName, tempObj, content['newcol'], obj.dateType); + }); + } + break; case FRAME_EDIT_TYPE.DISCRETIZE: let newColumn = com_util.convertToStr(content['input'], content['inputastext']); let method = content['type']; @@ -2493,11 +3614,18 @@ define([ code.appendFormat("{0} = {1}.fillna(", subsetObjStr, subsetObjStr); if (content['method'] == 'value') { code.append(com_util.convertToStr(content['value'], content['valueastext'])); - } else { + } else if (content['method'] === 'ffill' || content['method'] === 'bfill') { code.appendFormat("method='{0}'", content['method']); if (content['limit'] && content['limit'] !== '') { code.appendFormat(", limit={0}", content['limit']); } + } else { + if (content['method'] === 'mode') { + // get mode()'s first element (mode returns Series) + code.appendFormat("{0}.{1}()[0]", subsetObjStr, content['method']); + } else { + code.appendFormat("{0}.{1}()", subsetObjStr, content['method']); + } } code.append(')'); break; @@ -2526,8 +3654,9 @@ define([ code.appendLine(codeStr); code.appendFormat("{0}.iloc[{1}:{2}].to_json(orient='{3}')", tempObj, prevLines, lines, 'split'); + this.loading = true; - vpKernel.execute(code.toString()).then(function(resultObj) { + vpKernel.execute(com_util.ignoreWarning(code.toString())).then(function(resultObj) { let { result } = resultObj; try { if (!result || result.length <= 0) { @@ -2547,7 +3676,7 @@ define([ var indexList = data.index; var dataList = data.data; - columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value } }); + columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value, isNumeric: col.is_numeric } }); indexList = indexList.map(idx => { return { label: idx, code: idx } }); if (!more) { @@ -2563,6 +3692,12 @@ define([ while (colIdx < columnList.length) { let col = columnList[colIdx]; let colCode = col.code.slice(0, colLevIdx + 1).join(','); + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } let nextCol = columnList[colIdx + 1]; if (nextCol && nextCol.code.slice(0, colLevIdx + 1).join(',') === colCode) { colSpan++; @@ -2577,8 +3712,8 @@ define([ } else { colClass = VP_FE_TABLE_COLUMN_GROUP; } - table.appendFormatLine('{8}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, col.label[colLevIdx]); + table.appendFormatLine('{8}{9}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, colIcon, col.label[colLevIdx]); colSpan = 1; } colIdx++; @@ -2587,9 +3722,9 @@ define([ // add column // LAB: img to url // table.appendFormatLine('

', VP_FE_ADD_COLUMN, com_Const.IMAGE_PATH + 'plus.svg'); - // if (colLevIdx === 0) { - // table.appendFormatLine('

', VP_FE_ADD_COLUMN, 'vp-icon-plus'); - // } + if (colLevIdx === 0) { + table.appendFormatLine('

', VP_FE_ADD_COLUMN, 'vp-icon-plus'); + } table.appendLine(''); } @@ -2597,15 +3732,21 @@ define([ table.appendLine(''); columnList && columnList.forEach(col => { var colCode = col.code; + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } var colClass = ''; if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code).includes(colCode)) { colClass = 'selected'; } - table.appendFormatLine('{6}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, col.label); + table.appendFormatLine('{6}{7}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, colIcon, col.label); }); // // add column - // table.appendFormatLine('

', VP_FE_ADD_COLUMN, 'vp-icon-plus'); + table.appendFormatLine('

', VP_FE_ADD_COLUMN, 'vp-icon-plus'); table.appendLine(''); } @@ -2698,6 +3839,11 @@ define([ var replacedCode = codeStr.replaceAll(that.state.tempObj, that.state.returnObj); that.setPreview(replacedCode); } + + // if add column, go to the right position + if (that.state.popup.type === FRAME_EDIT_TYPE.ADD_COL) { + $(that.wrapSelector('.vp-fe-add-column'))[0].scrollIntoView(); + } // if scrollPos is saved, go to the position if (scrollPos >= 0) { @@ -2716,11 +3862,21 @@ define([ that.loading = false; } }).catch(function(resultObj) { - let { result, type, msg } = resultObj; - vpLog.display(VP_LOG_TYPE.ERROR, result.ename + ': ' + result.evalue, msg, code.toString()); - if (that.isHidden() == false) { - // show alert modal only if this popup is visible - com_util.renderAlertModal(result.ename + ': ' + result.evalue); + let { result, msg, ename, evalue, status } = resultObj; + if (result) { + // NOTEBOOK: notebook error FIXME: migrate it on com_Kernel.execute + vpLog.display(VP_LOG_TYPE.ERROR, result?.ename + ': ' + result?.evalue, msg, code.toString()); + if (that.isHidden() == false) { + // show alert modal only if this popup is visible + com_util.renderAlertModal(result?.ename + ': ' + result?.evalue, { content: code.toString(), type: 'code' }); + } + } else { + // LAB: lab error FIXME: migrate it on com_Kernel.execute + vpLog.display(VP_LOG_TYPE.ERROR, ename + ': ' + evalue, status, code.toString()); + if (that.isHidden() == false) { + // show alert modal only if this popup is visible + com_util.renderAlertModal(ename + ': ' + evalue, { content: code.toString(), type: 'code' }); + } } that.loading = false; }); @@ -2808,6 +3964,7 @@ define([ DROP: 3, RENAME: 2, AS_TYPE: 10, + TO_DATETIME: 19, REPLACE: 9, DISCRETIZE: 15, @@ -2823,7 +3980,8 @@ define([ FILL_NA: 13, DROP_NA: 4, - DROP_OUT: 11, + FILL_OUT: 18, + DROP_OUT: 11, DROP_DUP: 5, SHOW: 99 diff --git a/visualpython/js/m_apps/Groupby.js b/visualpython/js/m_apps/Groupby.js index d1aff5d4..589882f0 100644 --- a/visualpython/js/m_apps/Groupby.js +++ b/visualpython/js/m_apps/Groupby.js @@ -177,7 +177,7 @@ define([ var colList = event.dataList; that.state.display = colList; - if (colList && colList.length == 1) { + if ((colList && colList.length == 1) || that.state.method === 'size') { $(that.wrapSelector('#vp_gbToFrame')).parent().show(); } else { $(that.wrapSelector('#vp_gbToFrame')).parent().hide(); @@ -196,6 +196,12 @@ define([ var method = $(this).val(); that.state.method = method; $(that.wrapSelector('#vp_gbMethod')).val(method); + + if (method === 'size' || (that.state.display && that.state.display.length == 1)) { + $(that.wrapSelector('#vp_gbToFrame')).parent().show(); + } else { + $(that.wrapSelector('#vp_gbToFrame')).parent().hide(); + } }); // advanced checkbox event @@ -543,20 +549,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } @@ -643,7 +649,7 @@ define([ page.appendFormatLine('

', 'vp-gb-method-user'); page.appendFormatLine('{0}', 'User option'); page.appendFormatLine('', 'Type user method'); - page.appendFormatLine('{0}' + page.appendFormatLine('{0}' , 'Text'); page.appendLine(''); page.appendLine('

'); @@ -744,7 +750,7 @@ define([ // Display columns //==================================================================== var colStr = ''; - if (display) { + if (display && display.length > 0) { if (toFrame || display.length > 1) { // over 2 columns colStr = '[[' + display.join(',') + ']]'; @@ -886,7 +892,16 @@ define([ // Method code generation //================================================================ if (method != '') { - methodStr.appendFormat('{0}()', method); + let numericOnlyList = ['std', 'sum', 'mean', 'median', 'quantile']; + if (numericOnlyList.includes(method)) { + methodStr.appendFormat('{0}(numeric_only=True)', method); + } else { + methodStr.appendFormat('{0}()', method); + if (method === 'size' && toFrame === true) { + // if to_Frame on size() method + methodStr.append(".to_frame(name='size')"); + } + } } } diff --git a/visualpython/js/m_apps/Import.js b/visualpython/js/m_apps/Import.js index 5621b637..bc0321d5 100644 --- a/visualpython/js/m_apps/Import.js +++ b/visualpython/js/m_apps/Import.js @@ -22,26 +22,27 @@ define([ const importTemplates = { 'data-analysis': [ - { i0: 'numpy', i1: 'np', type: 'module'}, - { i0: 'pandas', i1: 'pd', type: 'module'}, + { i0: 'numpy', i1: 'np', type: 'module', checked: 'checked'}, + { i0: 'pandas', i1: 'pd', type: 'module', checked: 'checked'}, { i0: 'matplotlib.pyplot', i1: 'plt', type: 'module' , include: [ '%matplotlib inline' - ] + ], checked: 'checked' }, - { i0: 'seaborn', i1: 'sns', type: 'module'}, + { i0: 'seaborn', i1: 'sns', type: 'module', checked: 'checked'}, { i0: 'plotly.express', i1: 'px', type: 'module' , include: [ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' - ], checked: false - } + ], checked: '' + }, + { i0: 'pyarrow', i1: 'pa', type: 'module', checked: ''}, ], 'machine-learning': [ - { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function' }, - { i0: 'sklearn', i1: 'metrics', type: 'function' } + { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function', checked: 'checked' }, + { i0: 'sklearn', i1: 'metrics', type: 'function', checked: 'checked' } ] } @@ -69,8 +70,10 @@ define([ ...this.state } + this.importTemplatesCopy = JSON.parse(JSON.stringify(importTemplates)); + if (!this.state.importMeta || this.state.importMeta.length <= 0) { - this.state.importMeta = JSON.parse(JSON.stringify(importTemplates[this.state.tabType])); + this.state.importMeta = JSON.parse(JSON.stringify(this.importTemplatesCopy[this.state.tabType])); } } @@ -86,7 +89,7 @@ define([ $(that.wrapSelector('.vp-tab-button')).removeClass('vp-tab-selected'); $(this).addClass('vp-tab-selected'); // replace libraries - that.state.importMeta = importTemplates[tabType]; + that.state.importMeta = that.importTemplatesCopy[tabType]; $(that.wrapSelector('#vp_tblImport')).replaceWith(function() { return that.templateTable(that.state.importMeta); }); @@ -172,9 +175,9 @@ define([ let that = this; libraries && libraries.forEach((lib, idx) => { if (lib.type == 'function') { - page.appendLine(that.templateForFunction(idx, lib.i0, lib.i1, lib.checked)); + page.appendLine(that.templateForFunction(idx, lib.i0, lib.i1, (lib.checked === 'checked' || lib.checked === true))); } else { - page.appendLine(that.templateForModule(idx, lib.i0, lib.i1, lib.checked)); + page.appendLine(that.templateForModule(idx, lib.i0, lib.i1, (lib.checked === 'checked' || lib.checked === true))); } }); page.appendLine(''); @@ -243,7 +246,7 @@ define([ if (pacI0 == "") { continue; } - if (pacChecked) { + if (pacChecked === true) { if (sbCode.toString().trim().length > 0) { sbCode.appendLine(); } @@ -267,7 +270,7 @@ define([ } } - importMeta.push({ i0: pacI0, i1: pacI1, type: pacType, checked: pacChecked }); + importMeta.push({ i0: pacI0, i1: pacI1, type: pacType, checked: (pacChecked?'checked':'') }); } this.state.importMeta = importMeta; diff --git a/visualpython/js/m_apps/Information.js b/visualpython/js/m_apps/Information.js index 881a2e4a..319c8877 100644 --- a/visualpython/js/m_apps/Information.js +++ b/visualpython/js/m_apps/Information.js @@ -17,10 +17,11 @@ define([ __VP_CSS_LOADER__('vp_base/css/m_apps/information'), // INTEGRATION: unified version of css loader 'vp_base/js/com/com_String', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/DataSelector', 'vp_base/js/com/component/LoadingSpinner' -], function(varHtml, varCss, com_String, com_util, PopupComponent, DataSelector, LoadingSpinner) { +], function(varHtml, varCss, com_String, com_util, com_interface, PopupComponent, DataSelector, LoadingSpinner) { /** * Information @@ -30,7 +31,7 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 4; + this.config.sizeLevel = 5; this.config.dataview = false; this.config.runButton = false; this.config.checkModules = ['pd', 'plt']; @@ -74,15 +75,37 @@ define([ code: "pd.DataFrame({'Null Count': ${data}.isnull().sum(), 'Non-Null Count': ${data}.notnull().sum()})", dtype: ['DataFrame', 'Series'], toframe: true }, // { id: 'duplicates', label: 'Duplicated', code: '${data}.duplicated()', dtype: ['DataFrame', 'Series'] }, { id: 'duplicates', label: 'Duplicated', code: "_duplicated = ([${data}.duplicated().sum()] + [${data}[col].duplicated().sum() for col in ${data}.columns])\ - \n_duplicated_df = pd.DataFrame({\ - \n 'Rows':[len(${data})]*len(_duplicated),\ - \n 'Unique':[len(${data}) - dups for dups in _duplicated],\ - \n 'Duplicated': _duplicated,\ - \n 'Duplicated by': ['All columns'] + ${data}.columns.to_list()\ - \n}, index=['Combination']+${data}.columns.to_list())\ - \n_duplicated_df", dtype: ['DataFrame', 'Series'], toframe: true }, +\n_duplicated_df = pd.DataFrame({\ +\n 'Rows':[len(${data})]*len(_duplicated),\ +\n 'Unique':[len(${data}) - dups for dups in _duplicated],\ +\n 'Duplicated': _duplicated,\ +\n 'Duplicated by': ['All columns'] + ${data}.columns.to_list()\ +\n}, index=['Combination']+${data}.columns.to_list())\ +\n_duplicated_df", dtype: ['DataFrame', 'Series'], toframe: true }, { id: 'unique', label: 'Unique', code: '${data}.unique()', dtype: ['Series'] }, - { id: 'value_counts', label: 'Value counts', code: '${data}.value_counts()', dtype: ['DataFrame', 'Series'] }, + { id: 'value_counts', label: 'Value counts', + // code: "_value_counts_dict = {}\ + // \nfor col in ${data}.columns:\ + // \n if pd.api.types.is_numeric_dtype(${data}[col]):\ + // \n _value_counts = ${data}[col].value_counts(bins=10, sort=False)\ + // \n _value_counts_dict[(col, 'bins')] = list(_value_counts.index) + ['']*(10 - len(_value_counts))\ + // \n else:\ + // \n _value_counts = ${data}[col].value_counts()\ + // \n _value_counts_dict[(col, 'category')] = list(_value_counts.index) + ['']*(10 - len(_value_counts))\ + // \n _value_counts_dict[(col, 'count')] = list(_value_counts.values) + ['']*(10 - len(_value_counts))\ + // \npd.DataFrame(_value_counts_dict)", + code: "_dfr = pd.DataFrame()\ +\nfor col in ${data}.columns:\ +\n if pd.api.types.is_numeric_dtype(${data}[col]) and ${data}[col].value_counts().size > 10:\ +\n _value_counts = ${data}[col].value_counts(bins=10, sort=False)\ +\n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'bins'): _value_counts.index})], axis=1)\ +\n else:\ +\n _value_counts = ${data}[col].value_counts()\ +\n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'category'): _value_counts.index})], axis=1)\ +\n _dfr = pd.concat([_dfr, pd.DataFrame({(col,'count'): _value_counts.values})], axis=1)\ +\n_dfr.replace(np.nan,'')", + code2: "${data}.value_counts()", + dtype: ['DataFrame', 'Series'], toframe: true }, ] }, { @@ -90,17 +113,20 @@ define([ label: 'Statistics', dtype: ['DataFrame', 'Series'], child: [ + /** checkbox */ { id: 'count', label: 'count', code: '${data}.count()' }, - { id: 'min', label: 'min', code: '${data}.min()' }, - { id: 'max', label: 'max', code: '${data}.max()' }, + { id: 'min', label: 'min', code: '${data}.min(numeric_only=True)' }, + { id: 'max', label: 'max', code: '${data}.max(numeric_only=True)' }, { id: 'quantile', label: 'quantile', code: '${data}.quantile(numeric_only=True)' }, - { id: 'sum', label: 'sum', code: '${data}.sum()' }, + { id: 'sum', label: 'sum', code: '${data}.sum(numeric_only=True)' }, { id: 'mean', label: 'mean', code: '${data}.mean(numeric_only=True)' }, { id: 'median', label: 'median', code: '${data}.median(numeric_only=True)' }, // { id: 'mad', label: 'mad', code: '${data}.mad(numeric_only=True)' }, // FutureWarning: Deprecated and will be removed { id: 'var', label: 'var', code: '${data}.var(numeric_only=True)' }, { id: 'std', label: 'std', code: '${data}.std(numeric_only=True)' }, { id: 'skew', label: 'skew', code: '${data}.skew(numeric_only=True)' }, + { id: 'kurtosis', label: 'kurtosis', code: '${data}.kurtosis(numeric_only=True)' }, + /** radio */ { id: 'cumsum', label: 'cumsum', code: '${data}.cumsum()', type: 'radio' }, { id: 'cummin', label: 'cummin', code: '${data}.cummin()', type: 'radio' }, { id: 'cummax', label: 'cummax', code: '${data}.cummax()', type: 'radio' }, @@ -129,7 +155,7 @@ define([ , code: "pd.plotting.scatter_matrix(${data}, marker='o', hist_kwds={'bins': 30}, s=30, alpha=.8)\nplt.show()" , dtype: ['DataFrame'] }, { id: 'boxplot', label: 'Box plot', code: "${data}.plot(kind='box')\nplt.show()", dtype: ['DataFrame', 'Series'] }, - { id: 'count_plot', label: 'Count plot', code: "${data}.value_counts().plot(kind='bar')\nplt.show()", dtype: ['DataFrame', 'Series'] } + { id: 'count_plot', label: 'Count plot', code: "${data}.value_counts().plot(kind='bar', rot=30)\nplt.show()", dtype: ['DataFrame', 'Series'] } ] } @@ -382,15 +408,17 @@ define([ // click run button $(this.wrapSelector('.vp-information-run-button')).click(function(event) { // get code - var code = that.generateCode(); + var code = that.generateCodeForInfo(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Data Info'); }); } @@ -402,6 +430,7 @@ define([ this.state.selected = []; this.state.selection = { start: -1, end: -1 }; this.renderMenu(); + $(this.wrapSelector('.' + VP_FE_TABLE)).html(''); this.loadCode(data); this.loadInfo(data, this.state.menu); } @@ -507,6 +536,10 @@ define([ } generateCode() { + return this.tempCode; + } + + generateCodeForInfo() { let { data, dtype, menu, menuItem } = this.state; var selected = []; @@ -549,7 +582,7 @@ define([ let childObj = infoObj.child.find(obj=>obj.id === itemId); statList.push(com_util.formatString("'{0}': {1}", itemId, childObj.code)); }); - if (currentDtype === 'Series') { + if (currentDtype === 'Series' && selected.length > 0) { // if multiple stats selected, set series data as dataframe dataVar = new com_String(); dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); @@ -557,6 +590,12 @@ define([ } codePattern = com_util.formatString("pd.DataFrame({{0}})", statList.join(',')); } else { + if (currentDtype === 'Series' && selected.length > 0) { + // if multiple stats selected, set series data as dataframe + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } let childObj = infoObj.child.find(obj=>obj.id === menuItem[0]); codePattern = childObj.code; } @@ -567,12 +606,26 @@ define([ // only one method selected if (menuItem.length > 0 && infoObj.child) { let childObj = infoObj.child.find(obj=>obj.id === menuItem[0]); - if (childObj.toframe === true && currentDtype === 'Series') { - dataVar = new com_String(); - dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); - currentDtype = 'DataFrame'; + if (menuItem[0] === 'value_counts') { + if (currentDtype === 'Series') { + codePattern = childObj.code2; + } else { + codePattern = childObj.code; + } + } else { + if (childObj.toframe === true) { + if (dtype === 'Series') { + dataVar = new com_String(); + dataVar.appendFormat("{0}.to_frame()", data); + currentDtype = 'DataFrame'; + } else if (currentDtype === 'Series') { // DataFrame with single column selected + dataVar = new com_String(); + dataVar.appendFormat("{0}[[{1}]]", data, selected.map(col=>col.code).join(',')); + currentDtype = 'DataFrame'; + } + } + codePattern = childObj.code; } - codePattern = childObj.code; } else { codePattern = infoObj.code; } @@ -773,11 +826,11 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { - errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + if (msg.content && msg.content.ename) { + errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content?.evalue, msg.content?.detail); } + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); $(variablePreviewTag).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); that.loading = false; }); @@ -802,7 +855,8 @@ define([ // load preview content let $infoPreviewTag = $(this.wrapSelector('#informationPreview')); $infoPreviewTag.html(''); - let code = this.generateCode(); + let code = this.generateCodeForInfo(); + this.tempCode = code; // use default pandas option // let defaultPOCode = new com_String(); @@ -832,18 +886,18 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $infoPreviewTag.html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg, ename, evalue } = resultObj; var errorContent = ''; if (msg && msg?.content?.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } else if (ename && evalue) { errorContent = com_util.templateForErrorBox(ename, evalue); vpLog.display(VP_LOG_TYPE.ERROR, ename, evalue, resultObj); diff --git a/visualpython/js/m_apps/Instance.js b/visualpython/js/m_apps/Instance.js index 129162b2..1095ef31 100644 --- a/visualpython/js/m_apps/Instance.js +++ b/visualpython/js/m_apps/Instance.js @@ -33,7 +33,7 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 4; + this.config.sizeLevel = 5; this.config.checkModules = ['pd']; this.state = { @@ -208,10 +208,11 @@ define([ }); $(page).find('#vp_instanceTarget').replaceWith(targetSelector.toTagString()); - let allocateSelector = new DataSelector({ - pageThis: this, id: 'vp_instanceAllocate', placeholder: 'Variable name' - }); - $(page).find('#vp_instanceAllocate').replaceWith(allocateSelector.toTagString()); + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'vp_instanceAllocate', placeholder: 'Variable name' + // }); + // $(page).find('#vp_instanceAllocate').replaceWith(allocateSelector.toTagString()); return page; } @@ -262,7 +263,8 @@ define([ let cmObj = this.getCodemirror('vp_instanceVariable'); let rightCode = (cmObj && cmObj.cm)?cmObj.cm.getValue():''; if (leftCode && leftCode != '') { - sbCode.appendFormat('{0} = {1}', leftCode, rightCode); + sbCode.appendFormatLine('{0} = {1}', leftCode, rightCode); + sbCode.append(leftCode); // show allocation (from version 2.4.10) } else { sbCode.appendFormat('{0}', rightCode); } @@ -330,20 +332,25 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#instancePreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { - let { msg } = resultObj; + let { msg, ename, evalue, status } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg && msg.content && msg.content.ename) { + // NOTEBOOK: notebook error FIXME: migrate it on com_Kernel.execute errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); + } else { + // LAB: lab error FIXME: migrate it on com_Kernel.execute + errorContent = com_util.templateForErrorBox(ename, evalue, ''); + vpLog.display(VP_LOG_TYPE.ERROR, ename, evalue); } $(that.wrapSelector('#instancePreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); }); } diff --git a/visualpython/js/m_apps/Markdown.js b/visualpython/js/m_apps/Markdown.js index 69b6ba2b..ccceec89 100644 --- a/visualpython/js/m_apps/Markdown.js +++ b/visualpython/js/m_apps/Markdown.js @@ -221,7 +221,7 @@ define([ MathJax.Hub.Queue(["Typeset", MathJax.Hub, "vp_markdownPreview"]); }); - } else if (vpConfig.extensionType === 'lab') { + } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { var marked = require('marked'); // get block diff --git a/visualpython/js/m_apps/PDF.js b/visualpython/js/m_apps/PDF.js index c4ea125b..e5fdf2ce 100644 --- a/visualpython/js/m_apps/PDF.js +++ b/visualpython/js/m_apps/PDF.js @@ -21,9 +21,9 @@ define([ 'vp_base/js/com/component/FileNavigation' ], function(pdfHtml, pdfCss, com_String, com_interface, PopupComponent, FileNavigation) { - const PDF_SHOW = '!pip show PyMuPDF nltk' - const PDF_INSTALL1 = '!pip install PyMuPDF' - const PDF_INSTALL2 = '!pip install nltk' + var PDF_SHOW = '!pip show PyMuPDF nltk' + var PDF_INSTALL1 = '!pip install PyMuPDF' + var PDF_INSTALL2 = '!pip install nltk' const PDF_IMPORT = `import pandas as pd import fitz @@ -80,6 +80,12 @@ nltk.download('punkt')`; vp_pdfReturn: '', ...this.state } + + if (vpConfig.extensionType === 'lite') { + PDF_SHOW = PDF_SHOW.replace('!', '%'); + PDF_INSTALL1 = PDF_INSTALL1.replace('!', '%'); + PDF_INSTALL2 = PDF_INSTALL2.replace('!', '%'); + } } _bindEvent() { diff --git a/visualpython/js/m_apps/PandasOption.js b/visualpython/js/m_apps/PandasOption.js index 0d6d1dfd..359de409 100644 --- a/visualpython/js/m_apps/PandasOption.js +++ b/visualpython/js/m_apps/PandasOption.js @@ -17,9 +17,8 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', - 'vp_base/js/com/component/PopupComponent', - 'vp_base/js/com/component/DataSelector' -], function(poHTML, com_util, com_Const, com_String, PopupComponent, DataSelector) { + 'vp_base/js/com/component/PopupComponent' +], function(poHTML, com_util, com_Const, com_String, PopupComponent) { /** * PandasOption @@ -31,6 +30,7 @@ define([ this.config.sizeLevel = 2; this.config.dataview = false; this.config.checkModules = ['pd']; + this.config.docs = 'https://pandas.pydata.org/docs/user_guide/options.html'; this.state = { filter_warning: '', diff --git a/visualpython/js/m_apps/Reshape.js b/visualpython/js/m_apps/Reshape.js index 022739a3..995ddd1e 100644 --- a/visualpython/js/m_apps/Reshape.js +++ b/visualpython/js/m_apps/Reshape.js @@ -277,7 +277,7 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); @@ -285,7 +285,7 @@ define([ }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); diff --git a/visualpython/js/m_apps/Snippets.js b/visualpython/js/m_apps/Snippets.js index d0d76129..1fecfce7 100644 --- a/visualpython/js/m_apps/Snippets.js +++ b/visualpython/js/m_apps/Snippets.js @@ -18,10 +18,11 @@ define([ 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/LoadingSpinner' -], function(snHtml, snCss, com_util, com_Const, com_String, PopupComponent, FileNavigation, LoadingSpinner) { +], function(snHtml, snCss, com_util, com_Const, com_String, com_interface, PopupComponent, FileNavigation, LoadingSpinner) { /** * Snippets @@ -126,7 +127,7 @@ define([ filesPath.forEach(fileObj => { var fileName = fileObj.file; var selectedPath = fileObj.path; - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { vpKernel.readFile(selectedPath).then(function(resultObj) { try { var snippetData = JSON.parse(resultObj.result); @@ -412,7 +413,7 @@ define([ // prevent occuring header click event by clicking input $(this.wrapSelector('.vp-sn-item-title')).off('click'); $(this.wrapSelector('.vp-sn-item-title')).on('click', function(evt) { - evt.stopPropagation(); + // evt.stopPropagation(); }); // item title save @@ -471,14 +472,16 @@ define([ let cmCode = that.codemirrorList[title]; cmCode.save(); var code = cmCode.getValue(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Data Analysis > Snippets'); } else if (menu == 'duplicate') { var dupNo = 1; var timestamp = new Date().getTime(); @@ -509,7 +512,7 @@ define([ } else if (menu == 'delete') { let loadingSpinner = new LoadingSpinner($(that.wrapSelector('.vp-sn-table'))); // remove key - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { vpConfig.getData('').then(function(data) { let dataObj = data; // remove data @@ -522,7 +525,7 @@ define([ // vp-multilang for success message com_util.renderSuccessMessage('Successfully removed!'); }).catch(function(err) { - com_util.renderAlertModal('Failed to remove data...', err); + com_util.renderAlertModal('Failed to remove data...', { content: err }); // load again that.loadUdfList(); }).finally(function() { @@ -540,7 +543,7 @@ define([ // vp-multilang for success message com_util.renderSuccessMessage('Successfully removed!'); }).catch(function(err) { - com_util.renderAlertModal('Failed to remove data...', err); + com_util.renderAlertModal('Failed to remove data...', { content: err }); // load again that.loadUdfList(); }).finally(function() { @@ -666,21 +669,23 @@ define([ // load udf list to table 'vp_udfList' let loadingSpinner = new LoadingSpinner($(this.wrapSelector('.vp-sn-table'))); vpConfig.getData().then(function(udfObj) { - vpLog.display(VP_LOG_TYPE.DEVELOP, udfObj); + vpLog.display(VP_LOG_TYPE.DEVELOP, 'Snippets get data', udfObj); var snippets = new com_String(); - Object.keys(udfObj).forEach(key => { - let obj = udfObj[key]; - if (obj.code != null && obj.code != undefined) { - - var hasImported = false; - if (that.importedList.includes(key)) { - // set new label - hasImported = true; + if (udfObj !== undefined) { + Object.keys(udfObj).forEach(key => { + let obj = udfObj[key]; + if (obj.code != null && obj.code != undefined) { + + var hasImported = false; + if (that.importedList.includes(key)) { + // set new label + hasImported = true; + } + var item = that.renderSnippetItem(key, obj.code, obj.timestamp, hasImported); + snippets.append(item); } - var item = that.renderSnippetItem(key, obj.code, obj.timestamp, hasImported); - snippets.append(item); - } - }); + }); + } $(that.wrapSelector('.vp-sn-table')).html(snippets.toString()); // bind snippet item @@ -693,7 +698,8 @@ define([ that.bindCodeMirror(title, tag); }); }).catch(function(err) { - com_util.renderAlertModal(err); + // com_util.renderAlertModal(err); + vpLog.display(VP_LOG_TYPE.ERROR, err); }).finally(function() { loadingSpinner.remove(); }); diff --git a/visualpython/js/m_apps/Subset.js b/visualpython/js/m_apps/Subset.js index a6272232..41af3ef4 100644 --- a/visualpython/js/m_apps/Subset.js +++ b/visualpython/js/m_apps/Subset.js @@ -26,6 +26,20 @@ define([ /** * Subset + * ==================================== + * Special mode + * 1. useAsModule : Use subset as module like DataSelector + * - No allocation + * - No run to cell (able to use apply button instead) + * - renders button to target + * 2. useInputVariable : Use subset as module but use applied variable + * - No allocation + * - No data selection + * - No run to cell + * - renders button to target + * 3. useInputColumns : Use subset as module but use applied columns + * - No allocation + * - No column selection */ class Subset extends PopupComponent { _init() { @@ -39,8 +53,9 @@ define([ this.targetSelector = this.prop.targetSelector; this.pageThis = this.prop.pageThis; + this.useAsModule = this.prop.useAsModule; this.useInputVariable = this.prop.useInputVariable; - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { this.eventTarget = this.targetSelector; this.useCell = false; // show apply button only } @@ -77,6 +92,7 @@ define([ useCopy: false, toFrame: false, subsetType: 'loc', // subset / loc / iloc / query + returnType: '', rowType: 'condition', rowList: [], @@ -115,6 +131,13 @@ define([ this.loadStateAfterRender(); // render button + if (this.useAsModule) { + // render button + this.renderButton(); + + // hide allocate to + $(this.wrapSelector('.' + VP_DS_ALLOCATE_TO)).closest('tr').hide(); + } if (this.useInputVariable) { // set readonly $(this.wrapSelector('.' + VP_DS_PANDAS_OBJECT)).attr('disabled', true); @@ -145,13 +168,14 @@ define([ let page = $(subsetHtml); let that = this; - let allocateSelector = new DataSelector({ - pageThis: this, id: 'allocateTo', classes: VP_DS_ALLOCATE_TO, placeholder: 'New variable name', - finish: function() { - that.generateCode(); - } - }); - $(page).find('.' + VP_DS_ALLOCATE_TO).replaceWith(allocateSelector.toTagString()); + // Removed dataselector for Allocation input + // let allocateSelector = new DataSelector({ + // pageThis: this, id: 'allocateTo', classes: VP_DS_ALLOCATE_TO, placeholder: 'New variable name', + // finish: function() { + // that.generateCode(); + // } + // }); + // $(page).find('.' + VP_DS_ALLOCATE_TO).replaceWith(allocateSelector.toTagString()); return page; } @@ -175,7 +199,7 @@ define([ buttonTag.appendFormat('', VP_DS_BTN, this.uuid, 'vp-button', 'Subset'); if (this.pageThis) { - $(this.targetSelector).parent().append(buttonTag.toString()); + $(buttonTag.toString()).insertAfter($(this.targetSelector)); } } renderSubsetType(dataType) { @@ -319,10 +343,13 @@ define([ $(this.wrapSelector('.select-row .vp-ds-select-box.left')).on('scroll', function() { if ($(this).scrollTop() + $(this).innerHeight() >= ($(this)[0].scrollHeight - 2)) { let scrollPos = $(this).scrollTop(); + if (that.state.rowLimit > that.state.rowList.length){ + return; // Prevents scroll from being fixed downwards + } let start = that.state.rowLimit; let end = start + 10; let subsetVariable = com_util.formatString('{0}.iloc[{1}:{2}]', that.state.pandasObject, start, end); - vpKernel.getRowList(subsetVariable).then(function (resultObj) { + vpKernel.getRowList(subsetVariable, start).then(function (resultObj) { let { result } = resultObj; var { list:rowList } = JSON.parse(result); rowList = rowList.map(function (x) { @@ -337,9 +364,9 @@ define([ rowList = rowList.map(function (x) { return { ...x, - label: x.location + '', - value: x.location + '', - code: x.location + '', + label: x.label + '', + value: x.value + '', + code: x.code + '', }; }); } @@ -500,8 +527,14 @@ define([ // col.array parsing var colInfo = com_util.safeString(col.array); // render column box - tag.appendFormatLine('

{8}

', - VP_DS_SELECT_ITEM, 'select-col', VP_DS_DRAGGABLE, col.location, col.value, col.dtype, col.code, col.label + ': \n' + colInfo, col.label); + let numIconStr = ''; + if (col.isNumeric === true) { + numIconStr = ''; + } else { + numIconStr = ''; + } + tag.appendFormatLine('

{8}{9}

', + VP_DS_SELECT_ITEM, 'select-col', VP_DS_DRAGGABLE, col.location, col.value, col.dtype, col.code, col.label + ': \n' + colInfo, numIconStr, col.label); }); tag.appendLine('

'); // VP_DS_SELECT_BOX $(this.wrapSelector('.select-col .' + VP_DS_SELECT_BOX + '.left')).replaceWith(function () { @@ -563,6 +596,7 @@ define([ renderColumnConditionList(colList) { var tag = new com_String(); tag.appendFormatLine('', VP_DS_CONDITION_TBL); + tag.appendLine(this.templateForConditionBox(colList)); tag.appendLine(''); tag.appendFormatLine('', VP_DS_BUTTON_ADD_CONDITION, 'vp-add-col', '+ Condition'); @@ -706,20 +740,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } that.renderDataPage(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); } /** @@ -743,6 +777,7 @@ define([ var varType = JSON.parse(result); that.state.pandasObject = prevValue; that.state.dataType = varType; + that.state.returnType = varType; $(that.wrapSelector('.' + VP_DS_PANDAS_OBJECT_BOX)).replaceWith(function () { return $(com_util.formatString('

', 'vp-input', VP_DS_PANDAS_OBJECT, prevValue)); @@ -783,6 +818,7 @@ define([ $(this.wrapSelector()).data('dtype', item.dtype); that.state.pandasObject = value; that.state.dataType = item.dtype; + that.state.returnType = item.dtype; $(this.wrapSelector()).trigger('change'); }); variableInput.setNormalFilter(true); @@ -1161,6 +1197,7 @@ define([ that.state.pandasObject = varName; that.state.dataType = event.dataType ? event.dataType : that.state.dataType; + that.state.returnType = that.state.dataType; that.state.rowList = []; that.state.rowLimit = 10; that.state.columnList = []; @@ -1185,6 +1222,7 @@ define([ ...x, value: x.label, code: x.value, + isNumeric: x.is_numeric }; }); that.loadColumnList(list); @@ -1707,7 +1745,7 @@ define([ var connector = i > 0 ? $(condList[i - 1]).find('.vp-oper-connect').val() : undefined; // if no variable selected, pass - if (varName == "") + if (varName === "" || oper === "") continue; if (useCondition) { rowSelection.append(connector); @@ -1775,6 +1813,10 @@ define([ rowSelection.append(')'); } else { rowSelection.appendFormat('({0}', varName); + if (colName == '.index') { + // index + rowSelection.append('.index'); + } oper && rowSelection.appendFormat(' {0}', oper); if (cond) { // condition value as text @@ -1813,10 +1855,14 @@ define([ if (this.state.colType == 'indexing') { if (this.useInputColumns == true) { colList = this.state.selectedColumns; - if (colList.length == 1) { - colSelection.appendFormat('{0}', colList.toString()); - } else { - colSelection.appendFormat('[{0}]', colList.toString()); + if (colList.length > 0) { + if (colList.length == 1) { + colSelection.appendFormat('{0}', colList.toString()); + this.state.returnType = 'Series'; + } else { + colSelection.appendFormat('[{0}]', colList.toString()); + this.state.returnType = 'DataFrame'; + } } } else { var colTags = $(this.wrapSelector('.' + VP_DS_SELECT_ITEM + '.select-col.added:not(.moving)')); @@ -1824,7 +1870,7 @@ define([ var colList = []; for (var i = 0; i < colTags.length; i++) { var colValue = $(colTags[i]).data('code'); - if (colValue) { + if (colValue !== undefined) { colList.push(colValue); } } @@ -1834,13 +1880,16 @@ define([ $(this.wrapSelector('.' + VP_DS_TO_FRAME)).parent().show(); // to frame - if (this.state.toFrame) { + if (this.state.toFrame === true) { colSelection.appendFormat('[{0}]', colList.toString()); + this.state.returnType = 'DataFrame'; } else { colSelection.appendFormat('{0}', colList.toString()); + this.state.returnType = 'Series'; } } else { colSelection.appendFormat('[{0}]', colList.toString()); + this.state.returnType = 'DataFrame'; } } else { @@ -1855,31 +1904,34 @@ define([ } // use simple selection - if (this.state.subsetType == 'subset') { - if (rowSelection.toString() != ':' && rowSelection.toString() != '') { - code.appendFormat('[{0}]', rowSelection.toString()); - } - if (colSelection.toString() != ':' && colSelection.toString() != '') { - code.appendFormat('[{0}]', colSelection.toString()); - } - } else if (this.state.subsetType == 'loc') { - if (this.state.dataType == 'DataFrame') { - code.appendFormat('.loc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); - } else { - code.appendFormat('.loc[{0}]', rowSelection.toString()); - } - } else if (this.state.subsetType == 'iloc') { - if (this.state.dataType == 'DataFrame') { - code.appendFormat('.iloc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); - } else { - code.appendFormat('.iloc[{0}]', rowSelection.toString()); - } - } else if (this.state.subsetType == 'query') { - if (rowSelection.toString() != ':' && rowSelection.toString() != '') { - code.appendFormat('.query("{0}")', rowSelection.toString()); - } - if (colSelection.toString() != ':' && colSelection.toString() != '') { - code.appendFormat('[{0}]', colSelection.toString()); + if ((rowSelection.toString() !== ':' && rowSelection.toString() !== '') + || (colSelection.toString() !== ':' && colSelection.toString() !== '')) { + if (this.state.subsetType == 'subset') { + if (rowSelection.toString() != ':' && rowSelection.toString() != '') { + code.appendFormat('[{0}]', rowSelection.toString()); + } + if (colSelection.toString() != ':' && colSelection.toString() != '') { + code.appendFormat('[{0}]', colSelection.toString()); + } + } else if (this.state.subsetType == 'loc') { + if (this.state.dataType == 'DataFrame') { + code.appendFormat('.loc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); + } else { + code.appendFormat('.loc[{0}]', rowSelection.toString()); + } + } else if (this.state.subsetType == 'iloc') { + if (this.state.dataType == 'DataFrame') { + code.appendFormat('.iloc[{0}, {1}]', rowSelection.toString(), colSelection.toString()); + } else { + code.appendFormat('.iloc[{0}]', rowSelection.toString()); + } + } else if (this.state.subsetType == 'query') { + if (rowSelection.toString() != ':' && rowSelection.toString() != '') { + code.appendFormat('.query("{0}")', rowSelection.toString()); + } + if (colSelection.toString() != ':' && colSelection.toString() != '') { + code.appendFormat('[{0}]', colSelection.toString()); + } } } @@ -1911,9 +1963,11 @@ define([ if (this.useInputVariable) { this.loadVariables(); this.reloadSubsetData(); + } + if (this.useCell === false) { // show save button only this.setSaveOnlyMode(); - } + } // generate code after displaying page // - codemirror can be set after display this.generateCode(); @@ -1924,13 +1978,13 @@ define([ //==================================================================== hideButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)).hide(); } } disableButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { var buttonEle = $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)); if (!buttonEle.hasClass('disabled')) { buttonEle.addClass('disabled'); @@ -1939,12 +1993,12 @@ define([ } enableButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)).removeClass('disabled'); } } showButton() { - if (this.useInputVariable) { + if (this.useInputVariable === true || this.useAsModule === true) { $(this.pageThis.wrapSelector('.' + VP_DS_BTN + '.' + this.uuid)).show(); } } diff --git a/visualpython/js/m_apps/Sweetviz.js b/visualpython/js/m_apps/Sweetviz.js index d0071449..2059189c 100644 --- a/visualpython/js/m_apps/Sweetviz.js +++ b/visualpython/js/m_apps/Sweetviz.js @@ -285,12 +285,14 @@ define([ this.checking = true; // check installed - vpKernel.execute('!pip show sweetviz').then(function(resultObj) { + let code = "_vp_print(_vp_check_package_list(['sweetviz']))"; + vpKernel.execute(code).then(function(resultObj) { let { result, msg } = resultObj; if (!that.checking) { return; } - if (msg.content['text'].includes('not found')) { + let installed = result['sweetviz'].installed; + if (installed === false) { that.toggleCheckState('install'); } else { that.toggleCheckState('installed'); diff --git a/visualpython/js/m_library/m_pandas/readFile.js b/visualpython/js/m_library/m_pandas/readFile.js index fafb85ba..c9144db9 100644 --- a/visualpython/js/m_library/m_pandas/readFile.js +++ b/visualpython/js/m_library/m_pandas/readFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -57,10 +58,11 @@ define([ }; this.fileState = { fileTypeId: { - 'csv': 'pd004', - 'excel': 'pd123', - 'json': 'pd076', - 'pickle': 'pd079' + 'csv': 'pd_readCsv', + 'excel': 'pd_readExcel', + 'json': 'pd_readJson', + 'pickle': 'pd_readPickle', + 'parquet': 'pd_readParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_library/m_pandas/toFile.js b/visualpython/js/m_library/m_pandas/toFile.js index a19dc303..08bb45c8 100644 --- a/visualpython/js/m_library/m_pandas/toFile.js +++ b/visualpython/js/m_library/m_pandas/toFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -57,10 +58,11 @@ define([ }; this.fileState = { fileTypeId: { - 'csv': 'pd005', - 'excel': 'pd124', - 'json': 'pd077', - 'pickle': 'pd078' + 'csv': 'pd_toCsv', + 'excel': 'pd_toExcel', + 'json': 'pd_toJson', + 'pickle': 'pd_toPickle', + 'parquet': 'pd_toParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_logic/Def.js b/visualpython/js/m_logic/Def.js index c5062925..0dd2e11e 100644 --- a/visualpython/js/m_logic/Def.js +++ b/visualpython/js/m_logic/Def.js @@ -77,6 +77,7 @@ define([ } loadState() { + super.loadState(); let { v1, v2 } = this.state; } diff --git a/visualpython/js/m_ml/AutoML.js b/visualpython/js/m_ml/AutoML.js index 18251639..73e30ce0 100644 --- a/visualpython/js/m_ml/AutoML.js +++ b/visualpython/js/m_ml/AutoML.js @@ -88,7 +88,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > AutoML'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > AutoML'); } }); diff --git a/visualpython/js/m_ml/Classification.js b/visualpython/js/m_ml/Classification.js index fb4940fe..f327b611 100644 --- a/visualpython/js/m_ml/Classification.js +++ b/visualpython/js/m_ml/Classification.js @@ -50,7 +50,7 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Classfication': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], + 'Classification': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], } @@ -88,7 +88,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > Classification'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > Classification'); } }); diff --git a/visualpython/js/m_ml/Clustering.js b/visualpython/js/m_ml/Clustering.js index df060086..95ee735c 100644 --- a/visualpython/js/m_ml/Clustering.js +++ b/visualpython/js/m_ml/Clustering.js @@ -51,7 +51,7 @@ define([ this.modelTypeList = { // 'Regression': ['ln-rgs', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], - // 'Classfication': ['lg-rgs', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], + // 'Classification': ['lg-rgs', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'], // 'Auto ML': ['tpot-rgs', 'tpot-clf'], 'Clustering': ['k-means', 'agg-cls', 'gaus-mix', 'dbscan'], // 'Dimension Reduction': ['pca', 'lda', 'svd', 'nmf'] @@ -91,7 +91,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > Clustering'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > Clustering'); } }); diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index 6683f559..10b9625f 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -57,13 +57,13 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], - 'ETC': ['make-column-transformer'] + 'ETC': ['prep-simple-imputer', 'prep-smote', 'make-column-transformer'] } this.mctEstimator = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], } @@ -94,10 +94,19 @@ define([ $(that.wrapSelector('#vp_installLibrary')).hide(); } - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { // load mct-targetData that.loadVariableList(); that.bindMCT(); + } else if (modelType === 'prep-simple-imputer') { + $(that.wrapSelector('#missing_values')).replaceWith(`

+ ${$(that.wrapSelector('#missing_values'))[0].outerHTML} + Text +

`); + $(that.wrapSelector('#fill_value')).replaceWith(`

+ ${$(that.wrapSelector('#fill_value'))[0].outerHTML} + Text +

`); } }); @@ -106,7 +115,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > DataPrep'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > DataPrep'); } }); @@ -407,6 +420,7 @@ define([ generateCode() { let { modelControlType, modelType, userOption, allocateToCreation, model } = this.state; + let state = JSON.parse(JSON.stringify(this.state)); let code = new com_String(); if (modelControlType == 'creation') { /** @@ -418,22 +432,33 @@ define([ let config = this.modelConfig[modelType]; code.appendLine(config.import); + if (modelType === 'prep-simple-imputer') { + let checkList = ['missing_values', 'fill_value']; + checkList.forEach(checkKey => { + try { + state[checkKey] = com_util.convertToStr(state[checkKey], state[checkKey + '_istext']); + } catch(e) { + ; + } + }); + } + // model code let modelCode = config.code; - modelCode = com_generator.vp_codeGenerator(this, config, this.state, (userOption != ''? ', ' + userOption : '')); + modelCode = com_generator.vp_codeGenerator(this, config, state, (userOption != ''? ', ' + userOption : '')); // generate mct code - if (modelType == 'make-column-transformer') { + if (modelType === 'make-column-transformer') { let mctCodes = []; - let { mct_estimator1, mct_columns1, mct_estimator2, mct_columns2 } = this.state; + let { mct_estimator1, mct_columns1='', mct_estimator2, mct_columns2='' } = state; if (mct_estimator1 != undefined && mct_estimator1 != '') { code.appendLine(this.modelConfig[mct_estimator1].import); - let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator1code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator1], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator1code, mct_columns1)); } if (mct_estimator2 != undefined && mct_estimator2 != '') { code.appendLine(this.modelConfig[mct_estimator2].import); - let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], this.state, (userOption != ''? ', ' + userOption : '')); + let estimator2code = com_generator.vp_codeGenerator(this, this.modelConfig[mct_estimator2], state, (userOption != ''? ', ' + userOption : '')); mctCodes.push(com_util.formatString('({0}, [{1}])', estimator2code, mct_columns2)); } modelCode = modelCode.replace('${mct_code}', mctCodes.join(', ')); diff --git a/visualpython/js/m_ml/DimensionReduction.js b/visualpython/js/m_ml/DimensionReduction.js index 2a4ec2fa..c35be792 100644 --- a/visualpython/js/m_ml/DimensionReduction.js +++ b/visualpython/js/m_ml/DimensionReduction.js @@ -87,7 +87,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > DimensionReduction'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > DimensionReduction'); } }); diff --git a/visualpython/js/m_ml/FitPredict.js b/visualpython/js/m_ml/FitPredict.js index 8cf91de5..d48c20c5 100644 --- a/visualpython/js/m_ml/FitPredict.js +++ b/visualpython/js/m_ml/FitPredict.js @@ -41,6 +41,8 @@ define([ method: '', action: {}, optionConfig: {}, + modelEditorType: '', + modelEditorName: '', userOption: '', ...this.state } @@ -315,7 +317,7 @@ define([ } generateCode() { - let { model } = this.state; + let { model, modelType, modelEditorName } = this.state; let code = new com_String(); let replaceDict = {'${model}': model}; @@ -336,6 +338,31 @@ define([ code.appendLine(); code.append(allocateCode); } + // Data Preparation > Scaling + const scalingTypeList = ['StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer']; + // Dimension Reduction + const dimensionTypeList = ['PCA', 'NMF']; + if (scalingTypeList.includes(modelType) || dimensionTypeList.includes(modelType)) { + // fit_transform, transform returns df_trans also + switch (modelEditorName) { + case 'fit_transform': + const allocatedFitTrans = this.state.fit_trans_allocate || 'trans'; + code.appendLine(); + code.appendLine(); + code.appendFormatLine("df_{0} = pd.DataFrame({1}, columns=[{2}])", allocatedFitTrans, allocatedFitTrans, this.state.fit_trans_featureData); + code.append("df_" + allocatedFitTrans); + break; + case 'transform': + const allocatedTrans = this.state.trans_allocate || 'trans'; + code.appendLine(); + code.appendLine(); + code.appendFormatLine("df_{0} = pd.DataFrame({1}, columns=[{2}])", allocatedTrans, allocatedTrans, this.state.trans_featureData); + code.append("df_" + allocatedTrans); + break; + default: + break; + } + } } return code.toString(); @@ -409,7 +436,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit Encoder/Scaler to X.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -418,16 +445,21 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit Encoder/Scaler to X, then transform X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, 'transform': { - ...defaultActions['transform'], - description: 'Transform labels to normalized encoding.' + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -437,12 +469,75 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform binary labels back to multi-class labels.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] } } } + if (modelType == 'LabelEncoder') { + actions = { + ...actions, + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData})', + description: 'Fit Encoder/Scaler to X.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' } + ] + }, + 'fit_transform': { + name: 'fit_transform', + label: 'Fit and transform', + code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', + description: 'Fit Encoder/Scaler to X, then transform X.', + options: [ + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Transform labels to normalized encoding.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', + columnSelection: 'single', returnFrameType: 'Series' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -581,7 +676,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -590,7 +685,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit X into an embedded space and return that transformed output.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -605,7 +700,7 @@ define([ code: '${model}.fit(${fit_featureData}, ${fit_targetData})', description: 'Fit the Linear Discriminant Analysis model.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' } ] }, @@ -615,7 +710,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData}${fit_trans_targetData})', description: 'Fit to data, then transform it.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] @@ -626,7 +721,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Predict class labels for samples in X.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -636,7 +731,7 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Project data to maximize class separation.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } @@ -650,7 +745,7 @@ define([ code: '${model}.fit(${fit_featureData})', description: 'Fit X into an embedded space.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' } ] }, 'fit_transform': { @@ -659,7 +754,7 @@ define([ code: '${fit_trans_allocate} = ${model}.fit_transform(${fit_trans_featureData})', description: 'Fit the model with X and apply the dimensionality reduction on X.', options: [ - { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'fit_trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'fit_trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] }, @@ -669,7 +764,7 @@ define([ code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', description: 'Transform data back to its original space.', options: [ - { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } ] }, @@ -679,12 +774,58 @@ define([ code: '${trans_allocate} = ${model}.transform(${trans_featureData})', description: 'Apply dimensionality reduction to X.', options: [ - { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X', returnFrameType: 'DataFrame' }, { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } ] } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}${fit_targetData})', + description: 'Run fit with all sets of parameters.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } + ] + }, + 'predict': { + name: 'predict', + label: 'Predict', + code: '${pred_allocate} = ${model}.predict(${pred_featureData})', + description: 'Call predict on the estimator with the best found parameters.', + options: [ + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, + { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } + ] + }, + 'inverse_transform': { + name: 'inverse_transform', + label: 'Inverse transform', + code: '${inverse_allocate} = ${model}.inverse_transform(${inverse_featureData})', + description: 'Call inverse_transform on the estimator with the best found params.', + options: [ + { name: 'inverse_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'inverse_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'inv_trans' } + ] + }, + 'transform': { + name: 'transform', + label: 'Transform', + code: '${trans_allocate} = ${model}.transform(${trans_featureData})', + description: 'Call transform on the estimator with the best found parameters.', + options: [ + { name: 'trans_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'trans_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'trans' } + ] + } + } + } + break; } return actions; } diff --git a/visualpython/js/m_ml/GridSearch.js b/visualpython/js/m_ml/GridSearch.js new file mode 100644 index 00000000..1dfdbaa4 --- /dev/null +++ b/visualpython/js/m_ml/GridSearch.js @@ -0,0 +1,564 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : GridSearch.js + * Author : Black Logic + * Note : GridSearch + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] GridSearch +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/gridSearch.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/gridSearch'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, com_util, com_interface, com_String, com_generator, ML_LIBRARIES, PopupComponent, MultiSelector, SuggestInput, ModelEditor) { + + /** + * GridSearch + */ + class GridSearch extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 3; + this.config.dataview = false; + this.config.docs = 'https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html'; + + this.state = { + modelType: 'ln-rgs', + userOption: '', + allocateToCreation: 'model', + ...this.state + } + + this.targetSetTag = null; + + this.modelConfig = ML_LIBRARIES; + + this.modelTypeList = { + 'Regression': ['ln-rgs', 'ridge', 'lasso', 'elasticnet', 'sv-rgs', 'dt-rgs', 'rf-rgs', 'gbm-rgs', 'xgb-rgs', 'lgbm-rgs', 'cb-rgs'], + 'Classification': ['lg-rgs', 'bern-nb', 'mulnom-nb', 'gaus-nb', 'sv-clf', 'dt-clf', 'rf-clf', 'gbm-clf', 'xgb-clf', 'lgbm-clf', 'cb-clf'] + } + + + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('click', this.wrapSelector('.vp-param-set-del')); + $(document).off('click', this.wrapSelector('.vp-param-item-add')); + $(document).off('click', this.wrapSelector('.vp-param-item-del')); + $(document).off('keyup', this.wrapSelector('.vp-param-val')); + $(document).off('click', this.wrapSelector('.vp-param-result-item-del')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select model type + $(this.wrapSelector('#modelType')).on('change', function() { + let modelType = $(this).val(); + that.state.modelType = modelType; + + // show install button + if (that.modelConfig[modelType].install != undefined) { + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + + that.handleScoringOptions(modelType); + + // reset model param set + $(that.wrapSelector('.vp-param-grid-box')).html(''); + $(that.wrapSelector('.vp-param-grid-box')).html(that.templateForParamSet()); + }); + + // Add param set + $(this.wrapSelector('#vp_addParamSet')).on('click', function() { + let newSet = $(that.templateForParamSet()); + $(that.wrapSelector('.vp-param-grid-box')).append(newSet); + // focus + $(newSet)[0].scrollIntoView(); + }); + + // delete param set + $(document).on('click', this.wrapSelector('.vp-param-set-del'), function() { + $(this).closest('.vp-param-set-box').remove(); + + // rename param set + $(that.wrapSelector('.vp-param-set-name')).each((i, tag) => { + $(tag).text('Param set ' + (i + 1)); + }); + }); + + // Add param item + $(document).on('click', this.wrapSelector('.vp-param-item-add'), function() { + that.targetSetTag = $(this).parent(); // target param-set-box + that.openParamPopup(); + }); + + // Delete param item + $(document).on('click', this.wrapSelector('.vp-param-item-del'), function() { + $(this).closest('.vp-param-item').remove(); + }); + + // add param value - using enter + $(document).on('keyup', this.wrapSelector('.vp-param-val'), function(event) { + var keycode = event.keyCode + ? event.keyCode + : event.which; + if (keycode == vpEvent.keyManager.keyCode.enter) { // enter + that.handleAddParamValue($(this)); + } + if (keycode === 188) {// ,< + let val = $(this).val(); + $(this).val(val.split(',')[0]); // remove , and add param + that.handleAddParamValue($(this)); + } + }); + + // delete param set item + $(document).on('click', this.wrapSelector('.vp-param-result-item-del'), function() { + $(this).closest('.vp-param-result-item').remove(); + }); + } + + handleAddParamValue(thisTag) { + let parentTag = $(thisTag).parent(); + let paramIsText = $(parentTag).find('.vp-param-val').data('type') === 'text'; // text / var + let paramVal = $(parentTag).find('.vp-param-val').val(); + let reservedKeywordList = ['None', 'True', 'False', 'np.nan', 'np.NaN']; + if (reservedKeywordList.includes(paramVal)) { + paramIsText = false; + } + // check , and split it + let paramSplit = paramVal.split(','); + paramSplit && paramSplit.forEach(val => { + // add only if it is not empty value + if (val.trim() !== '') { + val = com_util.convertToStr(val.trim(), paramIsText); + $(parentTag).find('.vp-param-result-box').append(` + ${val} + + + `); + // clear param val + $(parentTag).find('.vp-param-val').val(''); + } + }); + } + + bindEventForInnerPopup() { + let that = this; + // Inner popup: Select param + $(document).off('click', this.wrapSelector('.vp-inner-param-list-item')); + $(document).on('click', this.wrapSelector('.vp-inner-param-list-item'), function() { + if ($(this).hasClass('selected')) { + $(this).removeClass('selected'); + } else { + $(this).addClass('selected'); + } + }); + + // Inner popup: Add param + $(this.wrapSelector('.vp-add-param-btn')).on('click', function() { + let newParam = $(that.wrapSelector('.vp-add-param-name')).val(); + if (newParam != undefined && newParam.trim() != '') { + // check if exist + let checkTag = $(that.wrapSelector(`.vp-inner-param-list-item[data-name="${newParam}"]`)); + if (checkTag.length > 0) { + if (!checkTag.hasClass('selected')) { + checkTag.addClass('selected'); + } + checkTag[0].scrollIntoView(); + } else { + // add to param list + $(that.wrapSelector('.vp-inner-param-list-box')).append( + `

${newParam}

` + ); + // scroll to added item + $(that.wrapSelector('.vp-inner-param-list-item.selected:last'))[0].scrollIntoView(); + } + + // clear input value + $(that.wrapSelector('.vp-add-param-name')).val(''); + } + }); + } + + handleScoringOptions(modelType) { + let options = { + 'Classification': [ + "'accuracy'", + "'balanced_accuracy'", + "'top_k_accuracy'", + "'average_precision'", + "'neg_brier_score'", + "'f1'", + "'f1_micro'", + "'f1_macro'", + "'f1_weighted'", + "'f1_samples'", + "'neg_log_loss'", + "'precision' etc.", + "'recall' etc.", + "'jaccard' etc.", + "'roc_auc'", + "'roc_auc_ovr'", + "'roc_auc_ovo'", + "'roc_auc_ovr_weighted'", + "'roc_auc_ovo_weighted'", + ], + 'Regression': [ + "'explained_variance'", + "'max_error'", + "'neg_mean_absolute_error'", + "'neg_mean_squared_error'", + "'neg_root_mean_squared_error'", + "'neg_mean_squared_log_error'", + "'neg_median_absolute_error'", + "'r2'", + "'neg_mean_poisson_deviance'", + "'neg_mean_gamma_deviance'", + "'neg_mean_absolute_percentage_error'", + "'d2_absolute_error_score'", + "'d2_pinball_score'", + "'d2_tweedie_score'" + ] + } + let modelCategory = this.modelTypeList['Regression'].includes(modelType)?'Regression':'Classification'; + + // Set suggestInput on scoring option + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('scoring'); + suggestInput.setPlaceholder('Select option'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return options[modelCategory]; }); + suggestInput.setNormalFilter(true); + $(this.wrapSelector('#scoring')).replaceWith(suggestInput.toTagString()); + } + + templateForParamSet() { + let paramSetNo = 1; + // set param set number + paramSetNo += $(this.wrapSelector('.vp-param-set-box')).length; + return `

+ Param set ${paramSetNo} +

+ +

`; + } + + templateForParamPopup() { + let config = this.modelConfig[this.state.modelType]; + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('

{2}

' + , opt.name, opt.name, opt.name); + }); + + return ` +

NOTE: Select parameters to add.

+ ${optBox.toString()} +

+ + +

+ `; + } + + openParamPopup() { + let size = { width: 400, height: 300 }; + + $(this.wrapSelector('.vp-inner-popup-body')).empty(); + + // set size and position + $(this.wrapSelector('.vp-inner-popup-box')).css({ + width: size.width, + height: size.height, + left: 'calc(50% - ' + (size.width/2) + 'px)', + top: 'calc(50% - ' + (size.height/2) + 'px)', + }); + + $(this.wrapSelector('.vp-inner-popup-body')).html(this.templateForParamPopup()); + + this.bindEventForInnerPopup(); + + // show popup box + this.openInnerPopup('Add parameter'); + } + + handleInnerOk() { + // get selected param list + let paramList = []; + $(this.wrapSelector('.vp-inner-param-list-item.selected')).each((i, tag) => { + paramList.push($(tag).data('name')); + }); + + if (paramList.length > 0) { + // add param box + $(this.targetSetTag).find('.vp-param-item-box').append(this.templateForParamItemList(paramList)); + + this.closeInnerPopup(); + } else { + com_util.renderAlertModal('No params selected. Please select more than one param.'); + } + + this.targetSetTag = null; + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Model creation + //================================================================ + // model types + let modelTypeTag = new com_String(); + Object.keys(this.modelTypeList).forEach(modelCategory => { + let modelOptionTag = new com_String(); + that.modelTypeList[modelCategory].forEach(opt => { + let optConfig = that.modelConfig[opt]; + let selectedFlag = ''; + if (opt == that.state.modelType) { + selectedFlag = 'selected'; + } + modelOptionTag.appendFormatLine('', + opt, selectedFlag, optConfig.name); + }) + modelTypeTag.appendFormatLine('{1}', + modelCategory, modelOptionTag.toString()); + }); + $(page).find('#modelType').html(modelTypeTag.toString()); + + //================================================================ + // GridSearch option + //================================================================ + $(page).find('.vp-model-option-box').html(this.templateForOption('grid-search', ['estimator', 'param_grid'])); + + // show install button + if (this.modelConfig[this.state.modelType].install != undefined) { + this.showInstallButton(); + } else { + this.hideInstallButton(); + } + + // render option page + // $(page).find('.vp-model-param-box').html(this.templateForOption(this.state.modelType)); + + return page; + } + + templateForOption(modelType, excludeOptList=[]) { + let config = this.modelConfig[modelType]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + if (!excludeOptList.includes(opt.name)) { + optBox.appendFormatLine('{2}' + , opt.name, opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + } + }); + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + return optBox.toString(); + } + + /** + * template for param box + * @param {*} paramList ['param-name1', 'param-name2', ...] + */ + templateForParamItemList(paramList) { + let that = this; + let config = this.modelConfig[this.state.modelType]; + let paramSet = new com_String(); + paramList && paramList.forEach(name => { + // get option component + let componentType = 'input'; // default + let paramObj = config.options.filter(x => x.name === name).at(0); + if ((paramObj !== undefined) && (['option_select', 'bool_select'].includes(paramObj.component[0]))) { + componentType = 'option'; + } + paramSet.appendLine('

'); + paramSet.appendFormatLine('{1}', name, com_util.optionToLabel(name)); + paramSet.appendLine('

'); + paramSet.appendFormatLine('

', name); + paramSet.appendFormatLine('', name); + if (componentType === 'input') { + paramSet.appendFormatLine('', name); + + } else { + // paramSet.appendFormatLine('', name); + var suggestInput = new SuggestInput(); + suggestInput.setPlaceholder('Type here'); + suggestInput.addClass('vp-param-val'); + suggestInput.addAttribute('data-name', name); + suggestInput.addAttribute('data-type', (paramObj.type === 'text' ? 'text' : 'var')); + let optionList = paramObj.options; + if (paramObj.component[0] === 'bool_select') { + optionList = ['True', 'False']; + } + suggestInput.setSuggestList(function() { return optionList; }); + suggestInput.setNormalFilter(false); + suggestInput.setSelectEvent(function(selectedValue) { + // trigger change + let thisTag = $(that.wrapSelector('.' + suggestInput.uuid)); + that.handleAddParamValue($(thisTag)); + $(thisTag).val(''); + return false; + }); + paramSet.appendLine(suggestInput.toTagString()); + } + paramSet.appendLine('

'); + paramSet.appendLine('

'); // vp-param-item + }); + return paramSet.toString(); + } + + render() { + super.render(); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + this.handleScoringOptions(this.state.modelType); + } + + generateInstallCode() { + let installCode = this.modelConfig[this.state.modelType].install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + return [ installCode ]; + } + + checkBeforeRun() { + // if no param is registered, stop and show alert + if ($(this.wrapSelector('.vp-param-result-item')).length <= 0) { + com_util.renderAlertModal('No params added. Please add params.'); + return false; + } + + return true; + } + + generateCode() { + let { modelType, userOption, allocateToCreation, model } = this.state; + let code = new com_String(); + /** + * Model Creation + * --- + * from module import model_function + * model = Model(key=value, ...) + */ + let gsConfig = this.modelConfig['grid-search']; + + + let state = JSON.parse(JSON.stringify(this.state)); + let estConfig = this.modelConfig[this.state.modelType]; + let estimator = com_generator.vp_codeGenerator(null, estConfig, {}, ''); + state['estimator'] = estimator; + state['param_grid'] = '{}'; + + let paramGrid = []; + // generate param_grid + $(this.wrapSelector('.vp-param-set-box')).each((i, tag) => { + let paramSet = {}; + $(tag).find('.vp-param-result-box').each((j, resTag) => { + let paramName = $(resTag).data('name'); + let paramValList = []; + $(resTag).find('.vp-param-result-item').each((k, itemTag) => { + let val = $(itemTag).data('value'); + if (!isNaN(val)) { + // numeric string -> numeric + val = parseFloat(val); + } + paramValList.push(val); + }); + if (paramValList.length > 0) { + // Add only its result exists + paramSet[paramName] = paramValList; + } + }); + if (Object.keys(paramSet).length > 0) { + paramGrid.push(paramSet); + } + }); + + state['param_grid'] = this.paramStringify(paramGrid); + + // import code + code.appendLine(gsConfig.import); + code.appendLine(estConfig.import); + + // model code + let gsCode = gsConfig.code; + gsCode = com_generator.vp_codeGenerator(this, gsConfig, state, (userOption != ''? ', ' + userOption : '')); + code.appendLine(); + code.appendFormat('{0} = {1}', allocateToCreation, gsCode); + + return code.toString(); + } + + paramStringify(paramGridList=[]) { + let paramGridCode = new com_String(); + if (paramGridList.length > 1) { + paramGridCode.append('['); + } + paramGridList.forEach((paramSet, i) => { + if (i > 0) { + paramGridCode.append(', \n '); + } + paramGridCode.append('{'); + Object.keys(paramSet).forEach((paramName, j) => { + if (j > 0) { + paramGridCode.append(', '); + } + paramGridCode.appendFormat("'{0}': [{1}]", paramName, paramSet[paramName].toString()); + }); + paramGridCode.append('}'); + }) + + if (paramGridList.length > 1) { + paramGridCode.append(']'); + } + return paramGridCode.toString(); + } + + } + + return GridSearch; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 00cdcf4a..23bf679b 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -260,6 +260,10 @@ define([ that.config.checkModules = ['pd', 'vp_create_feature_importances']; } else if (name == 'plot_feature_importances') { that.config.checkModules = ['pd', 'plt', 'vp_create_feature_importances', 'vp_plot_feature_importances']; + } else if (name == 'permutation_importance') { + that.config.checkModules = ['pd', 'vp_create_permutation_importances']; + } else if (name == 'plot_permutation_importance') { + that.config.checkModules = ['pd', 'vp_create_permutation_importances', 'vp_plot_permutation_importances']; } else { that.config.checkModules = ['pd']; } @@ -409,16 +413,46 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, - { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error', + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', @@ -514,6 +548,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] @@ -545,7 +592,44 @@ define([ { name: 'cvs_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'scores' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'], + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'explained_variance', 'max_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_root_mean_squared_error', + 'neg_mean_squared_log_error', 'neg_median_absolute_error', 'r2', 'neg_mean_poisson_deviance', 'neg_mean_gamma_deviance', + 'neg_mean_absolute_percentage_error' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'Coefficient': { name: 'coef_', label: 'Coefficient', @@ -631,7 +715,44 @@ define([ { name: 'auc_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_test' } ] }, - 'permutation_importance': defaultInfos['permutation_importance'] + 'permutation_importance': { + name: 'permutation_importance', + label: 'Permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } + ] + }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['option_suggest'], usePair: true, type: 'text', + options: [ + 'accuracy', 'balanced_accuracy', 'top_k_accuracy', 'average_precision', 'neg_brier_score', + 'f1', 'f1_micro', 'f1_macro', 'f1_weighted', 'f1_samples', 'neg_log_loss', 'precision', 'recall', 'jaccard', + 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', 'roc_auc_ovr_weighted', 'roc_auc_ovo_weighted' + ] }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, } // feature importances @@ -779,6 +900,39 @@ define([ } } break; + case 'ETC': + if (modelType === 'GridSearchCV') { + infos = { + 'best_estimator_': { + name: 'best_estimator_', + label: 'Best estimator', + code: '${best_estimator_allocate} = ${model}.best_estimator_', + description: 'Estimator that was chosen by the search.', + options: [ + { name: 'best_estimator_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_estimator' } + ] + }, + 'best_score_': { + name: 'best_score_', + label: 'Best score', + code: '${best_score_allocate} = ${model}.best_score_', + description: 'Mean cross-validated score of the best_estimator.', + options: [ + { name: 'best_score_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_score' } + ] + }, + 'best_params_': { + name: 'best_params_', + label: 'Best params', + code: '${best_params_allocate} = ${model}.best_params_', + description: 'Parameter setting that gave the best results on the hold out data.', + options: [ + { name: 'best_params_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'best_params' } + ] + } + } + } + break; } return infos; } diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js new file mode 100644 index 00000000..856c33db --- /dev/null +++ b/visualpython/js/m_ml/Pipeline.js @@ -0,0 +1,798 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Pipeline.js + * Author : Black Logic + * Note : Pipeline + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 08. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] Pipeline +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_ml/pipeline.html'), // INTEGRATION: unified version of text loader + __VP_CSS_LOADER__('vp_base/css/m_ml/pipeline'), + __VP_RAW_LOADER__('vp_base/data/libraries.json'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_ml/mlLibrary', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/com/component/ModelEditor' +], function(msHtml, msCss, librariesJson, com_util, com_String, com_generator, ML_LIBRARIES, PopupComponent, SuggestInput, ModelEditor) { + + /** + * Pipeline + */ + class Pipeline extends PopupComponent { + _init() { + super._init(); + this.config.sizeLevel = 4; + this.config.dataview = false; + this.config.autoScroll = false; + + this.state = { + templateType: '', + modelStep: -1, // get modelStep sequence + modelType: '', // get modelType + modelTypeName: '', // get modelTypeName + model: 'model', + // now selected pipeline info + pipeline: [ + // copy of step + info + // { name, label, useApp, state, app } + ], + ...this.state + } + + /** + * useApp + - Data Split + - Data Prep + - Regressor + - Classifier + - Clustering + - Dimension + - GridSearch + - Evaluation + + - Fit + - Transform + - Predict + - Fit and Predict + - Fit and Transform + - Fit and Resample + */ + this.templateList = { + 'data-prep': { + label: 'Data Preparation', + modelStep: 0, + step: [ + /** + * ml_* is pre-defined app + * pp_* is defined only for Pipeline + */ + { name: 'ml_dataPrep', label: 'Data Prep', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_resample'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_resample', label: 'Fit and Resample' } + ] + }, + 'regression': { + label: 'Regression', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_regression', label: 'Regressor', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'rgs' } }, + ] + }, + 'classification': { + label: 'Classification', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_classification', label: 'Classifier', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'clf' } }, + ] + }, + 'clustering': { + label: 'Clustering', + modelStep: 0, + step: [ + { name: 'ml_clustering', label: 'Clustering', useApp: true, child: ['pp_fit', 'pp_predict', 'pp_fit_predict', 'pp_transform'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'pp_fit_predict', label: 'Fit and Predict' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true, state: { modelType: 'cls' } }, + ] + }, + 'dimension': { + label: 'Dimension Reduction', + modelStep: 0, + step: [ + { name: 'ml_dimensionReduction', label: 'Dimension Reduction', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_transform'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_transform', label: 'Fit and Transform' }, + ] + }, + 'gridSearch': { + label: 'GridSearch', + modelStep: 1, + step: [ + { name: 'ml_dataSplit', label: 'Data Split', useApp: true }, + { name: 'ml_gridSearch', label: 'GridSearch', useApp: true, child: ['pp_fit', 'pp_predict'] }, + { name: 'pp_fit', label: 'Fit' }, + { name: 'pp_predict', label: 'Predict' }, + { name: 'ml_evaluation', label: 'Evaluation', useApp: true }, + ] + } + } + + // menu libraries for ml + let libObj = {}; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + libObj = librariesJson; + + this.MlAppComponent = {}; + this.MlAppComponent['ml_dataSplit'] = require('./dataSplit'); + this.MlAppComponent['ml_dataPrep'] = require('./DataPrep'); + this.MlAppComponent['ml_regression'] = require('./Regression'); + this.MlAppComponent['ml_classification'] = require('./Classification'); + this.MlAppComponent['ml_clustering'] = require('./Clustering'); + this.MlAppComponent['ml_dimensionReduction'] = require('./DimensionReduction'); + this.MlAppComponent['ml_gridSearch'] = require('./GridSearch'); + this.MlAppComponent['ml_evaluation'] = require('./evaluation'); + } else { + libObj = JSON.parse(librariesJson); + } + this.mlAppList = libObj.library.item.filter(x => x.id === 'pkg_ml')[0].item; + + this.modelConfig = ML_LIBRARIES; + } + + _unbindEvent() { + super._unbindEvent(); + + $(document).off('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]')); + $(document).off('click', this.wrapSelector('.vp-pp-item-toggle')); + $(document).off('change', this.wrapSelector(`#modelType`)); + $(document).off('change', this.wrapSelector('#allocateToCreation')); + + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // select template + $(this.wrapSelector('#templateType')).on('change', function() { + let type = $(this).val(); + that.state.templateType = type; + that.state.model = 'model'; + that.state.modelType = ''; + that.state.modelTypeName = ''; + that.state.modelStep = -1; + + // reset pp-page + $(that.wrapSelector('.vp-pp-template')).html(''); + $(that.wrapSelector('.vp-pp-step-content')).html(''); + $(that.wrapSelector('.vp-pp-step-title')).text(''); + that.state.pipeline = []; + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + + that.handleChangeTemplate(type); + }); + + // select pipeline item + $(document).on('click', this.wrapSelector('.vp-pp-item[data-flag="enabled"]'), function() { + if (!$(this).hasClass('selected')) { + let title = $(this).data('label'); + let stepSeq = parseInt($(this).data('step')); // 0 ~ n + let name = $(this).data('name'); + let ppObj = that.state.pipeline[stepSeq]; + // set title + $(that.wrapSelector('.vp-pp-step-title')).text(title); + + // show page + $(that.wrapSelector(`.vp-pp-step-page:not([data-name="${name}"])`)).hide(); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).show(); + if (ppObj.useApp === true) { + ppObj.app && that.handleAppView(name, ppObj.app); + ppObj.app && ppObj.app.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`))); + } else { + that.renderApp(name); + } + + // check selected + $(that.wrapSelector('.vp-pp-item')).removeClass('selected'); + $(this).addClass('selected'); + + // check prev/next + let prevTagList = $(this).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(this).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } + } + }); + + // pipeline item toggle (enable/disable) + $(document).on('click', this.wrapSelector('.vp-pp-item-toggle'), function(evt) { + evt.stopPropagation(); + let itemTag = $(this).closest('.vp-pp-item'); + let name = $(itemTag).attr('data-name'); + let flag = $(itemTag).attr('data-flag'); // enabled / disabled + if (flag === 'enabled') { + $(itemTag).attr('data-flag', 'disabled'); + $(this).prop('checked', false); + } else if (flag === 'disabled') { + $(itemTag).attr('data-flag', 'enabled'); + $(this).prop('checked', true); + } + + if ($(itemTag).hasClass('selected')) { + // check if this page is this item's, then hide it. + $(that.wrapSelector('.vp-pp-step-title')).text(''); + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + $(itemTag).removeClass('selected'); + + // disable prev/next + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + // check prev/next + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + if (prevTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-prev')).removeClass('disabled'); + } + if (nextTagList.length == 0) { + $(that.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } else { + $(that.wrapSelector('.vp-pp-step-next')).removeClass('disabled'); + } + } + + }); + + // model type change event + $(document).on('change', this.wrapSelector(`#modelType`), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelCreatorList = ['ml_dataPrep', 'ml_regression', 'ml_classification', 'ml_clustering', 'ml_dimensionReduction', 'ml_gridSearch']; + if (modelCreatorList.includes(name)) { + let modelType = $(this).val(); + let modelObj = that.modelConfig[modelType]; + let modelTypeName = modelObj.returnType; + + that.state.modelType = modelType; + that.state.modelTypeName = modelTypeName; + + // show fit / predict / transform depends on model selection + let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform', 'fit_resample']; + let actions = that.modelEditor.getAction(modelTypeName); + defaultActions.forEach(actKey => { + if (actions[actKey] === undefined) { + // if undefined, hide step + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).show(); + if (actKey === 'fit_predict') { + if (actions['fit'] === undefined || actions['predict'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_predict"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_transform') { + if (actions['fit'] === undefined || actions['transform'] === undefined) { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } else { + $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); + } + } else if (actKey === 'fit_resample') { + // for SMOTE: show fit_resample only + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); + } + } + $(that.wrapSelector('.vp-pp-item')).removeClass('vp-last-visible'); + $(that.wrapSelector('.vp-pp-item:visible:last')).addClass('vp-last-visible'); + }); + + // change evaluation page's default selection + if (name === 'ml_gridSearch') { + let modelCategory = that.modelEditor.getModelCategory(modelTypeName); // Regression / Classification / Clustering + let evalModelTypeValue = 'rgs'; + switch (modelCategory) { + case 'Regression': + evalModelTypeValue = 'rgs'; + break; + case 'Classification': + evalModelTypeValue = 'clf'; + break; + case 'Clustering': + evalModelTypeValue = 'cls'; + break; + } + let evalPageTag = $(that.wrapSelector('.vp-pp-step-page[data-name="ml_evaluation"]')); + if (evalPageTag.is(':empty') === true) { + // evaluation page is not opened yet + // set state for evaluation page + let stepSeq = $(evalPageTag).data('step'); + let ppObj = that.state.pipeline[stepSeq]; + if (ppObj.app) { + ppObj.app.state.modelType = evalModelTypeValue; + } + } else { + let evalModelTypeTag = $(evalPageTag).find('#modelType'); + $(evalModelTypeTag).val(evalModelTypeValue); + $(evalModelTypeTag).trigger('change'); + } + } + } + }); + + // model allocation variable change + $(document).on('change', this.wrapSelector('#allocateToCreation'), function() { + let name = $(this).closest('.vp-pp-step-page').data('name'); + let modelAllocation = $(this).val(); + that.state.model = modelAllocation; + }); + + // click prev button + $(this.wrapSelector('.vp-pp-step-prev:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let prevTagList = $(selectedTag).prevAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(prevTagList[0]).trigger('click'); + }); + + // click next button + $(this.wrapSelector('.vp-pp-step-next:not(.disabled)')).on('click', function() { + let selectedTag = $(that.wrapSelector('.vp-pp-item.selected')); + let nextTagList = $(selectedTag).nextAll('.vp-pp-item[data-flag="enabled"]:visible'); + $(nextTagList[0]).trigger('click'); + }); + } + + /** + * + * @param {*} type template type + */ + handleChangeTemplate(type) { + let that = this; + if (type !== '') { + let tplObj = this.templateList[type]; + this.state.modelStep = tplObj.modelStep; + + let ppTag = new com_String(); + let appFileList = []; + // load pipeline items + tplObj.step.forEach((stepObj, idx) => { + let { name, label, useApp=false, child=[], state={} } = stepObj; + ppTag.appendFormatLine(`

+ {3} + +

`, name, idx, label, label); + + // get pages + if (useApp === true) { + let mlObj = that.mlAppList.filter(x => x.id === name)[0]; + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } else { + appFileList.push({ index: idx, name: name, file: 'vp_base/js/' + mlObj.file}); + } + } + let pipeObj = { + name: name, + label: label, + useApp: useApp, + state: state + }; + if (tplObj.modelStep === idx) { + pipeObj.modelStep = true; + pipeObj.child = child; + } + that.state.pipeline.push(pipeObj); + // append pages + $(that.wrapSelector('.vp-pp-step-content')).append( + `

`); + if (useApp === false) { + that.renderApp(name); + // hide + $(that.wrapSelector(`.vp-pp-step-page[data-name="${name}"]`)).hide(); + } + + }); + $(that.wrapSelector('.vp-pp-template')).html(ppTag.toString()); + + // render pages + // for lite and lab + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { + appFileList.forEach((obj, argIdx) => { + let MlComponent = that.MlAppComponent[obj.name]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: that.state.pipeline[index].label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + mlComponent.loadState(); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + + that.state.pipeline[index].child.forEach(childId => { + that.renderApp(childId); + }); + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); + // end of DUP AREA: pp-1 + } + }); + + } else { + // for notebook and others + window.require(appFileList.map(x => x.file), function() { + appFileList.forEach((obj, argIdx) => { + let MlComponent = arguments[argIdx]; + if (MlComponent) { + // DUP AREA: pp-1 + let { name, index, file } = obj; + let mlComponent = new MlComponent({ + config: { id: name, name: that.state.pipeline[index].label, path: file, category: 'Pipeline', resizable: false }, + ...that.state.pipeline[index].state + }); + mlComponent.loadState(); + // mlComponent.open($(that.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`))); + that.state.pipeline[index].app = mlComponent; + + if (that.state.pipeline[index].modelStep === true) { + // set default model type + that.state.model = mlComponent.state.allocateToCreation; + that.state.modelType = mlComponent.state.modelType; + let modelObj = that.modelConfig[that.state.modelType]; + that.state.modelTypeName = modelObj.code.split('(')[0]; + + that.state.pipeline[index].child.forEach(childId => { + that.renderApp(childId); + }); + } + // handle app view + that.handleAppView(name, mlComponent); + + // select first step + $(that.wrapSelector('.vp-pp-item[data-step="0"]')).click(); + $(that.wrapSelector('#modelType')).trigger('change'); + // end of DUP AREA: pp-1 + } + }) + + }); + } + } + } + + templateForBody() { + let page = $(msHtml); + + let that = this; + + //================================================================ + // Template list creation + //================================================================ + let tplTag = new com_String(); + tplTag.appendLine(''); + Object.keys(this.templateList).forEach(tplKey => { + let tplObj = that.templateList[tplKey]; + let selectedFlag = ''; + if (tplKey == that.state.templateType) { + selectedFlag = 'selected'; + } + tplTag.appendFormatLine('', + tplKey, selectedFlag, tplObj.label); + }); + $(page).find('#templateType').html(tplTag.toString()); + + return page; + } + + render() { + super.render(); + + // resize codeview + $(this.wrapSelector('.vp-popup-codeview-box')).css({'height': '300px'}); + + // Model Editor + this.modelEditor = new ModelEditor(this, "model", "instanceEditor"); + + // load state + if (this.state.templateType !== '') { + $(this.wrapSelector('#templateType')).trigger('change'); + } else { + $(this.wrapSelector('.vp-pp-step-prev')).addClass('disabled'); + $(this.wrapSelector('.vp-pp-step-next')).addClass('disabled'); + } + } + + /** + * Handle app view before open + * @param {string} appId + * @param {*} mlApp + */ + handleAppView(appId, mlApp) { + switch (appId) { + case 'ml_dataSplit': + $(mlApp.wrapSelector('#inputData')).parent().hide(); + break; + case 'ml_evaluation': + // for pipeline + $(mlApp.wrapSelector('.vp-upper-box')).hide(); + $(mlApp.wrapSelector('.vp-upper-box.' + mlApp.state.modelType)).show(); + + $(mlApp.wrapSelector('.vp-eval-box')).hide(); + $(mlApp.wrapSelector('.vp-eval-' + mlApp.state.modelType)).show(); + + if (mlApp.state.modelType == 'rgs') { + // Regression + + } else if (mlApp.state.modelType == 'clf') { + // Classification + // if (this.state.roc_curve == false && this.state.auc == false) { + // $(page).find('.vp-ev-model.roc-auc').prop('disabled', true); + // } + } else { + // Clustering + if (mlApp.state.silhouetteScore == false) { + $(mlApp.wrapSelector('.vp-ev-model.silhouette')).prop('disabled', true); + } + if (mlApp.state.ari == false && mlApp.state.nmi == false) { + $(mlApp.wrapSelector('.vp-ev-model.ari-nmi')).prop('disabled', true); + } + } + break; + } + } + + /** + * Custom pages + * @param {*} appId + */ + renderApp(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let tag = ''; + switch (appId) { + case 'pp_fit': + tag = this.templateForOptionPage(actions['fit']); + break; + case 'pp_predict': + tag = this.templateForOptionPage(actions['predict']); + break; + case 'pp_transform': + tag = this.templateForOptionPage(actions['transform']); + break; + case 'pp_fit_predict': + tag = this.templateForOptionPage(actions['fit_predict']); + break; + case 'pp_fit_transform': + tag = this.templateForOptionPage(actions['fit_transform']); + break; + case 'pp_fit_resample': + tag = this.templateForOptionPage(actions['fit_resample']); + break; + } + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(` +

${tag}

+ `); + } + + templateForOptionPage(packageObj) { + let optBox = new com_String(); + // render tag + packageObj && packageObj.options && packageObj.options.forEach(opt => { + let label = opt.name; + if (opt.label != undefined) { + label = opt.label; + } + // fix label + label = com_util.optionToLabel(label); + optBox.appendFormatLine('{2}' + , opt.name, opt.name, label); + let tmpState = {}; + if (opt.value && opt.value !== '') { + tmpState[opt.name] = opt.value; + } + let content = com_generator.renderContent(this, opt.component[0], opt, tmpState); + optBox.appendLine(content[0].outerHTML); + }); + return optBox.toString(); + } + + checkBeforeRun() { + let that = this; + var result = true; + for (let idx = 0; idx < this.state.pipeline.length; idx++) { + let ppObj = this.state.pipeline[idx]; + var { name, label, useApp, app } = ppObj; + let requiredList = []; + result = true; + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; + if (isVisible && isEnabled) { + switch (name) { + case 'ml_dataSplit': + requiredList = ['featureData', 'targetData']; + // check required data + for (let i = 0; i < requiredList.length; i++) { + let reqKey = requiredList[i]; + result = that._checkIsEmpty($(app.wrapSelector('#' + reqKey))); + if (result === false) { + // show page and focus it + $(that.wrapSelector(`.vp-pp-item[data-name="${name}"]`)).click(); + $(app.wrapSelector('#' + reqKey)).focus(); + break; + } + } + break; + case 'ml_gridSearch': + result = app.checkBeforeRun(); + if (result === false) { + // show page + $(that.wrapSelector(`.vp-pp-item[data-name="${name}"]`)).click(); + break; + } + break; + } + } + if (result === false) { + break; + } + } + return result; + + } + + _checkIsEmpty(tag) { + let requiredFilled = true; + // if it's empty, focus on it + if (tag && $(tag) && $(tag).val() == '') { + requiredFilled = false; + } + return requiredFilled; + } + + generateCodeForOptionPage(appId) { + let actions = this.modelEditor.getAction(this.state.modelTypeName); + let actObj = {}; + switch (appId) { + case 'pp_fit': + actObj = actions['fit']; + break; + case 'pp_predict': + actObj = actions['predict']; + break; + case 'pp_transform': + actObj = actions['transform']; + break; + case 'pp_fit_predict': + actObj = actions['fit_predict']; + break; + case 'pp_fit_transform': + actObj = actions['fit_transform']; + break; + case 'pp_fit_resample': + actObj = actions['fit_resample']; + break; + } + + let code = new com_String(); + if (actObj.import != undefined) { + code.appendLine(actObj.import); + code.appendLine(); + } + + let that = this; + let state = { + model: this.state.model + }; + $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"] .vp-state`)).each((idx, tag) => { + let id = tag.id; + let value = that._getTagValue(tag); + state[id] = value; + }); + + let modelCode = com_generator.vp_codeGenerator(this, actObj, state); + modelCode = modelCode.replace('${model}', state.model); + code.append(modelCode); + return { state: state, code: code.toString() }; + } + + generateCode() { + let that = this; + let { template } = this.state; + let code = new com_String(); + + let stepNo = 1; + this.state.pipeline.forEach((ppObj, idx) => { + let { name, label, useApp, app } = ppObj; + + // check disabled + let isVisible = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).is(':visible') === true; + let isEnabled = $(that.wrapSelector(`.vp-pp-item[data-step="${idx}"]`)).attr('data-flag') === 'enabled'; + if (isVisible && isEnabled) { + if (code.toString() !== '') { + code.appendLine(); + code.appendLine(); + } + if (useApp === true) { + let appCode = app.generateCode(); + if (appCode instanceof Array) { + appCode = appCode.join('\n'); + } + if (appCode && appCode.trim() !== '') { + code.appendFormatLine("# [{0}] {1}", stepNo++, label); + if (name === 'ml_evaluation') { + // import auto generate + code.appendLine(app.generateImportCode().join('\n')); + } + code.append(appCode); + } + // save state + that.state.pipeline[idx].state = app.state; + } else { + let ppResult = that.generateCodeForOptionPage(name); + if (ppResult && ppResult?.code?.trim() !== '') { + code.appendFormatLine("# [{0}] {1}", stepNo++, label); + code.append(ppResult.code); + } + // save state + that.state.pipeline[idx].state = ppResult.state; + } + } + }); + + return code.toString(); + } + + } + + return Pipeline; +}); \ No newline at end of file diff --git a/visualpython/js/m_ml/Regression.js b/visualpython/js/m_ml/Regression.js index b4fde181..3c21d077 100644 --- a/visualpython/js/m_ml/Regression.js +++ b/visualpython/js/m_ml/Regression.js @@ -88,7 +88,11 @@ define([ let config = that.modelConfig[that.state.modelType]; if (config && config.install != undefined) { // insert install code - com_interface.insertCell('code', config.install, true, 'Machine Learning > Regression'); + let installCode = config.install; + if (vpConfig.extensionType === 'lite') { + installCode = installCode.replace('!', '%'); + } + com_interface.insertCell('code', installCode, true, 'Machine Learning > Regression'); } }); diff --git a/visualpython/js/m_ml/SaveLoad.js b/visualpython/js/m_ml/SaveLoad.js index 0940b028..454f468c 100644 --- a/visualpython/js/m_ml/SaveLoad.js +++ b/visualpython/js/m_ml/SaveLoad.js @@ -86,7 +86,7 @@ define([ // render tag config.options.forEach(opt => { optBox.appendFormatLine('{2}' - , opt.name, opt.name, opt.name); + , opt.name, opt.name, com_util.optionToLabel(opt.name)); let content = com_generator.renderContent(this, opt.component[0], opt, state); optBox.appendLine(content[0].outerHTML); }); diff --git a/visualpython/js/m_ml/dataSplit.js b/visualpython/js/m_ml/dataSplit.js index bfa2aac2..a8c9417f 100644 --- a/visualpython/js/m_ml/dataSplit.js +++ b/visualpython/js/m_ml/dataSplit.js @@ -175,7 +175,7 @@ define([ options.appendFormat(', shuffle={0}', shuffle); } if (shuffle != 'False' && stratify && stratify != '') { - options.appendFormat(', startify={0}', stratify); + options.appendFormat(', stratify={0}', stratify); } let code = new com_String(); diff --git a/visualpython/js/m_ml/evaluation.js b/visualpython/js/m_ml/evaluation.js index 6231b4a3..ed63120c 100644 --- a/visualpython/js/m_ml/evaluation.js +++ b/visualpython/js/m_ml/evaluation.js @@ -33,7 +33,7 @@ define([ this.config.checkModules = ['metrics']; this.state = { - modelType: 'rgs', + modelType: 'rgs', // rgs / clf / cls predictData: 'pred', targetData: 'y_test', // regression @@ -156,37 +156,11 @@ define([ }); $(page).find('#targetData2').replaceWith(targetData2Selector.toTagString()); - // load state - let that = this; - Object.keys(this.state).forEach(key => { - let tag = $(page).find('#' + key); - let tagName = $(tag).prop('tagName'); // returns with UpperCase - let value = that.state[key]; - if (value == undefined) { - return; - } - switch(tagName) { - case 'INPUT': - let inputType = $(tag).prop('type'); - if (inputType == 'text' || inputType == 'number' || inputType == 'hidden') { - $(tag).val(value); - break; - } - if (inputType == 'checkbox') { - $(tag).prop('checked', value); - break; - } - break; - case 'TEXTAREA': - case 'SELECT': - default: - $(tag).val(value); - break; - } - }); - $(page).find('.vp-upper-box').hide(); $(page).find('.vp-upper-box.' + this.state.modelType).show(); + + $(page).find('.vp-eval-box').hide(); + $(page).find('.vp-eval-' + this.state.modelType).show(); if (this.state.modelType == 'rgs') { // Regression @@ -228,15 +202,20 @@ define([ sizeOfClusters, silhouetteScore, ari, nmi, clusteredIndex, featureData2, targetData2 } = this.state; + // add import code for display and Markdown + let needDisplay = false; + let needMarkdown = false; //==================================================================== - // Classfication + // Classification //==================================================================== if (modelType == 'clf') { if (confusion_matrix) { code = new com_String(); code.appendLine("# Confusion Matrix"); - code.appendFormat('pd.crosstab({0}, {1}, margins=True)', targetData, predictData); + code.appendLine("display(Markdown('### Confusion Matrix'))"); + code.appendFormat('display(pd.crosstab({0}, {1}, margins=True))', targetData, predictData); + needMarkdown = true; codeCells.push(code.toString()); } if (report) { @@ -248,25 +227,25 @@ define([ if (accuracy) { code = new com_String(); code.appendLine("# Accuracy"); - code.appendFormat('metrics.accuracy_score({0}, {1})', targetData, predictData); + code.appendFormat("print('Accuracy: {}'.format(metrics.accuracy_score({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (precision) { code = new com_String(); code.appendLine("# Precision"); - code.appendFormat("metrics.precision_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('Precision: {}'.format(metrics.precision_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } if (recall) { code = new com_String(); code.appendLine("# Recall"); - code.appendFormat("metrics.recall_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('Recall: {}'.format(metrics.recall_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } if (f1_score) { code = new com_String(); code.appendLine("# F1-score"); - code.appendFormat("metrics.f1_score({0}, {1}, average='weighted')", targetData, predictData); + code.appendFormat("print('F1-score: {}'.format(metrics.f1_score({0}, {1}, average='weighted')))", targetData, predictData); codeCells.push(code.toString()); } // if (roc_curve) { @@ -300,14 +279,14 @@ define([ // } if (r_squared) { code = new com_String(); - code.appendLine("# R square"); - code.appendFormat('metrics.r2_score({0}, {1})', targetData, predictData); + code.appendLine("# R squared"); + code.appendFormat("print('R squared: {}'.format(metrics.r2_score({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (mae) { code = new com_String(); code.appendLine("# MAE(Mean Absolute Error)"); - code.appendFormat('metrics.mean_absolute_error({0}, {1})', targetData, predictData); + code.appendFormat("print('MAE: {}'.format(metrics.mean_absolute_error({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (mape) { @@ -316,22 +295,24 @@ define([ code.appendLine('def MAPE(y_test, y_pred):'); code.appendLine(' return np.mean(np.abs((y_test - pred) / y_test)) * 100'); code.appendLine(); - code.appendFormat('MAPE({0}, {1})', targetData, predictData); + code.appendFormat("print('MAPE: {}'.format(MAPE({0}, {1})))", targetData, predictData); codeCells.push(code.toString()); } if (rmse) { code = new com_String(); code.appendLine("# RMSE(Root Mean Squared Error)"); - code.appendFormat('metrics.mean_squared_error({0}, {1})**0.5', targetData, predictData); + code.appendFormat("print('RMSE: {}'.format(metrics.mean_squared_error({0}, {1})**0.5))", targetData, predictData); codeCells.push(code.toString()); } if (scatter_plot) { code = new com_String(); code.appendLine('# Regression plot'); + code.appendLine("display(Markdown('### Regression plot'))"); code.appendFormatLine('plt.scatter({0}, {1})', targetData, predictData); code.appendFormatLine("plt.xlabel('{0}')", targetData); code.appendFormatLine("plt.ylabel('{0}')", predictData); code.append('plt.show()'); + needMarkdown = true; codeCells.push(code.toString()); } } @@ -346,23 +327,34 @@ define([ if (silhouetteScore) { code = new com_String(); code.appendLine("# Silhouette score"); - code.appendFormat("print(f'Silhouette score: {metrics.cluster.silhouette_score({0}, {1})}')", featureData2, clusteredIndex); + code.appendFormat("print('Silhouette score: {}'.format(metrics.cluster.silhouette_score({0}, {1})))", featureData2, clusteredIndex); codeCells.push(code.toString()); } if (ari) { code = new com_String(); code.appendLine("# ARI(Adjusted Rand score)"); - code.appendFormat("print(f'ARI: {metrics.cluster.adjusted_rand_score({0}, {1})}')", targetData2, clusteredIndex); + code.appendFormat("print('ARI: {}'.format(metrics.cluster.adjusted_rand_score({0}, {1})))", targetData2, clusteredIndex); codeCells.push(code.toString()); } if (nmi) { code = new com_String(); code.appendLine("# NMI(Normalized Mutual Info Score)"); - code.appendFormat("print(f'NM: {metrics.cluster.normalized_mutual_info_score({0}, {1})}')", targetData2, clusteredIndex); + code.appendFormat("print('NM: {}'.format(metrics.cluster.normalized_mutual_info_score({0}, {1})))", targetData2, clusteredIndex); codeCells.push(code.toString()); } } - // return as seperated cells + if (needMarkdown === true) { + codeCells = [ + "from IPython.display import display, Markdown", + ...codeCells + ]; + } else if (needDisplay === true) { + codeCells = [ + "from IPython.display import display", + ...codeCells + ]; + } + // return as separated cells return codeCells; } diff --git a/visualpython/js/m_stats/Anova.js b/visualpython/js/m_stats/Anova.js new file mode 100644 index 00000000..4d6f901f --- /dev/null +++ b/visualpython/js/m_stats/Anova.js @@ -0,0 +1,511 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Anova.js + * Author : Black Logic + * Note : ANOVA + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] Anova +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/anova.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/m_apps/Subset' +], function(nmHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { + + /** + * Anova + */ + class Anova extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd', 'np', 'stats', 'vp_confidence_interval', 'vp_sem']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'one-way', + data: '', + dataType: '', + depVar: '', + factor: '', + factorA: '', + factorB: '', + covariate: '', + sigLevel: 0.05, + // Post hoc analysis option + tukeyHSD: true, + tukey: false, + scheffe: false, + duncan: false, + bonferroni: false, + // Display option + statistics: true, + boxplot: true, + equalVariance: true, + interPlot: true, + ...this.state + }; + + this.columnBindList = ['depVar', 'factor', 'factorA', 'factorB', 'covariate']; + + this.tmpInstallCode = []; // install codes + + this.subsetEditor = {}; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + + that.tmpInstallCode = []; + that.hideInstallButton(); + + if (testType === 'one-way' || testType === 'two-way') { + if (that.state.tukey || that.state.scheffe || that.state.duncan) { + // Add installation code + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install scikit-posthocs"]; + } else { + that.tmpInstallCode = ["!pip install scikit-posthocs"]; + } + that.showInstallButton(); + } + } else if (testType === 'ancova') { + // Add installation code : # pip install pingouin + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install pingouin"]; + } else { + that.tmpInstallCode = ["!pip install pingouin"]; + } + that.showInstallButton(); + } + }); + + $(this.wrapSelector('#data')).on('change', function() { + if (that.state.dataType === 'Series') { + // Series + that.columnBindList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } else { + // DataFrame + that.columnBindList.forEach(id => { + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', that.columnBindList, 'select', false, false); + } + }); + + $(this.wrapSelector('.vp-st-posthoc-box .vp-state')).on('change', function() { + let id = $(this)[0].id; + let checked = $(this).prop('checked') === true; + that.state[id] = checked; + let { testType, tukey, scheffe, duncan } = that.state; + if (testType === 'one-way' || testType === 'two-way') { + if (tukey || scheffe || duncan) { + // Add installation code + if (vpConfig.extensionType === 'lite') { + that.tmpInstallCode = ["%pip install scikit-posthocs"]; + } else { + that.tmpInstallCode = ["!pip install scikit-posthocs"]; + } + that.showInstallButton(); + } else { + that.hideInstallButton(); + } + } + }); + + $(this.wrapSelector('')) + } + + templateForBody() { + let page = $(nmHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor['data'] = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + $(that.wrapSelector('#data')).val(code); + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', this.columnBindList, 'select', false, false); + } + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + } + + generateInstallCode() { + return this.tmpInstallCode; + } + + generateCode() { + let { + testType, data, depVar, factor, factorA, factorB, covariate, sigLevel, + // Post hoc analysis option + tukeyHSD, tukey, scheffe, duncan, bonferroni, + // Display option + statistics, boxplot, equalVariance, interPlot + } = this.state; + + // get only text without '' or "" + let depVarText = $(this.wrapSelector('#depVar option:selected')).text(); + let factorText = $(this.wrapSelector('#factor option:selected')).text(); + let factorAText = $(this.wrapSelector('#factorA option:selected')).text(); + let factorBText = $(this.wrapSelector('#factorB option:selected')).text(); + let covariateText = $(this.wrapSelector('#covariate option:selected')).text(); + + let codeList = []; + let code = new com_String(); + + // test type label + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + code.appendFormatLine("# {0}", testTypeLabel); + code.appendFormat("vp_df = {0}.dropna().copy()", data); + + switch (testType) { + case 'one-way': + // 1. One-way ANOVA + code.appendLine(); + code.appendLine(); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby({0})[{1}])).items():", factor, depVar); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.append(" _df = pd.concat([_df, _df_t], axis=1)"); + + // Display - Statistics + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Count':_df.count(),'Mean':_df.mean(numeric_only=True),'Std. Deviation':_df.std(numeric_only=True),'Min':_df.min(),'Max':_df.max(),"); + code.appendLine(" 'Std. Error Mean':_df.apply(vp_sem),'Confidence interval':0.95,"); + code.append(" 'Lower':_df.apply(vp_confidence_interval).T[0],'Upper':_df.apply(vp_confidence_interval).T[1] }))"); + } + // Display - Boxplot + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Boxplot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.boxplot(data=_df)"); + code.append(" plt.show()"); + } + // Display - Equal Variance + if (equalVariance === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Equal Variance test (Levene)"); + code.appendLine("from scipy import stats"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendLine("_res = stats.levene(*_lst, center='mean')"); + code.appendLine("display(Markdown('### Equal Variance test (Levene)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue}, index=['Equal Variance test (Levene)']))"); + } + + code.appendLine(); + code.appendLine(); + code.appendLine("# One-way ANOVA"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendLine("from statsmodels.stats.anova import anova_lm"); + code.appendFormatLine("_model = smf.ols('{0} ~ C({1})', vp_df)", depVarText, factorText); + code.appendLine("_result = _model.fit()"); + code.appendLine("_dfr = anova_lm(_result)"); + code.appendLine("_dfr.loc['Total','df'] = _dfr['df'].sum()"); + code.appendLine("_dfr.loc['Total','sum_sq'] = _dfr['sum_sq'].sum()"); + code.appendLine("display(Markdown('### One-way ANOVA'))"); + code.append("display(_dfr)"); + + // Post hoc analysis - Tukey HSD + if (tukeyHSD === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey HSD"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).tukeyhsd(alpha={2})", depVar, factor, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Tukey HSD'))"); + code.append("display(_res.summary())"); + } + // Post hoc analysis - Bonferroni + if (bonferroni === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Bonferroni"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).allpairtest(stats.ttest_ind,alpha={2},method='bonf')", depVar, factor, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Bonferroni'))"); + code.append("display(_res[0])"); + } + + if (tukey === true || scheffe === true || duncan === true) { + // Post hoc analysis - Tukey + if (tukey === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Tukey'))"); + code.appendFormat("display(sph.posthoc_tukey(vp_df, val_col={0}, group_col={1}))", depVar, factor); + } + // Post hoc analysis - Scheffe + if (scheffe === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Scheffe"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Scheffe'))"); + code.appendFormat("display(sph.posthoc_scheffe(vp_df, val_col={0}, group_col={1}))", depVar, factor); + + } + // Post hoc analysis - duncan + if (duncan === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Duncan"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Duncan'))"); + code.appendFormat("display(sph.posthoc_dunn(vp_df, val_col={0}, group_col={1}))", depVar, factor); + } + } + + break; + case 'two-way': + // 1. Two-way ANOVA + code.appendLine(); + code.appendLine(); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby([{0},{1}])[{2}])).items():", factorB, factorA, depVar); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.appendLine(" _df = pd.concat([_df, _df_t], axis=1)"); + code.append(" _df.columns = [[x[0] for x in _df.columns],[x[1] for x in _df.columns]]"); + + // Display - Statistics + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Count':_df.count(),'Mean':_df.mean(numeric_only=True),'Std. Deviation':_df.std(numeric_only=True),'Min':_df.min(),'Max':_df.max(),"); + code.appendLine(" 'Std. Error Mean':_df.apply(vp_sem),'Confidence interval':0.95,"); + code.append(" 'Lower':_df.apply(vp_confidence_interval).T[0],'Upper':_df.apply(vp_confidence_interval).T[1] }))"); + } + // Display - Boxplot + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Boxplot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.boxplot(data=_df)"); + code.append(" plt.show()"); + } + // Display - Equal Variance test + if (equalVariance === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Equal Variance test (Levene)"); + code.appendLine("from scipy import stats"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendLine("_res = stats.levene(*_lst, center='mean')"); + code.appendLine("display(Markdown('### Equal Variance test (Levene)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue}, index=['Equal Variance test (Levene)']))"); + } + + code.appendLine(); + code.appendLine(); + code.appendLine("# Two-way ANOVA"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendLine("from statsmodels.stats.anova import anova_lm"); + code.appendFormatLine("_model = smf.ols('{0} ~ C({1}) + C({2}) + C({3}):C({4})', vp_df)", depVarText, factorAText, factorBText, factorAText, factorBText); + code.appendLine("_result = _model.fit()"); + code.appendLine("_dfr = anova_lm(_result)"); + code.appendLine("_dfr.loc['Total','df'] = _dfr['df'].sum()"); + code.appendLine("_dfr.loc['Total','sum_sq'] = _dfr['sum_sq'].sum()"); + code.appendLine("display(Markdown('### Two-way ANOVA'))"); + code.append("display(_dfr)"); + + // Display - Interaction plot + if (interPlot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Interaction plot"); + code.appendLine("from statsmodels.graphics.factorplots import interaction_plot"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendFormatLine(" fig = interaction_plot(x=vp_df[{0}], trace=vp_df[{1}], response=vp_df[{2}])", factorA, factorB, depVar); + code.append(" plt.show()"); + } + // Post hoc analysis - Tukey HSD + if (tukeyHSD === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey HSD"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).tukeyhsd(alpha={2})", depVar, factorA, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Tukey HSD'))"); + code.append("display(_res.summary())"); + } + // Post hoc analysis - Bonferroni + if (bonferroni === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Bonferroni"); + code.appendLine("from statsmodels.sandbox.stats.multicomp import MultiComparison"); + code.appendFormatLine("_res = MultiComparison(vp_df[{0}], vp_df[{1}]).allpairtest(stats.ttest_ind,alpha={2},method='bonf')", depVar, factorA, sigLevel); + code.appendLine("display(Markdown('### Post-hoc: Bonferroni'))"); + code.append("display(_res[0])"); + } + if (tukey === true || scheffe === true || duncan === true) { + // Post hoc analysis - Tukey + if (tukey === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Tukey"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Tukey'))"); + code.appendFormat("display(sph.posthoc_tukey(vp_df, val_col={0}, group_col={1}))", depVar, factorA); + } + // Post hoc analysis - Scheffe + if (scheffe === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Scheffe"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Scheffe'))"); + code.appendFormat("display(sph.posthoc_scheffe(vp_df, val_col={0}, group_col={1}))", depVar, factorA); + } + // Post hoc analysis - Duncan + if (duncan === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Post-hoc: Duncan"); + code.appendLine("import scikit_posthocs as sph"); + code.appendLine("display(Markdown('### Post-hoc: Duncan'))"); + code.appendFormat("display(sph.posthoc_dunn(vp_df, val_col={0}, group_col={1}))", depVar, factorA); + } + } + break; + case 'ancova': + // 1. ANCOVA + code.appendLine(); + code.appendLine(); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby({0})[{1}])).items():", factor, depVar); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.append(" _df = pd.concat([_df, _df_t], axis=1)"); + + // Display - Statistics + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Count':_df.count(),'Mean':_df.mean(numeric_only=True),'Std. Deviation':_df.std(numeric_only=True),'Min':_df.min(),'Max':_df.max(),"); + code.appendLine(" 'Std. Error Mean':_df.apply(vp_sem),'Confidence interval':0.95,"); + code.append(" 'Lower':_df.apply(vp_confidence_interval).T[0],'Upper':_df.apply(vp_confidence_interval).T[1] }))"); + } + // Display - Boxplot + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Boxplot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.boxplot(data=_df)"); + code.append(" plt.show()"); + } + + code.appendLine(); + code.appendLine(); + code.appendLine("# ANCOVA - Analysis of covariance"); + code.appendLine("import pingouin as pg"); + code.appendLine("display(Markdown('### ANCOVA - Analysis of covariance'))"); + code.appendFormat("display(pg.ancova(data=vp_df, dv={0}, between={1}, covar={2}))", depVar, factor, covariate); + break; + } + + codeList.push(code.toString()); + + return codeList; + } + + } + + return Anova; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/Chi2test.js b/visualpython/js/m_stats/Chi2test.js new file mode 100644 index 00000000..03d3cab2 --- /dev/null +++ b/visualpython/js/m_stats/Chi2test.js @@ -0,0 +1,215 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Chi2test.js + * Author : Black Logic + * Note : Chi-square test of independence + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] Chi2test +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/chi2test.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/m_apps/Subset' +], function(nmHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { + + /** + * Chi2test + */ + class Chi2test extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + data: '', + dataType: '', + row: '', + column: '', + barplot: true, + crossTab: true, + cramersCoef: true, + ...this.state + }; + + this.subsetEditor = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let bindIdList = ['row', 'column']; + if (that.state.dataType === 'Series') { + // Series + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } else { + // DataFrame + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', bindIdList, 'select', false, false); + } + }); + } + + handleVariableChange(data) { + this.state.data = data; + let bindIdList = ['row', 'column']; + if (that.state.dataType === 'DataFrame') { + // DataFrame + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', bindIdList, 'select', false, false); + } else { + // Others + bindIdList.forEach(id => { + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } + } + + templateForBody() { + let page = $(nmHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).val(code); + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', ['row', 'column'], 'select', false, false); + } + } + + generateCode() { + let { data, row, column, barplot, crossTab, cramersCoef } = this.state; + let codeList = []; + let code = new com_String(); + + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + + // Display option + if (barplot === true) { + code.appendLine(); + code.appendLine("# Count plot"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendFormatLine(" sns.countplot(data=vp_df, x={0}, hue={1})", row, column); + code.appendLine(" plt.show()"); + } + + code.appendLine(""); + code.appendLine("# Chi-square test of independence"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_obs = pd.crosstab(index=vp_df[{0}], columns=vp_df[{1}])", row, column); + code.appendLine("_res1 = stats.chi2_contingency(_obs)"); + code.appendLine("_res2 = stats.chi2_contingency(_obs, lambda_='log-likelihood')"); + + if (crossTab === true) { + code.appendLine(""); + code.appendLine("# Cross tabulation: Count"); + code.appendFormatLine("_dfc = pd.crosstab(index=vp_df[{0}],columns=vp_df[{1}],margins=True,margins_name='Total')", row, column); + code.appendLine("_dfc = _dfc.reset_index().reset_index()"); + code.appendLine("_dfc[' '] = 'Count'"); + code.appendLine(""); + code.appendLine("# Cross tabulation: Expected count"); + code.appendLine("_dfe = pd.DataFrame(_res1.expected_freq, index=_obs.index, columns=_obs.columns).round(1)"); + code.appendLine("_dfe.loc['Total',:] = _dfe.sum(axis=0)"); + code.appendLine("_dfe.loc[:,'Total'] = _dfe.sum(axis=1)"); + code.appendLine("_dfe = _dfe.reset_index().reset_index()"); + code.appendLine("_dfe[' '] = 'Expected count'"); + code.appendLine(""); + code.appendLine("# Cross tabulation: Count + Expected count"); + code.appendLine("display(Markdown('### Cross tabulation'))"); + code.appendFormatLine("display(pd.concat([_dfc, _dfe]).set_index([{0},' ']).sort_values('index').drop('index',axis=1))", row); + } + + code.appendLine(""); + code.appendLine("# Chi-square test"); + code.appendLine("display(Markdown('### Chi-square test'))"); + code.appendLine("display(pd.DataFrame(data = {'Value':[_res1.statistic,_res2.statistic,vp_df.dropna().shape[0]],"); + code.appendLine(" 'df':[_res1.dof,_res2.dof,np.nan],"); + code.appendLine(" 'p-value(two-sided)':[_res1.pvalue,_res2.pvalue,np.nan]},"); + code.append(" index= ['Pearson Chi-square','Likelihood ratio','N of valid cases']))"); + + if (cramersCoef === true) { + code.appendLine(""); + code.appendLine(""); + code.appendLine("# Cramers' V coefficient"); + code.appendLine("_X2 = stats.chi2_contingency(_obs)[0]"); + code.appendLine("_sum = _obs.sum().sum()"); + code.appendLine("_minDim = min(_obs.shape)-1"); + code.appendLine("display(Markdown('### Cramers V coefficient'))"); + code.append("display(pd.DataFrame(data={'Value':np.sqrt((_X2/_sum) / _minDim)}, index=['Cramers V coefficient']))"); + } + codeList.push(code.toString()); + + return codeList; + } + + } + + return Chi2test; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/CorrAnalysis.js b/visualpython/js/m_stats/CorrAnalysis.js new file mode 100644 index 00000000..92251434 --- /dev/null +++ b/visualpython/js/m_stats/CorrAnalysis.js @@ -0,0 +1,202 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : CorrAnalysis.js + * Author : Black Logic + * Note : Correlation Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] CorrAnalysis +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/corrAnalysis.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * CorrAnalysis + */ + class CorrAnalysis extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + data: '', + variable: [], + corrType: 'pearson', + corrAnlaysis: true, + corrMatrix: true, + corrHeatmap: false, + scatterMatrix: false, + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variable?.map(x=>x.code), showDescription: false } + ); + } + + generateCode() { + let { data, variable, corrType, corrAnlaysis, corrMatrix, corrHeatmap, scatterMatrix } = this.state; + let codeList = []; + let code = new com_String(); + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + this.state.variable = columns; + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.append('.dropna().copy()'); + + let corrTypeLabel = $(this.wrapSelector('#corrType option:selected')).text(); + + // Display option : Correlation Analysis + if (corrAnlaysis === true) { + // Inner function : vp_confidence_interval_corr + this.addCheckModules('vp_confidence_interval_corr'); + + code.appendLine(); + code.appendLine(); + code.appendLine("# Correlation Analysis"); + code.appendLine("from scipy import stats"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("_dfr = pd.DataFrame()"); + code.appendLine("for i, col1 in enumerate(vp_df.columns):"); + code.appendLine(" for j, col2 in enumerate(vp_df.columns):"); + code.appendLine(" if i >= j: continue"); + code.appendLine(" if pd.api.types.is_numeric_dtype(vp_df[col1]) and pd.api.types.is_numeric_dtype(vp_df[col2]):"); + code.appendFormatLine(" _res = vp_confidence_interval_corr(vp_df[col1], vp_df[col2], method='{0}')", corrType); + code.appendLine(" _df_t = pd.DataFrame(data={'Variable1':col1,'Variable2':col2,'N':vp_df[col1].size,'Correlation coefficient':_res[0],"); + code.appendLine(" 'p-value':_res[1],'Lower(95%)':_res[2],'Upper(95%)':_res[3]}, index=[0])"); + code.appendLine(" _dfr = pd.concat([_dfr, _df_t]).reset_index(drop=True)"); + code.appendFormatLine("display(Markdown('### Correlation Analysis: {0}'))", corrTypeLabel.replace("'", "\\'")); + code.append("display(_dfr)"); + } + + // Display option : Correlation Matrix + if (corrMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Correlation matrix: {0}", corrTypeLabel); + code.appendLine("from IPython.display import display"); + code.appendFormat("display(vp_df.corr(method='{0}', numeric_only=True).round(2))", corrType); + } + + if (corrHeatmap === true || scatterMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Chart"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.append(" warnings.simplefilter(action='ignore', category=Warning)"); + // Display option : Correlation Heatmap + if (corrHeatmap === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Heatmap"); + code.appendFormatLine(" sns.heatmap(vp_df.corr(method='{0}', numeric_only=True), annot=True, fmt='.2f', cmap='coolwarm')", corrType); + code.appendFormatLine(" plt.title('Correlation heatmap: {0}')", corrTypeLabel.replace("'", "\\'")); + code.append(" plt.show()"); + } + // Display option : Scatter Matrix + if (scatterMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Scatter matrix"); + code.appendLine(" pd.plotting.scatter_matrix(vp_df)"); + code.append(" plt.show()"); + } + } + codeList.push(code.toString()); + + return codeList; + } + + } + + return CorrAnalysis; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/DescStats.js b/visualpython/js/m_stats/DescStats.js new file mode 100644 index 00000000..e6a9aa45 --- /dev/null +++ b/visualpython/js/m_stats/DescStats.js @@ -0,0 +1,320 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : DescStats.js + * Author : Black Logic + * Note : Descriptive Statistics + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 31 + * Change Date : + */ + +//============================================================================ +// [CLASS] DescStats +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/descStats.html'), + __VP_CSS_LOADER__('vp_base/css/m_stats/descStats'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, dsCss, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * DescStats + */ + class DescStats extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + + this.state = { + data: '', + variable: [], + // Central tendency + mean: true, + median: false, + mode: false, + sum: true, + // Dispersion + min: false, + max: false, + range: false, + std: true, + var: true, + semean: false, + skew: false, + kurtosis: false, + // Percentile values + quantile: true, + usePercentile: false, + percentiles: [], + // Frequency table + frequency: true, + percent: true, + validPercent: true, + cumulativePercent: true, + noUniqVals: 10, + // Display + histogram: true, + scatterMatrix: true, + boxplot: true, + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // data selection + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // use percentile + $(this.wrapSelector('#usePercentile')).on('change', function() { + let checked = $(this).prop('checked'); + if (checked === true) { + // enable percentile editing + $(that.wrapSelector('#percentile')).prop('disabled', false); + $(that.wrapSelector('#addPercentile')).prop('disabled', false); + $(that.wrapSelector('.vp-percentile-box')).removeClass('disabled'); + } else { + // disable percentile editing + $(that.wrapSelector('#percentile')).prop('disabled', true); + $(that.wrapSelector('#addPercentile')).prop('disabled', true); + $(that.wrapSelector('.vp-percentile-box')).addClass('disabled'); + + } + }); + + // add percentile + $(this.wrapSelector('#addPercentile')).on('click', function() { + let newVal = $(that.wrapSelector('#percentile')).val(); + if (newVal && newVal !== '') { + let newValNum = parseInt(newVal); + that.addPercentile(newValNum); + that.state.percentiles.push(newValNum); + $(that.wrapSelector('#percentile')).val(''); + } + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + addPercentile(percentile) { + if (this.state.percentiles.indexOf(percentile) === -1) { + $(this.wrapSelector('.vp-percentile-box')).append( + $(`

${percentile}

`)); + + // delete percentile + let that = this; + $(this.wrapSelector('.vp-percentile-box:not(.disabled) .vp-percentile-remove')).on('click', function() { + if (that.state.usePercentile === true) { + let delVal = parseInt($(this).parent().find('.vp-percentile-value').text()); + that.state.percentiles = that.state.percentiles.filter(x => x !== delVal); + $(this).closest('.vp-percentile-item').remove(); + } + }); + } + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, showDescription: false } + ); + } + + generateCode() { + let { data, variable, + // Central tendency + mean,median,mode,sum, + // Dispersion + min,max,range,std,variance,semean,skew,kurtosis, + // Percentile values + quantile,usePercentile,percentiles, + // Frequency table + frequency,percent,validPercent,cumulativePercent,noUniqVals, + // Display + histogram,scatterMatrix,boxplot + } = this.state; + let codeList = []; + let code = new com_String(); + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.appendLine('.copy()'); + + // Descriptive statistics + code.appendLine(); + code.appendLine("# Descriptive statistics"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("display(Markdown('### Descriptive statistics'))"); + code.appendLine("display(pd.DataFrame({"); + code.appendLine(" 'N Total':vp_df.shape[0],"); + code.appendLine(" 'N Valid':vp_df.count(numeric_only=True),"); + code.appendLine(" 'N Missing':vp_df.loc[:,vp_df.apply(pd.api.types.is_numeric_dtype)].isnull().sum(),"); + if (mean === true) code.appendLine(" 'Mean':vp_df.mean(numeric_only=True),"); + if (median === true) code.appendLine(" 'Median':vp_df.median(numeric_only=True),"); + if (mode === true) code.appendLine(" 'Mode':vp_df.mode(numeric_only=True).iloc[0],"); + if (sum === true) code.appendLine(" 'Sum':vp_df.sum(numeric_only=True),"); + if (min === true) code.appendLine(" 'Minimun':vp_df.min(numeric_only=True),"); + if (max === true) code.appendLine(" 'Maximum':vp_df.max(numeric_only=True),"); + if (range === true) code.appendLine(" 'Range':vp_df.max(numeric_only=True) - vp_df.min(numeric_only=True),"); + if (std === true) code.appendLine(" 'Std. deviation':vp_df.std(numeric_only=True),"); + if (variance === true) code.appendLine(" 'Variance':vp_df.var(numeric_only=True),"); + if (semean === true) code.appendLine(" 'S.E. mean':vp_df.std(numeric_only=True)/np.sqrt(vp_df.count(numeric_only=True)),"); + if (skew === true) code.appendLine(" 'Skewness':vp_df.skew(numeric_only=True),"); + if (kurtosis === true) code.appendLine(" 'Kurtosis':vp_df.kurtosis(numeric_only=True),"); + let sortedPercentiles = []; + if (quantile === true) { + sortedPercentiles = [25, 50, 75]; + } + if (usePercentile === true && percentiles.length > 0) { + sortedPercentiles = [...sortedPercentiles, ...percentiles]; + } + sortedPercentiles.sort((a, b) => { return a - b; }); + sortedPercentiles.forEach(num => { + code.appendFormatLine(" 'Percentile: {0}':vp_df.quantile(q={1}, numeric_only=True),", num, (num / 100).toFixed(2)); + }); + code.appendLine("}).round(3).T)"); + + // Frequency table + code.appendLine(); + code.appendLine("# Frequency table"); + code.appendLine("display(Markdown('### Frequency table'))"); + code.appendLine("for col in vp_df.columns:"); + code.appendFormatLine(" if pd.api.types.is_numeric_dtype(vp_df[col]) and vp_df[col].value_counts().size > {0}:", noUniqVals); + code.appendFormatLine(" _bins = {0}", noUniqVals); + code.appendLine(" else: _bins = None"); + code.appendLine(" "); + code.appendLine(" _dfr = pd.DataFrame({"); + if (frequency === true) code.appendLine(" 'Frequency':vp_df[col].value_counts(bins=_bins, sort=False),"); + if (percent === true) code.appendLine(" 'Percent':100*(vp_df[col].value_counts(bins=_bins, sort=False) / vp_df[col].size),"); + if (validPercent === true) code.appendLine(" 'Valid percent':100*(vp_df[col].value_counts(bins=_bins, sort=False)/vp_df[col].count())"); + code.appendLine("}).round(2)"); + if (cumulativePercent === true) code.appendLine(" _dfr['Cumulative percent'] = _dfr['Valid percent'].cumsum()"); + code.appendLine(" _dfr.loc['N Valid',:] = _dfr.iloc[:,:3].sum()"); + code.appendLine(" _dfr.loc['N Missing','Frequency'] = vp_df[col].isnull().sum()"); + code.appendLine(" _dfr.loc['N Total','Frequency'] = vp_df[col].size"); + code.append(" display(_dfr)"); + + // Display option + if (histogram || scatterMatrix || boxplot) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Charts"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.append(" warnings.simplefilter(action='ignore', category=Warning)"); + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Histogram"); + code.appendLine(" idx = 1"); + code.appendLine(" for col in vp_df.columns:"); + code.appendLine(" plt.subplot(2,2, idx)"); + code.appendFormatLine(" if pd.api.types.is_numeric_dtype(vp_df[col]) and vp_df[col].value_counts().size > {0}:", noUniqVals); + code.appendLine(" sns.histplot(data=vp_df, x=col, kde=True)"); + code.appendLine(" else:"); + code.appendLine(" sns.countplot(data=vp_df, x=col)"); + code.appendLine(" "); + code.appendLine(" if idx < 4:"); + code.appendLine(" idx += 1"); + code.appendLine(" else:"); + code.appendLine(" idx = 1"); + code.appendLine(" plt.tight_layout()"); + code.append(" plt.show()"); + } + if (scatterMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Scatter matrix "); + code.appendLine(" pd.plotting.scatter_matrix(vp_df, marker='o', hist_kwds={'bins': 30}, s=30, alpha=.8)"); + code.append(" plt.show()"); + } + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Boxplot"); + code.appendLine(" sns.boxplot(vp_df)"); + code.append(" plt.show()"); + } + } + + return code.toString(); + } + + } + + return DescStats; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/EqualVarTest.js b/visualpython/js/m_stats/EqualVarTest.js index 3fe50bb9..3f6569e1 100644 --- a/visualpython/js/m_stats/EqualVarTest.js +++ b/visualpython/js/m_stats/EqualVarTest.js @@ -3,7 +3,7 @@ * Description : GUI-based Python code generator * File Name : EqualVarTest.js * Author : Black Logic - * Note : Equal Variance Test + * Note : Equal Variance test * License : GNU GPLv3 with Visual Python special exception * Date : 2023. 05. 09 * Change Date : @@ -13,12 +13,16 @@ // [CLASS] EqualVarTest //============================================================================ define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/equalVarTest.html'), 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/component/PopupComponent', - 'vp_base/js/com/component/DataSelector' -], function(com_util, com_Const, com_String, PopupComponent, DataSelector) { + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, MultiSelector, Subset) { /** * EqualVarTest @@ -27,36 +31,213 @@ define([ _init() { super._init(); /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'bartlett', + inputType: 'long-data', + data: '', + variableMulti: [], + variable: '', + factor: '', + center: 'mean', + histogram: true, + ...this.state + }; + + this.subsetEditor = {}; + this.columnSelector = {}; } _bindEvent() { super._bindEvent(); /** Implement binding events */ var that = this; + + // change test type + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + }); + + // change input type + $(this.wrapSelector('input[name="inputType"]:radio')).on('change', function() { + let inputType = $(this).val(); + that.state.inputType = inputType; + $(that.wrapSelector('.vp-variable-box')).hide(); + $(that.wrapSelector('.vp-variable-box.' + inputType)).show(); + }); + + // data change event + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); } - templateForBody() { - /** Implement generating template */ - return `This is sample. - `; + handleVariableChange(data) { + this.state.data = data; + // render column selector + com_generator.vp_bindColumnSource(this, 'data', ['variable', 'factor'], 'select', false, false); + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variableMulti'), + { mode: 'columns', parent: data, showDescription: false } + ); } - render() { - super.render(); + templateForBody() { + let page = $(eqHTML); + let that = this; + // generate dataselector let dataSelector = new DataSelector({ - type: 'data', - pageThis: this, - id: 'sample', - finish: function() { - ; + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); } }); - $(this.wrapSelector('#sample')).replaceWith(dataSelector.toTagString()); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + if (this.state.data !== '') { + // render column selector + com_generator.vp_bindColumnSource(this, 'data', ['variable', 'factor'], 'select', false, false); + } + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variableMulti'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variableMulti?.map(x => x.code), showDescription: false } + ); + + $(this.wrapSelector('.vp-variable-box')).hide(); + $(this.wrapSelector('.vp-variable-box.' + this.state.inputType)).show(); + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); } generateCode() { - return "print('sample code')"; + let { testType, inputType, data, variable, factor, center, histogram } = this.state; + let codeList = []; + let code = new com_String(); + let that = this; + + // test type label + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + code.appendFormatLine("# {0}", testTypeLabel); + + if (inputType === 'long-data') { + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + code.appendLine("_df = pd.DataFrame()"); + code.appendFormatLine("for k, v in dict(list(vp_df.groupby({0})[{1}])).items():", factor, variable); + code.appendLine(" _df_t = v.reset_index(drop=True)"); + code.appendLine(" _df_t.name = k"); + code.append(" _df = pd.concat([_df, _df_t], axis=1)"); + } else if (inputType === 'wide-data') { + // get variable multi + let columns = this.columnSelector.getDataList(); + this.state.variableMulti = columns; + code.appendFormatLine("vp_df = {0}[[{1}]].copy()", data, columns.map(x => x.code).join(', ')); // without dropna + code.append("_df = vp_df.copy()"); + } + + // add variance code + code.appendLine(); + code.appendLine(); + code.appendLine("# Variance"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendLine("_dfr = _df.var().to_frame()"); + code.appendLine("_dfr.columns = ['Variance']"); + code.append("display(_dfr)"); + + switch (testType) { + case 'bartlett': + // 1. Bartlett test + code.appendLine(); + code.appendLine(); + code.appendLine("# Bartlett test"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendLine("_res = stats.bartlett(*_lst)"); + code.appendLine("display(Markdown('### Bartlett test'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Equal Variance test (Bartlett)']))"); + break; + case 'levene': + // 1. Levene test + code.appendLine(); + code.appendLine(); + code.appendLine("# Levene test"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendFormatLine("_res = stats.levene(*_lst, center='{0}')", center); + code.appendLine("display(Markdown('### Levene test'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Equal Variance test (Levene)']))"); + break; + case 'fligner': + // 1. Fligner test + code.appendLine(); + code.appendLine(); + code.appendLine("# Fligner test"); + code.appendLine("_lst = []"); + code.appendLine("_df.apply(lambda x: _lst.append(x.dropna()))"); + code.appendFormatLine("_res = stats.fligner(*_lst, center='{0}')", center); + code.appendLine("display(Markdown('### Fligner test'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Equal Variance test (Fligner)']))"); + break; + } + + // Display option + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Histogram"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.histplot(_df, stat='density', kde=True)"); + code.appendLine(" plt.title('Histogram')"); + code.append(" plt.show()"); + } + codeList.push(code.toString()); + + return codeList; } } diff --git a/visualpython/js/m_stats/FactorAnalysis.js b/visualpython/js/m_stats/FactorAnalysis.js new file mode 100644 index 00000000..188efe19 --- /dev/null +++ b/visualpython/js/m_stats/FactorAnalysis.js @@ -0,0 +1,301 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : FactorAnalysis.js + * Author : Black Logic + * Note : Factor Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] FactorAnalysis +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/factorAnalysis.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * FactorAnalysis + */ + class FactorAnalysis extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.installButton = true; + this.config.docs = 'https://factor-analyzer.readthedocs.io/en/latest/factor_analyzer.html'; + + this.state = { + data: '', + variable: [], + rotation: "'varimax'", + method: 'principal', + impute: 'drop', + extract: 'eigenvalue', + eigenvalue: 1, + factor: '', + corrMatrix: true, + screePlot: true, + ...this.state + }; + + this.rotationList = [ + { label: "None", value: "None" }, + { label: "varimax", value: "'varimax'" }, + { label: "promax", value: "'promax'" }, + { label: "oblimin", value: "'oblimin'" }, + { label: "oblimax", value: "'oblimax'" }, + { label: "quartimin", value: "'quartimin'" }, + { label: "quartimax", value: "'quartimax'" }, + { label: "equamax", value: "'equamax'" }, + ]; + this.methodList = [ + { label: "minres", value: "minres" }, + { label: "ml", value: "ml" }, + { label: "principal", value: "principal" }, + ]; + this.imputeList = [ + { label: "drop", value: "drop" }, + { label: "mean", value: "mean" }, + { label: "median", value: "median" }, + ] + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + this.state.variable = []; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + // generate rotation options + this.rotationList.forEach(obj => { + let selected = obj.value === that.state.rotation; + $(page).find('#rotation').append(``); + }); + + // generate method options + this.methodList.forEach(obj => { + let selected = obj.value === that.state.method; + $(page).find('#method').append(``); + }); + + // generate impute options + this.imputeList.forEach(obj => { + let selected = obj.value === that.state.impute; + $(page).find('#impute').append(``); + }); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variable.map(x=>x.code), showDescription: false } + ); + } + + generateInstallCode() { + let installCode = '!pip install factor-analyzer'; + // Add installation code + if (vpConfig.extensionType === 'lite') { + installCode = '%pip install factor-analyzer'; + } + return [ installCode ]; + } + + generateCode() { + let { data, variable, rotation, method, impute, extract, eigenvalue, factor, corrMatrix, screePlot } = this.state; + let codeList = []; + let code = new com_String(); + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + this.state.variable = columns; + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.appendLine('.dropna().copy()'); + + // KMO(Kaiser-Meyer-Olkin) measure of sampling adequacy + code.appendLine(); + code.appendLine("# KMO(Kaiser-Meyer-Olkin) measure of sampling adequacy"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from factor_analyzer.factor_analyzer import calculate_kmo"); + code.appendLine("_kmo = calculate_kmo(vp_df)"); + code.appendLine("display(Markdown('### KMO measure of sampling adequacy'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic ':_kmo[1]}, index=['KMO measure of sampling adequacy']))"); + + // Bartlett's test of sphericity + code.appendLine(); + code.appendLine("# Bartlett's test of sphericity"); + code.appendLine("from factor_analyzer.factor_analyzer import calculate_bartlett_sphericity"); + code.appendLine("_bartlett = calculate_bartlett_sphericity(vp_df)"); + code.appendLine("display(Markdown('### Bartlett\\'s test of sphericity'))"); + code.appendLine("display(pd.DataFrame(data={'Chi-square statistic':_bartlett[0],'p-value':_bartlett[1]}, index=['Bartlett test of sphericity']))"); + + // Initial of Factor Analysis + code.appendLine(); + code.appendLine("# Initial"); + code.appendLine("from factor_analyzer import FactorAnalyzer"); + code.appendFormatLine("_fa1 = FactorAnalyzer(n_factors=vp_df.shape[1], rotation=None, method='{0}', impute='{1}')", method, impute); + code.appendLine("_fa1.fit(vp_df)"); + + // Number of Factor + code.appendLine(); + code.appendLine("# Number of Factor"); + if (extract === 'eigenvalue') { + code.appendFormatLine("_nof = (_fa1.get_eigenvalues()[0] > {0}).sum()", eigenvalue); + } else if (extract === 'factor') { + code.appendFormatLine("_nof = {0}", factor); + } + + // Unrotated + code.appendLine(); + code.appendLine("# Un-rotated"); + code.appendFormatLine("_fa2 = FactorAnalyzer(n_factors=_nof, rotation=None, method='{0}', impute='{1}')", method, impute); + code.appendLine("_fa2.fit(vp_df)"); + + // Rotated + code.appendLine(); + code.appendLine("# Rotated"); + code.appendFormatLine("_fa3 = FactorAnalyzer(n_factors=_nof, rotation={0}, method='{1}', impute='{2}')", rotation, method, impute); + code.append("_fa3.fit(vp_df)"); + + // Display option : Correlation Matrix + if (corrMatrix === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Correlation matrix"); + code.appendLine("display(Markdown('### Correlation matrix'))"); + code.append("display(pd.DataFrame(data= _fa1.corr_ , index=vp_df.columns, columns=vp_df.columns).round(2))"); + } + + // Display option : Scree plot + if (screePlot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Scree plot"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" plt.plot(_fa1.get_factor_variance()[1], 'o-')"); + code.appendLine(" plt.title('Scree Plot')"); + code.appendLine(" plt.xlabel('Factors')"); + code.appendLine(" plt.ylabel('Eigenvalue')"); + code.append(" plt.show()"); + } + + // Communalities + code.appendLine(); + code.appendLine(); + code.appendLine("# Communalities"); + code.appendLine("display(Markdown('### Communalities'))"); + code.appendLine("display(pd.DataFrame(data={'Initial':_fa1.get_communalities(),'Extraction':_fa2.get_communalities()},index=vp_df.columns).round(3))"); + + // Total variance explained + code.appendLine(); + code.appendLine("# Total variance explained"); + code.appendLine("# Initial Eigenvalues"); + code.appendLine("_ss1 = pd.DataFrame(data=_fa1.get_factor_variance(),"); + code.appendLine(" index=[['Initial Eigenvalues' for i in range(3)],['Total','% of variance','Cumulative %']]).T"); + code.appendLine("# Extraction sums of squared loadings"); + code.appendLine("_ss2 = pd.DataFrame(data=_fa1.get_factor_variance(),"); + code.appendLine(" index=[['Extraction sums of squared loadings' for i in range(3)],['Total','% of variance','Cumulative %']]).T[:3]"); + code.appendLine("# Rotation sums of squared loadings"); + code.appendLine("_ss3 = pd.DataFrame(data=_fa3.get_factor_variance(),"); + code.appendLine(" index=[['Rotation sums of squared loadings' for i in range(3)],['Total','% of variance','Cumulative %']]).T"); + code.appendLine(" "); + code.appendLine("display(Markdown('### Total variance explained'))"); + code.appendLine("display(pd.concat([_ss1,_ss2,_ss3], axis=1).round(3))"); + + // Factor Matrix + code.appendLine(); + code.appendLine("# Factor matrix"); + code.appendLine("display(Markdown('### Factor matrix'))"); + code.appendLine("display(pd.DataFrame(data=_fa2.loadings_,index=vp_df.columns,"); + code.appendLine(" columns=list(range(_nof))).round(3))"); + + // Rotated Factor Matrix + code.appendLine(); + code.appendLine("# Rotated factor matrix"); + code.appendLine("display(Markdown('### Rotated factor matrix'))"); + code.appendLine("display(pd.DataFrame(data=_fa3.loadings_,index=vp_df.columns,"); + code.append(" columns=list(range(_nof))).round(3))"); + + codeList.push(code.toString()); + + return codeList; + } + + } + + return FactorAnalysis; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/LogisticRegression.js b/visualpython/js/m_stats/LogisticRegression.js new file mode 100644 index 00000000..318f63f6 --- /dev/null +++ b/visualpython/js/m_stats/LogisticRegression.js @@ -0,0 +1,193 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : LogisticRegression.js + * Author : Black Logic + * Note : Correlation Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 06. 02 + * Change Date : + */ + +//============================================================================ +// [CLASS] LogisticRegression +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/logisticRegression.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * LogisticRegression + */ + class LogisticRegression extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://www.statsmodels.org/stable/api.html'; + + this.state = { + data: '', + dependent: '', + encoding: true, + independent: [], + showOdds: true, + multiCollinearity: true, + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#dependent')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // data change + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // dependent change + $(document).on('change', this.wrapSelector('#dependent'), function() { + let depVal = $(this).val(); + that.columnSelector = new MultiSelector(that.wrapSelector('#independent'), + { + mode: 'columns', parent: that.state.data, showDescription: false, + excludeList: [ depVal ] + } + ); + }); + } + + handleVariableChange(data) { + this.state.data = data; + this.state.dependent = ''; + this.state.independent = []; + // render column + com_generator.vp_bindColumnSource(this, 'data', ['dependent'], 'select', false, false); + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#independent'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', ['dependent'], 'select', false, false); + } + + let excludeList = []; + if (this.state.dependent !== '') { + excludeList = [ this.state.dependent ]; + } + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#independent'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.independent, excludeList: excludeList, showDescription: false }); + } + + generateCode() { + let { data, dependent, encoding, independent, showOdds, multiCollinearity } = this.state; + let codeList = []; + let code = new com_String(); + + let dependentValue = $(this.wrapSelector('#dependent option:selected')).text(); + let independentMulti = this.columnSelector.getDataList(); + this.state.independent = independentMulti; + + // data declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + if (encoding === true) { + code.appendFormatLine("vp_df['{0}'+'_EL'] = pd.Categorical(vp_df[{1}]).codes", dependentValue, dependent); + dependentValue = dependentValue + '_EL'; + } + code.appendLine(); + code.appendLine("# Logistic regression"); + code.appendLine("from IPython.display import display"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.logit('{0} ~ {1}', vp_df)", dependentValue, independentMulti.map(x=>x.name).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.appendLine("print(_result.summary())"); + code.appendLine(""); + code.appendLine("# Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + if (showOdds === true) { + code.appendLine("_dfr['Odds'] = np.exp(_result.params)"); + code.appendLine("_dfr['Lower(Odds)'] = np.exp(_result.conf_int()[0])"); + code.appendLine("_dfr['Upper(Odds)'] = np.exp(_result.conf_int()[1])"); + } + if (multiCollinearity === true) { + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + } + code.append("display(_dfr)"); + + return code.toString(); + } + + } + + return LogisticRegression; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/NormTest.js b/visualpython/js/m_stats/NormTest.js index ebb9dc74..0669f6cf 100644 --- a/visualpython/js/m_stats/NormTest.js +++ b/visualpython/js/m_stats/NormTest.js @@ -3,7 +3,7 @@ * Description : GUI-based Python code generator * File Name : NormTest.js * Author : Black Logic - * Note : Norm test + * Note : Normality test * License : GNU GPLv3 with Visual Python special exception * Date : 2023. 05. 09 * Change Date : @@ -13,12 +13,15 @@ // [CLASS] NormTest //============================================================================ define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/normTest.html'), 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/component/PopupComponent', - 'vp_base/js/com/component/DataSelector' -], function(com_util, com_Const, com_String, PopupComponent, DataSelector) { + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/m_apps/Subset' +], function(nmHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, Subset) { /** * NormTest @@ -27,36 +30,216 @@ define([ _init() { super._init(); /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'shapiro-wilk', + data0: '', + alterHypo: 'two-sided', + histogram: false, + boxplot: false, + qqplot: true, + ...this.state + }; + + this.subsetEditor = {}; } _bindEvent() { super._bindEvent(); /** Implement binding events */ var that = this; + + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + }); + + $(this.wrapSelector('#data0')).on('change', function() { + if (that.state.data0type === 'DataFrame') { + // DataFrame + that.state.var0 = ''; + $(that.wrapSelector('#var0')).prop('disabled', false); + com_generator.vp_bindColumnSource(that, 'data0', ['var0'], 'select', false, false); + } else { + // Series + that.state.var0 = ''; + $(that.wrapSelector('#var0')).html(''); + $(that.wrapSelector('#var0')).prop('disabled', true); + } + }); } templateForBody() { - /** Implement generating template */ - return `This is sample. - `; + let page = $(nmHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data0', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame', 'Series'], withPopup: false, + finish: function(data, type) { + that.state.data0 = data; + that.state.data0type = type; + that.state.var0 = ''; + $(that.wrapSelector('#data0')).trigger('change'); + }, + select: function(data, type) { + that.state.data0 = data; + that.state.data0type = type; + that.state.var0 = ''; + $(that.wrapSelector('#data0')).trigger('change'); + } + }); + $(page).find('#data0').replaceWith(dataSelector.toTagString()); + + return page; } render() { super.render(); + let that = this; - let dataSelector = new DataSelector({ - type: 'data', - pageThis: this, - id: 'sample', - finish: function() { - ; - } - }); - $(this.wrapSelector('#sample')).replaceWith(dataSelector.toTagString()); + // render Subset + this.subsetEditor['data0'] = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data0'), + pageThis: this, + finish: function(code, state) { + that.state.data0 = code; + $(that.wrapSelector('#data0')).val(code); + that.state.data0type = state.returnType; + $(that.wrapSelector('#data0')).trigger('change'); + } + }); + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); } generateCode() { - return "print('sample code')"; + let { testType, data0, data0type, var0, alterHypo, histogram, boxplot, qqplot } = this.state; + let codeList = []; + let code = new com_String(); + + // test type label + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + code.appendFormatLine('# {0}', testTypeLabel); + + // variable declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data0); + + let dataVar = 'vp_df'; + if (var0 !== '') { + dataVar += com_util.formatString("[{0}]", var0); + } + + switch (testType) { + case 'shapiro-wilk': + // 1. Shapiro-wilk test + code.appendLine(); + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.shapiro({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},index=['Normality test (Shapiro-Wilk)']))"); + break; + case 'anderson-darling': + // 1. Anderson-Darling test + code.appendLine(); + code.appendLine("# Normality test (Anderson-Darling)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.anderson({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (Anderson-Darling)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':[_res.statistic],'Critical values':[_res.critical_values],"); + code.appendLine(" 'Significance level(%)':[_res.significance_level]},"); + code.append(" index=['Normality test (Anderson-Darling)']))"); + break; + case 'kolmogorov-smirnov': + // 1. Kolmogorov-Smirnov test + code.appendLine(); + code.appendLine("# Normality test (Kolmogorov-Smirnov)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.kstest({0}, 'norm', alternative='{1}')", dataVar, alterHypo); + code.appendLine("display(Markdown('### Normality test (Kolmogorov-Smirnov)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Normality test (Kolmogorov-Smirnov)']))"); + break; + case 'dagostino-pearson': + // 1. D Agostino and Pearson test + code.appendLine(); + code.appendLine("# Normality test (D Agostino and Pearson)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.normaltest({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (D Agostino and Pearson)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Normality test (D Agostino and Pearson)']))"); + break; + case 'jarque-bera': + // 1. Jarque-Bera test + code.appendLine(); + code.appendLine("# Normality test (Jarque-Bera)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.jarque_bera({0})", dataVar); + code.appendLine("display(Markdown('### Normality test (Jarque-Bera)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.append(" index=['Normality test (Jarque-Bera)']))"); + break; + } + + // Display option + if (histogram === true || boxplot === true || qqplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Charts"); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.append(" warnings.simplefilter(action='ignore', category=Warning)"); + let displayNum = 1; + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendFormatLine(" sns.histplot({0}, stat='density', kde=True)", dataVar); + code.append(" plt.title('Histogram')"); + } + if (boxplot === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendFormatLine(" sns.boxplot(y={0})", dataVar); + code.append(" plt.title('Boxplot')"); + } + + if (qqplot === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum); + code.appendFormatLine(" stats.probplot({0}, plot=plt)", dataVar); + code.append(" plt.title('Q-Q Plot')"); + } + code.appendLine(); + code.appendLine(); + code.appendLine(" plt.tight_layout()"); + code.append(" plt.show()"); + } + + return code.toString(); } } diff --git a/visualpython/js/m_stats/ProbDist.js b/visualpython/js/m_stats/ProbDist.js index e89b8c9c..245cd9f6 100644 --- a/visualpython/js/m_stats/ProbDist.js +++ b/visualpython/js/m_stats/ProbDist.js @@ -13,12 +13,16 @@ // [CLASS] ProbDist //============================================================================ define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/probDist.html'), + __VP_CSS_LOADER__('vp_base/css/m_stats/probDist'), 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/data/m_stats/statsLibrary', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/DataSelector' -], function(com_util, com_Const, com_String, PopupComponent, DataSelector) { +], function(pdHTML, pdCss, com_util, com_Const, com_String, com_generator, STATS_LIBRARIES, PopupComponent, DataSelector) { /** * ProbDist @@ -27,36 +31,407 @@ define([ _init() { super._init(); /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd', 'plt']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + this.config.helpview = true; + + this.state = { + distType: 'normal', + userOption: '', + action: 'random-number', + // random-number + size: 10000, + randomState: '', + allocateTo: 'samples', + sampledDist: true, + // distribution-plot + probDensityFunc: true, + probMassFunc: true, + cumDistFunc: true, + // stats-to-pvalue + stats: '', + pAlter: 'two-sided', + // pvalue-to-stats + pvalue: '', + statsAlter: 'two-sided', + ...this.state + }; + + this.distList = [ + { + label: 'Discrete probability distribution', + child: ['bernoulli', 'binomial', 'multinomial'] + }, + { + label: 'Continuous probability distribution', + child: ['uniform','normal','beta','gamma','studentst','chi2','f','dirichlet','multivariate_normal'] + } + ]; } _bindEvent() { super._bindEvent(); /** Implement binding events */ var that = this; + + $(this.wrapSelector('#distType')).on('change', function() { + let distType = $(this).val(); + that.state.distType = distType; + $(that.wrapSelector('.vp-pd-dist-option-box')).html(that.templateForOption(distType)); + + $(that.wrapSelector('.vp-pd-display-option')).hide(); + // show/hide display option + if (that.distList[0].child.includes(distType)) { + // discrete option + $(that.wrapSelector('.vp-pd-display-option.dist')).show(); + + // set size to 1000 + $(that.wrapSelector('#size')).val(1000); + that.state.size = 1000; + + // hide distribution plot for multinomial + if (distType === 'multinomial') { + $(that.wrapSelector('.vp-pd-display-option.dist-plot')).hide(); + // hide other actions + if (that.state.action !== 'random-number') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } + } else { + // hide continuous action + if (that.state.action === 'stats-to-pvalue' || that.state.action === 'pvalue-to-stats') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } + } + } else { + // continuous option + $(that.wrapSelector('.vp-pd-display-option.cont')).show(); + + // set size to 10000 + $(that.wrapSelector('#size')).val(10000); + that.state.size = 10000; + } + + // show install button + let thisDistObj = STATS_LIBRARIES[distType]; + if (thisDistObj.install != undefined) { + $(that.wrapSelector('#vp_installLibrary')).show(); + } else { + $(that.wrapSelector('#vp_installLibrary')).hide(); + } + + // set help content + that.setHelpContent(thisDistObj.help); + }); + + $(this.wrapSelector('#action')).on('change', function() { + let action = $(this).val(); + that.state.action = action; + + $(that.wrapSelector('.vp-pd-action-box')).hide(); + $(that.wrapSelector('.vp-pd-action-box.' + action)).show(); + + $(that.wrapSelector('.vp-pd-display-option')).hide(); + // show/hide display option + if (that.distList[0].child.includes(that.state.distType)) { + // discrete option + $(that.wrapSelector('.vp-pd-display-option.dist')).show(); + } else { + // continuous option + $(that.wrapSelector('.vp-pd-display-option.cont')).show(); + } + }); } templateForBody() { - /** Implement generating template */ - return `This is sample. - `; + let page = $(pdHTML); + let that = this; + + //================================================================ + // Distribution type creation + //================================================================ + // dist types + let distTypeTag = new com_String(); + this.distList.forEach(distObj => { + let { label, child } = distObj; + let distOptionTag = new com_String(); + child && child.forEach(opt => { + let optConfig = STATS_LIBRARIES[opt]; + let selectedFlag = ''; + if (opt == that.state.distType) { + selectedFlag = 'selected'; + } + distOptionTag.appendFormatLine('', + opt, selectedFlag, optConfig.name); + }) + distTypeTag.appendFormatLine('{1}', + label, distOptionTag.toString()); + }); + $(page).find('#distType').html(distTypeTag.toString()); + + // render option page + $(page).find('.vp-pd-dist-option-box').html(this.templateForOption(this.state.distType)); + + // control display option + $(this.wrapSelector('.vp-pd-display-option')).hide(); + // show/hide display option + if (this.distList[0].child.includes(this.state.distType)) { + // discrete option + $(this.wrapSelector('.vp-pd-display-option.dist')).show(); + } else { + // continuous option + $(this.wrapSelector('.vp-pd-display-option.cont')).show(); + } + + return page; + } + + templateForOption(distType) { + let config = STATS_LIBRARIES[distType]; + let state = this.state; + + let optBox = new com_String(); + // render tag + config.options.forEach(opt => { + optBox.appendFormatLine('{3}' + , opt.name, (opt.required===true?'vp-orange-text':''), opt.name, com_util.optionToLabel(opt.name)); + let content = com_generator.renderContent(this, opt.component[0], opt, state); + optBox.appendLine(content[0].outerHTML); + }); + // render user option + optBox.appendFormatLine('{1}', 'userOption', 'User option'); + optBox.appendFormatLine('', + 'userOption', 'key=value, ...', this.state.userOption); + return optBox.toString(); } render() { super.render(); - let dataSelector = new DataSelector({ + let allocateSelector = new DataSelector({ type: 'data', pageThis: this, - id: 'sample', + id: 'allocatedTo', + classes: 'vp-input vp-state', + placeholder: '_res', finish: function() { ; } }); - $(this.wrapSelector('#sample')).replaceWith(dataSelector.toTagString()); + $(this.wrapSelector('#allocatedTo')).replaceWith(allocateSelector.toTagString()); + + this.setHelpContent(STATS_LIBRARIES[this.state.distType].help, true, 'import scipy.stats as _vp_stats'); } generateCode() { - return "print('sample code')"; + this.config.checkModules = ['pd']; + let { + distType, userOption, action, + size, randomState, allocateTo, sampledDist, + probDensityFunc, probMassFunc, cumDistFunc, + stats, pAlter, + pvalue, statsAlter + } = this.state; + + let codeList = []; + let code = new com_String(); + /** + * Model Creation + */ + let config = STATS_LIBRARIES[distType]; + let label = config.name; + code.appendLine(config.import); + code.appendLine(); + + // model code + let modelCode = config.code; + modelCode = com_generator.vp_codeGenerator(this, config, this.state, (userOption != ''? ', ' + userOption : '')); + code.append(modelCode); + + switch (action) { + case 'random-number': + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Generate random numbers ({0})", label); + code.appendLine("from IPython.display import display"); + code.appendFormat('{0} = _rv.rvs(size={1}', allocateTo, size); + if (randomState !== '') { + code.appendFormat(", random_state={0}", randomState); + } + code.appendLine(')'); + code.appendFormat("display({0})", allocateTo); + + if (sampledDist === true) { + this.addCheckModules('plt'); + this.addCheckModules('sns'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Sample distribution ({0})", label); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + if (distType === 'multinomial') { + // code.appendFormatLine(" plt.boxplot(x={0})", allocateTo); + code.appendFormatLine(" for i in range(0, {0}.shape[1]):", allocateTo); + code.appendLine(" plt.subplot(2, 2, i+1)"); + code.appendLine(" plt.title('$x$=' + str(i))"); + code.appendFormatLine(" sns.countplot(x=[ x[i] for x in {0} ])", allocateTo); + code.appendLine(" plt.suptitle('Generate random numbers: Multinomial')"); + code.appendLine(" plt.tight_layout()"); + code.appendLine(" plt.show()"); + } else { + if (this.distList[0].child.includes(distType)) { + code.appendFormatLine(" sns.countplot(x={0})", allocateTo); + } else { + code.appendFormatLine(" sns.histplot({0}, stat='density', kde=True)", allocateTo); + } + code.appendFormatLine(" plt.title('Generate random numbers: {0}')", label.replace("'", "\\'")); + code.appendLine(" plt.xlabel('$x$')"); + code.append(" plt.show()"); + } + } + break; + case 'distribution-plot': + if (this.distList[0].child.includes(distType)) { + if (probMassFunc === true) { + this.addCheckModules('np'); + this.addCheckModules('plt'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Probability mass function ({0})", label); + if (distType === 'bernoulli') { + code.appendLine("plt.bar([0,1], _rv.pmf([0,1]))"); + code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); + code.appendLine("plt.xlim(-1, 2)"); + code.appendLine("plt.ylim(0, 1)"); + code.appendLine("plt.xticks([0, 1], ['x=0', 'x=1'])"); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } else if (distType === 'binomial') { + var { n=10 } = this.state; + code.appendFormatLine("plt.bar(range(0,{0}), _rv.pmf(range(0,{1})))", n, n); + code.appendFormatLine("plt.title('Probability mass function: {0}')", label.replace("'", "\\'")); + code.appendFormatLine("plt.xlim(-1, {0})", n); + code.appendFormatLine("plt.xticks(range(0, {0}), ['x='+str(i) for i in range(0, {1})])", n, n); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } else if (distType === 'multinomial') { + code.appendFormatLine("for i in range(0, {0}.shape[1]):", allocateTo); + code.appendLine(" plt.subplot(2, 2, i+1)"); + code.appendLine(" plt.title('$x$=' + str(i))"); + code.appendFormatLine(" sns.countplot(x=[ x[i] for x in {0} ])", allocateTo); + code.appendLine("plt.suptitle('Probability mass function: Multinomial')"); + code.appendLine("plt.tight_layout()"); + code.append("plt.show()"); + } + } + } else { + let start = -5; + let end = 5; + switch (distType) { + case 'normal': + case 'studentst': + case 'multivariate_normal': + start = -5; + end = 5; + break; + case 'uniform': + case 'beta': + case 'dirichlet': + start = 0; + end = 1; + break; + case 'gamma': + case 'chi2': + start = 0; + end = 30; + break; + case 'f': + start = 0; + end = 10; + break; + } + + if (probDensityFunc === true || cumDistFunc === true) { + code.appendLine(); + code.appendFormat("x = np.linspace({0}, {1}, 100)", start, end); + if (probDensityFunc === true) { + this.addCheckModules('np'); + this.addCheckModules('plt'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Probability density function ({0})", label); + code.appendLine("plt.plot(x, _rv.pdf(x))"); + code.appendFormatLine("plt.title('Probability density function: {0}')", label.replace("'", "\\'")); + code.appendLine("plt.xlabel('$x$')"); + code.appendLine("plt.ylabel('$p(x)$')"); + code.append("plt.show()"); + } + if (cumDistFunc === true) { + this.addCheckModules('np'); + this.addCheckModules('plt'); + + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Cumulative distribution function ({0})", label); + code.appendLine("import warnings"); + code.appendLine("with warnings.catch_warnings():"); + code.appendFormatLine(" _x = np.linspace({0}, {1}, 100)", start, end); + code.appendLine(" plt.plot(_x, _rv.cdf(_x))"); + code.appendLine(); + code.appendFormatLine(" plt.title('Cumulative distribution function: {0}')", label.replace("'", "\\'")); + code.appendLine(" plt.xlabel('$x$')"); + code.appendLine(" plt.ylabel('$F(x)$')"); + code.append(" plt.show()"); + } + } + } + break; + case 'stats-to-pvalue': + if (pAlter === 'one-sided') { + // one-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Proportional values"); + code.appendFormatLine("p_value = _rv.sf(abs({0}))", stats); + code.append("p_value"); + } else { + // two-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Proportional values"); + code.appendFormatLine("p_value = _rv.sf(abs({0}))*2", stats); + code.append("p_value"); + } + break; + case 'pvalue-to-stats': + if (statsAlter === 'one-sided') { + // one-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistic"); + code.appendFormatLine("statistic = _rv.isf({0})", pvalue); + code.append("statistic"); + } else { + // two-sided + code.appendLine(); + code.appendLine(); + code.appendLine("# Statistic"); + code.appendFormatLine("statistic = _rv.isf({0}/2)", pvalue); + code.append("statistic"); + } + break; + } + codeList.push(code.toString()); + + return codeList; } } diff --git a/visualpython/js/m_stats/Regression.js b/visualpython/js/m_stats/Regression.js new file mode 100644 index 00000000..a38c811b --- /dev/null +++ b/visualpython/js/m_stats/Regression.js @@ -0,0 +1,789 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : Regression.js + * Author : Black Logic + * Note : Equal Variance test + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 09 + * Change Date : + */ + +//============================================================================ +// [CLASS] EqualVarTest +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/regression.html'), + __VP_CSS_LOADER__('vp_base/css/m_stats/regression'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, rgCss, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * Regression + */ + class Regression extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://www.statsmodels.org/stable/api.html'; + + this.state = { + testType: 'simple', + // Data selection + data: '', + dataType: '', + dependent: '', + independent: '', + independentMulti: [], + moderated: '', + mediated: '', + // options + categorical: [], + method: 'enter', + meanCentering: true, + sobelTest: true, + // Multi-collinearity + multiCollinearity: true, + // Residual option + statistics: false, + normTest: true, + histogram: true, + scatterplot: true, + rmse: false, + ...this.state + }; + + this.colBindList = ['dependent', 'independent', 'moderated', 'mediated']; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#dependent')); + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + // change test type + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + + let excludeList = []; + if (that.state.testType === 'multiple' + || that.state.testType === 'hierarchical' + || that.state.testType === 'dummy') { + let depVal = $(that.wrapSelector('#dependent')).val(); + excludeList = [ depVal ]; + } + + // render variable selector + that.columnSelector = new MultiSelector(that.wrapSelector('#independentBox'), + { + mode: 'columns', parent: that.state.data, showDescription: false, + excludeList: excludeList, + change: function(type, list) { + that._handleMultiColumnChange(type, list); + } + }); + }); + + // data change + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // dependent change + $(document).on('change', this.wrapSelector('#dependent'), function() { + let depVal = $(this).val(); + if (that.state.testType === 'multiple' + || that.state.testType === 'hierarchical' + || that.state.testType === 'dummy') { + that.columnSelector = new MultiSelector(that.wrapSelector('#independentBox'), + { + mode: 'columns', parent: that.state.data, showDescription: false, + excludeList: [ depVal ], + change: function(type, list) { + that._handleMultiColumnChange(type, list); + } + } + ); + } + }); + } + + handleVariableChange(data) { + this.state.data = data; + this.state.independentMulti = []; + let that = this; + // bind column sources + if (this.state.dataType === 'DataFrame') { + // DataFrame + this.colBindList && this.colBindList.forEach(id => { + that.state[id] = ''; + $(that.wrapSelector('#' + id)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(this, 'data', this.colBindList, 'select', false, false); + } else { + // Others + this.colBindList && this.colBindList.forEach(id => { + that.state[id] = ''; + $(that.wrapSelector('#' + id)).html(''); + $(that.wrapSelector('#' + id)).prop('disabled', true); + }); + } + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#independentBox'), + { + mode: 'columns', parent: data, showDescription: false, + change: function(type, list) { + that._handleMultiColumnChange(type, list); + } + } + ); + } + + _handleMultiColumnChange(type, list) { + let $newCateBox = $('

'); + let that = this; + list && list.forEach(item => { + let checkedStr = 'checked'; + if ($(that.wrapSelector('.vp-categorical-box input[data-name="' + item.name + '"]')).length > 0) { + $(that.wrapSelector('.vp-categorical-box input[data-name="' + item.name + '"]')).prop('checked')?'checked':''; + } + $newCateBox.append(` + ${item.name} + `); + }); + $(this.wrapSelector('.vp-categorical-box')).replaceWith($newCateBox); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + // depend on test type + $(page).find('.vp-st-option').hide(); + $(page).find('.vp-st-option.' + this.state.testType).show(); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: this.state.data, + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + $(that.wrapSelector('#data')).val(code); + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + let excludeList = []; + if (this.state.testType === 'multiple' + || this.state.testType === 'hierarchical' + || this.state.testType === 'dummy') { + if (this.state.dependent !== '') { + excludeList = [ this.state.dependent ]; + } + } + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#indenpendentBox'), + { + mode: 'columns', parent: this.state.data, + selectedList: this.state.independentMulti.map(x=>x.code), excludeList: excludeList, showDescription: false + } + ); + + // bind column if data exist + if (this.state.data !== '') { + com_generator.vp_bindColumnSource(this, 'data', this.colBindList, 'select', false, false); + } + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + } + + generateCode() { + let { testType, + // Data selection + data, dataType, dependent, independent, independentMulti, moderated, mediated, + // options + method, meanCentering, sobelTest, + // Multi-collinearity + multiCollinearity, + // Residual option + statistics, normTest, histogram, scatterplot, rmse, + } = this.state; + let codeList = []; + let code = new com_String(); + let that = this; + let lastModelNum = 0; + + // Commentary + let testTypeLabel = $(this.wrapSelector('#testType option:selected')).text(); + let methodLabel = $(this.wrapSelector('#method option:selected')).text(); + if (testType === 'multiple') { + code.appendFormatLine("# {0} > Method: {1}", testTypeLabel, methodLabel); + } else { + code.appendFormatLine("# {0}", testTypeLabel); + } + + // data declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + + // data and columns + let dependentValue = $(this.wrapSelector('#dependent option:selected')).text(); + let independentValue = $(this.wrapSelector('#independent option:selected')).text(); + let moderatedValue = $(this.wrapSelector('#moderated option:selected')).text(); + let mediatedValue = $(this.wrapSelector('#mediated option:selected')).text(); + independentMulti = this.columnSelector.getDataList(); + this.state.independentMulti = independentMulti; + + switch (testType) { + case 'simple': + // 1. Simple + code.appendLine(); + code.appendLine("# Simple linear regression"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendLine("# Model - Dependent variable ~ Independent variable"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + // Multi-collinearity statistics + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + break; + case 'multiple': + // 2. Multiple + code.appendLine(); + if (method === 'enter') { + code.appendLine("# Model - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentMulti.map(x => x.name).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + } else if (method === 'stepwise') { + // Inner function : vp_stepwise_select + this.addCheckModules('sm'); + this.addCheckModules('vp_stepwise_select'); + + code.appendFormatLine("_selected_stepwise = vp_stepwise_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); + code.appendLine(""); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_stepwise[0]]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Stepwised variable"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_stepwise]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Stepwised variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 2; + } else if (method === 'backward') { + // Inner function : vp_backward_select + this.addCheckModules('sm'); + this.addCheckModules('vp_backward_select'); + + code.appendFormatLine("_selected_backward = vp_backward_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); + code.appendLine(); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[[{1}]]))", dependent, independentMulti.map(x => x.code).join(',')); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Backward variable"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_backward]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Backward variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 2; + } else if (method === 'forward') { + // Inner function : vp_forward_select + this.addCheckModules('sm'); + this.addCheckModules('vp_forward_select'); + + code.appendFormatLine("_selected_forward = vp_forward_select(vp_df[[{0}]], vp_df[{1}])", independentMulti.map(x => x.code).join(','), dependent); + code.appendLine(); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_forward[0]]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Forward variable"); + code.appendLine("import statsmodels.api as sm"); + code.appendFormatLine("_model = sm.OLS(vp_df[{0}], sm.add_constant(vp_df[_selected_forward]))", dependent); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Forward variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 2; + } + break; + case 'hierarchical': + // 3. Hierarchical + for (let i = 0; i < independentMulti.length; i++) { + if (i === 0) { + code.appendLine(); + } else { + code.appendLine(); + code.appendLine(); + } + code.appendFormatLine("# Model {0} - Hierarchical linear regression", (i + 1)); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentMulti.slice(0, i + 1).map(x => x.name).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.appendFormatLine("display(Markdown('### Model {0} - Dependent variable ~ Independent variable'))", (i + 1)); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# Model {0} - Multi-collinearity statistics", (i + 1)); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + } + lastModelNum = independentMulti.length; + break; + case 'moderated': + // 4. Moderated + // Mean centering + if (meanCentering === true) { + code.appendLine(); + code.appendLine("# Mean Centering "); + independentValue = com_util.formatString("{0}_MC", independentValue); + moderatedValue = com_util.formatString("{0}_MC", moderatedValue); + code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", independentValue, independent, independent); + code.appendFormatLine("vp_df['{0}'] = vp_df[{1}] - vp_df[{2}].mean(numeric_only=True)", moderatedValue, moderated, moderated); + } + // Model 1 to 3 + code.appendLine(); + code.appendLine("# Model 1 - Dependent variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Independent variable + Moderated variable"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1} + {2}', vp_df)", dependentValue, independentValue, moderatedValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Independent variable + Moderated variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Dependent variable ~ Independent variable + Moderated variable +Independent:Moderated"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1} + {2} + {3}:{4}', vp_df)", dependentValue, independentValue, moderatedValue, independentValue, moderatedValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 3 - Dependent variable ~ Independent variable + Moderated variable +Independent:Moderated'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + // set last model number + lastModelNum = 3; + break; + case 'mediated': + // 5. Mediated + if (sobelTest === true) { + this.addCheckModules('stats'); + this.addCheckModules('vp_sobel'); + } + code.appendLine(); + code.appendLine("# Model 1 - Mediated variable ~ Independent variable"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", mediatedValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 1 - Mediated variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + if (sobelTest === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 1 - Sobel test"); + code.appendFormatLine("_sobel_M1 = _result.params[{0}]", independent); + code.appendFormat("_sobel_M1se = _result.bse[{0}]", independent); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Dependent variable ~ Independent variable"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)", dependentValue, independentValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 2 - Dependent variable ~ Independent variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 2 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Dependent variable ~ Independent variable + Mediated variable"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1} + {2}', vp_df)", dependentValue, independentValue, mediatedValue); + code.appendLine("_result = _model.fit()"); + code.appendLine("display(Markdown('### Model 3 - Dependent variable ~ Independent variable + Mediated variable'))"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Multi-collinearity statistics"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + if (sobelTest === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Model 3 - Sobel test"); + code.appendFormatLine("_sobel_M3 = _result.params[{0}]", mediated); + code.appendFormatLine("_sobel_M3se = _result.bse[{0}]", mediated); + code.appendLine(); + code.appendLine("# Mediated linear regression: Sobel test"); + code.appendLine("from scipy import stats"); + code.appendLine("_res = vp_sobel(_sobel_M1, _sobel_M3, _sobel_M1se, _sobel_M3se)"); + code.appendLine("display(Markdown('### Sobel test'))"); + code.append("display(pd.DataFrame(data={'Sobel Z-score':_res[0],'p-value':_res[2]},index=['Sobel test']))"); + } + // set last model number + lastModelNum = 3; + break; + case 'dummy': + // 6. Dummy variable + code.appendLine(); + code.appendLine("# Dummy variable linear regression"); + code.appendLine("import statsmodels.formula.api as smf"); + code.appendFormatLine("_model = smf.ols('{0} ~ {1}', vp_df)" + , dependentValue, independentMulti.map(item => { + let checked = $(that.wrapSelector('.vp-categorical-box input[data-name="' + item.name + '"]')).prop('checked'); + if (checked === true) { + return 'C(' + item.name + ')'; + } else { + return item.name; + } + }).join(' + ')); + code.appendLine("_result = _model.fit()"); + code.append("print(_result.summary())"); + if (multiCollinearity === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Multi-collinearity statistics"); + code.appendLine("from IPython.display import display"); + code.appendLine("from statsmodels.stats.outliers_influence import variance_inflation_factor"); + code.appendLine("_dfr = pd.DataFrame(_result.summary().tables[1].data[1:],columns=_result.summary().tables[1].data[0]).set_index('')"); + code.appendLine("for i, col in enumerate(_model.exog_names[1:]):"); + code.appendLine(" _vif = variance_inflation_factor(_model.exog, i+1)"); + code.appendLine(" _dfr.loc[col,'Tolerance'] = 1/_vif"); + code.appendLine(" _dfr.loc[col,'VIF'] = _vif"); + code.append("display(_dfr)"); + } + break; + } + + // Residual option + if (statistics === true || normTest === true || histogram === true || scatterplot === true || rmse === true) { + let residualTitle = 'Residual' + if (lastModelNum > 0) { + residualTitle += ' - Model ' + lastModelNum; + } + code.appendLine(); + code.appendLine(); + code.appendFormatLine("# {0}", residualTitle); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendLine("import statsmodels.api as sm"); + if (testType === 'multiple' && ['stepwise', 'backward', 'forward'].includes(method)) { + code.appendLine("_predict = _result.predict(sm.add_constant(vp_df[_model.exog_names[1:]]))"); + } else { + code.appendLine("_predict = _result.predict(vp_df)"); + } + + code.appendLine("_residual = _result.resid"); + code.appendLine("vp_residual = pd.DataFrame({'predict':_predict,'residual':_residual,"); + code.appendLine(" 'predict_z':stats.zscore(_predict),'residual_z':stats.zscore(_residual)})"); + code.appendFormatLine("display(Markdown('### {0}'))", residualTitle); + code.append("display(vp_residual)"); + + if (statistics === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Residual statistics"); + code.appendLine("display(Markdown('### Residual statistics'))"); + code.appendLine("display(pd.DataFrame(data={'Min':vp_residual.min(),'Max':vp_residual.max(),'Mean':vp_residual.mean(numeric_only=True),"); + code.append(" 'Std. Deviation':vp_residual.std(numeric_only=True),'N':vp_residual.count()}))"); + } + if (normTest === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# Resisual Normality test (Shapiro-Wilk)"); + code.appendLine("_res = stats.shapiro(vp_residual['residual_z'])"); + code.appendLine("display(Markdown('### Residual Normality test (Shapiro-Wilk)'))"); + code.append("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},index=['Resisual Normality test (Shapiro-Wilk)']))"); + } + if (histogram === true || scatterplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("import seaborn as sns"); + code.appendLine("import warnings"); + code.append("with warnings.catch_warnings():"); + let displayNum = 1; + if (histogram === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Residual histogram"); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendLine(" warnings.simplefilter(action='ignore', category=Warning)"); + code.appendLine(" sns.histplot(data=vp_residual, x='residual_z', kde=True)"); + code.appendLine(" plt.title(f'Dependent variable: {_model.endog_names}')"); + code.append(" plt.xlabel('Regression Standardized residual')"); + } + if (scatterplot === true) { + code.appendLine(); + code.appendLine(); + code.appendLine(" # Residual scatterplot"); + code.appendFormatLine(" plt.subplot(2,2,{0})", displayNum++); + code.appendLine(" sns.scatterplot(data=vp_residual, x='predict_z', y='residual_z')"); + code.appendLine(" plt.title(f'Dependent variable: {_model.endog_names}')"); + code.appendLine(" plt.xlabel('Regression Standardized predicted value')"); + code.append(" plt.ylabel('Regression Standardized residual')"); + } + code.appendLine(); + code.appendLine(); + code.appendLine(" plt.tight_layout()"); + code.append(" plt.show()"); + } + if (rmse === true) { + code.appendLine(); + code.appendLine(); + code.appendLine("# RMSE (Root Mean Squared Error)"); + code.appendLine("_rmse = np.sqrt(_result.mse_resid)"); + code.appendLine("display(Markdown('### RMSE (Root Mean Squared Error)'))"); + code.append("display(Markdown(f'RMSE: {_rmse}'))"); + } + } + + codeList.push(code.toString()); + return codeList; + } + + } + + return Regression; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/ReliabAnalysis.js b/visualpython/js/m_stats/ReliabAnalysis.js new file mode 100644 index 00000000..989931b0 --- /dev/null +++ b/visualpython/js/m_stats/ReliabAnalysis.js @@ -0,0 +1,158 @@ +/* + * Project Name : Visual Python + * Description : GUI-based Python code generator + * File Name : ReliabAnalysis.js + * Author : Black Logic + * Note : Reliability Analysis + * License : GNU GPLv3 with Visual Python special exception + * Date : 2023. 05. 24 + * Change Date : + */ + +//============================================================================ +// [CLASS] ReliabAnalysis +//============================================================================ +define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/reliabAnalysis.html'), + 'vp_base/js/com/com_util', + 'vp_base/js/com/com_Const', + 'vp_base/js/com/com_String', + 'vp_base/js/com/component/PopupComponent', + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/MultiSelector', + 'vp_base/js/m_apps/Subset' +], function(eqHTML, com_util, com_Const, com_String, PopupComponent, DataSelector, MultiSelector, Subset) { + + /** + * ReliabAnalysis + */ + class ReliabAnalysis extends PopupComponent { + _init() { + super._init(); + /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd', 'np', 'vp_cronbach_alpha']; + + this.state = { + data: '', + variable: [], + ...this.state + }; + + this.subsetEditor = null; + this.columnSelector = null; + } + + _bindEvent() { + super._bindEvent(); + /** Implement binding events */ + var that = this; + + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + } + + handleVariableChange(data) { + this.state.data = data; + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: data, showDescription: false } + ); + } + + templateForBody() { + let page = $(eqHTML); + let that = this; + + // generate dataselector + let dataSelector = new DataSelector({ + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + $(that.wrapSelector('#data')).trigger('change'); + } + }); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset', category: this.name } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code) { + $(that.wrapSelector('#data')).val(code); + that.handleVariableChange(code); + } + }); + + // render variable selector + this.columnSelector = new MultiSelector(this.wrapSelector('#variable'), + { mode: 'columns', parent: this.state.data, selectedList: this.state.variable.map(x=>x.code), showDescription: false } + ); + } + + generateCode() { + let { data, variable } = this.state; + let codeList = []; + let code = new com_String(); + let that = this; + + // data declaration + code.appendFormat("vp_df = {0}", data); + if (this.columnSelector) { + let columns = this.columnSelector.getDataList(); + this.state.variable = columns; + if (columns.length > 0) { + code.appendFormat("[[{0}]]", columns.map(x => x.code).join(', ')); + } + } + code.appendLine('.dropna().copy()'); + + // Inner function : vp_cronbach_alpha + + // Cronbach alpha + code.appendLine(""); + code.appendLine("# Cronbach alpha"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("display(Markdown('### Cronbach alpha'))"); + code.appendLine("display(pd.DataFrame({'Cronbach alpha':vp_cronbach_alpha(vp_df), 'N':vp_df.shape[1]},index=['Reliability statistics']).round(3))"); + code.appendLine(""); + + // Item-total Statistics + code.appendLine("# Item-Total Statistics"); + code.appendLine("_dfr = pd.DataFrame()"); + code.appendLine("for i, col in enumerate(vp_df.columns):"); + code.appendLine(" _sr = vp_df.drop(col,axis=1).sum(axis=1)"); + code.appendLine(" _df_t = pd.DataFrame(data={'Scale Mean if Item Deleted':_sr.mean(),'Scale Variance if Item Deleted':_sr.var(),"); + code.appendLine(" 'Corrected Item-Total Correlation':_sr.corr(vp_df[col]),"); + code.appendLine(" 'Cronbach Alpha if Item Deleted':vp_cronbach_alpha(vp_df.drop(col,axis=1))}, index=[col])"); + code.appendLine(" _dfr = pd.concat([_dfr, _df_t])"); + code.appendLine("display(Markdown('### Item-Total Statistics'))"); + code.append("display(_dfr.round(3))"); + codeList.push(code.toString()); + + return codeList; + } + + } + + return ReliabAnalysis; +}); \ No newline at end of file diff --git a/visualpython/js/m_stats/StudentstTest.js b/visualpython/js/m_stats/StudentstTest.js index da5cdcbc..d9b5b1a4 100644 --- a/visualpython/js/m_stats/StudentstTest.js +++ b/visualpython/js/m_stats/StudentstTest.js @@ -13,50 +13,395 @@ // [CLASS] StudentstTest //============================================================================ define([ + __VP_TEXT_LOADER__('vp_base/html/m_stats/studentstTest.html'), 'vp_base/js/com/com_util', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_String', + 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/component/PopupComponent', - 'vp_base/js/com/component/DataSelector' -], function(com_util, com_Const, com_String, PopupComponent, DataSelector) { + 'vp_base/js/com/component/DataSelector', + 'vp_base/js/com/component/SuggestInput', + 'vp_base/js/m_apps/Subset' +], function(stHTML, com_util, com_Const, com_String, com_generator, PopupComponent, DataSelector, SuggestInput, Subset) { /** * StudentstTest + * - confidence_interval is available on upper 1.10.0 version of scipy */ class StudentstTest extends PopupComponent { _init() { super._init(); /** Write codes executed before rendering */ + this.config.sizeLevel = 2; + this.config.checkModules = ['pd']; + this.config.docs = 'https://docs.scipy.org/doc/scipy/reference/'; + + this.state = { + testType: 'one-sample', + inputType: 'long-data', + data: '', + dataType: '', + testVariable: '', + testVariable1: '', + testVariable2: '', + groupingVariable: '', + group1: '', + group2: '', + group1_istext: true, + group2_istext: true, + pairedVariable1: '', + pairedVariable2: '', + testValue: '', + alterHypo: 'two-sided', + confInt: '95', + ...this.state + }; + + this.columnBindDict = { + 'one-sample': ['testVariable'], + 'two-sample': ['testVariable', 'testVariable1', 'testVariable2', 'groupingVariable'], + 'paired-sample': ['pairedVariable1', 'pairedVariable2'] + }; + + this.subsetEditor = null; + } + + _unbindEvent() { + super._unbindEvent(); + var that = this; + + $(document).off(this.wrapSelector('#testVariable'), 'change'); + $(document).off(this.wrapSelector('#groupingVariable'), 'change'); } _bindEvent() { super._bindEvent(); /** Implement binding events */ var that = this; + + // change test type + $(this.wrapSelector('#testType')).on('change', function() { + let testType = $(this).val(); + that.state.testType = testType; + + that.handleVariableChange(that.state.data); + + $(that.wrapSelector('.vp-st-option')).hide(); + $(that.wrapSelector('.vp-st-option.' + testType)).show(); + if (testType === 'two-sample') { + $(that.wrapSelector('.vp-st-option.two-sample-' + that.state.inputType)).show(); + } + }); + + // change input type + $(this.wrapSelector('input[name="inputType"]:radio')).on('change', function() { + let inputType = $(this).val(); + that.state.inputType = inputType; + $(that.wrapSelector('.vp-st-option.two-sample-long-data')).hide(); + $(that.wrapSelector('.vp-st-option.two-sample-wide-data')).hide(); + $(that.wrapSelector('.vp-st-option.two-sample-' + inputType)).show(); + }); + + // data change event + $(this.wrapSelector('#data')).on('change', function() { + let data = $(this).val(); + that.handleVariableChange(data); + }); + + // change test variable + $(document).on('change', this.wrapSelector('#testVariable'), function() { + if (that.state.testType === 'one-sample') { + // get mean of data and show on placeholder + $(that.wrapSelector('#testValue')).prop('placeholder', ''); + vpKernel.execute(com_util.formatString("int({0}[{1}].mean())", that.state.data, that.state.testVariable)).then(function(resultObj) { + let { result } = resultObj; + $(that.wrapSelector('#testValue')).prop('placeholder', result); + }); + } + }); + + // change grouping variable + $(document).on('change', this.wrapSelector('#groupingVariable'), function() { + let colCode = $(this).val(); + var colName = $(this).find('option:selected').text(); + var colDtype = $(this).find('option:selected').attr('data-type'); + that.state.groupingVariable = colCode; + // get result and load column list + vpKernel.getColumnCategory(that.state.data, colCode).then(function(resultObj) { + let { result } = resultObj; + $(that.wrapSelector('#group1')).val(''); + $(that.wrapSelector('#group2')).val(''); + that.state.group1 = ''; + that.state.group2 = ''; + that.state.group1_istext = true; + that.state.group2_istext = true; + try { + var category = JSON.parse(result); + // if (category && category.length > 0 && colDtype == 'object') { + // // if it's categorical column and its dtype is object, check 'Text' as default + // category.forEach(obj => { + // let selected1 = obj.value === that.state.group1; + // let selected2 = obj.value === that.state.group2; + // $(that.wrapSelector('#group1')).append(``); + // $(that.wrapSelector('#group2')).append(``); + // }); + // } + var groupSuggest1 = new SuggestInput(); + groupSuggest1.setComponentID('group1'); + groupSuggest1.addClass('vp-input vp-state'); + groupSuggest1.setSuggestList(function() { return category; }); + groupSuggest1.setNormalFilter(true); + groupSuggest1.setPlaceholder('Select value'); + $(that.wrapSelector('#group1')).replaceWith(groupSuggest1.toTagString()); + var groupSuggest2 = new SuggestInput(); + groupSuggest2.setComponentID('group2'); + groupSuggest2.addClass('vp-input vp-state'); + groupSuggest2.setSuggestList(function() { return category; }); + groupSuggest2.setNormalFilter(true); + groupSuggest2.setPlaceholder('Select value'); + $(that.wrapSelector('#group2')).replaceWith(groupSuggest2.toTagString()); + + if (category && category.length > 0) { + that.state.group1 = category[0].value; + that.state.group2 = category[0].value; + } + + if (colDtype == 'object') { + // check as default + $(that.wrapSelector('#group1_istext')).prop('checked', true); + $(that.wrapSelector('#group2_istext')).prop('checked', true); + that.state.group1_istext = true; + that.state.group2_istext = true; + } else { + $(that.wrapSelector('#group1_istext')).prop('checked', false); + $(that.wrapSelector('#group2_istext')).prop('checked', false); + that.state.group1_istext = false; + that.state.group2_istext = false; + } + } catch { + $(that.wrapSelector('#group1')).val(''); + $(that.wrapSelector('#group2')).val(''); + } + }); + }); } - templateForBody() { - /** Implement generating template */ - return `This is sample. - `; + handleVariableChange(data) { + let that = this; + this.state.data = data; + let columnBindList = this.columnBindDict[this.state.testType]; + if (this.state.dataType === 'DataFrame') { + // DataFrame + columnBindList.forEach(col => { + $(that.wrapSelector('#' + col)).prop('disabled', false); + }); + com_generator.vp_bindColumnSource(that, 'data', columnBindList, 'select', false, false); + } else { + // Series + columnBindList.forEach(col => { + $(that.wrapSelector('#' + col)).html(''); + $(that.wrapSelector('#' + col)).prop('disabled', true); + }); + } } - render() { - super.render(); + templateForBody() { + let page = $(stHTML); + let that = this; + // generate dataselector let dataSelector = new DataSelector({ - type: 'data', - pageThis: this, - id: 'sample', - finish: function() { - ; + pageThis: this, id: 'data', placeholder: 'Select data', required: true, boxClasses: 'vp-flex-gap5', + allowDataType: ['DataFrame'], withPopup: false, + finish: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); + }, + select: function(data, type) { + that.state.data = data; + that.state.dataType = type; + $(that.wrapSelector('#data')).trigger('change'); } }); - $(this.wrapSelector('#sample')).replaceWith(dataSelector.toTagString()); + $(page).find('#data').replaceWith(dataSelector.toTagString()); + + return page; + } + + render() { + super.render(); + let that = this; + + // render Subset + this.subsetEditor = new Subset({ + pandasObject: '', + config: { name: 'Subset' } }, + { + useAsModule: true, + useInputColumns: true, + targetSelector: this.wrapSelector('#data'), + pageThis: this, + finish: function(code, state) { + that.state.data = code; + that.state.dataType = state.returnType; + $(that.wrapSelector('#data')).val(code); + $(that.wrapSelector('#data')).trigger('change'); + } + }); + + if (this.state.data !== '') { + let columnBindList = this.columnBindDict[this.state.testType]; + com_generator.vp_bindColumnSource(this, 'data', columnBindList, 'select', false, false); + } + + // control display option + $(this.wrapSelector('.vp-st-option')).hide(); + $(this.wrapSelector('.vp-st-option.' + this.state.testType)).show(); + if (this.state.testType === 'two-sample') { + $(this.wrapSelector('.vp-st-option.two-sample-' + this.state.inputType)).show(); + } } generateCode() { - return "print('sample code')"; + let { + testType, inputType, data, + testVariable, testVariable1, testVariable2, groupingVariable, + pairedVariable1, pairedVariable2, + group1, group2, group1_istext, group2_istext, + testValue, alterHypo, confInt + } = this.state; + let codeList = []; + let code = new com_String(); + + // 95% -> 0.95 + confInt = confInt/100; + + switch (testType) { + case 'one-sample': + code.appendLine("# One-sample t-test"); + // variable declaration + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + code.appendLine(""); + // 1. Normality test + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.shapiro(vp_df[{0}])", testVariable); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},index=['Normality test (Shapiro-Wilk)']))"); + code.appendLine(""); + // 2. One-sample Statistics + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendFormatLine("display(pd.DataFrame(data={'N':vp_df[{0}].size,'Mean':vp_df[{1}].mean(),", testVariable, testVariable); + code.appendFormatLine(" 'Std. Deviation':vp_df[{0}].std(),", testVariable); + code.appendFormatLine(" 'Std. Error Mean':vp_df[{0}].std()/np.sqrt(vp_df[{1}].size)},", testVariable, testVariable); + code.appendLine(" index=['Statistics']))"); + code.appendLine(""); + // 3. One-sample t-test + code.appendLine("# One-sample t-test"); + code.appendFormatLine("_res = stats.ttest_1samp(vp_df[{0}], popmean={1}, alternative='{2}')", testVariable, testValue, alterHypo); + code.appendFormatLine("_lower, _upper = _res.confidence_interval(confidence_level={0})", confInt); + code.appendLine("display(Markdown('### One-sample t-test'))"); + code.appendFormatLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'dof':_res.df,'Alternative':'{0}',", alterHypo); + code.appendFormatLine(" 'p-value':_res.pvalue,'Test Value':{0},'Mean difference':vp_df[{1}].mean()-{2},", testValue, testVariable, testValue); + code.appendFormatLine(" 'Confidence interval':{0},'Lower':_lower,'Upper':_upper},", confInt); + code.append(" index=['One-sample t-test']))"); + break; + case 'two-sample': + code.appendLine("# Independent two-sample t-test"); + // variable declaration + if (inputType === 'long-data') { + code.appendFormatLine("vp_df1 = {0}[({1}[{2}] == {3})][{4}].dropna().copy()", data, data, groupingVariable, com_util.convertToStr(group1, group1_istext), testVariable); + code.appendFormatLine("vp_df2 = {0}[({1}[{2}] == {3})][{4}].dropna().copy()", data, data, groupingVariable, com_util.convertToStr(group2, group2_istext), testVariable); + } else if (inputType === 'wide-data') { + code.appendFormatLine("vp_df1 = {0}[{1}].dropna().copy()", data, testVariable1); + code.appendFormatLine("vp_df2 = {0}[{1}].dropna().copy()", data, testVariable2); + } + code.appendLine(""); + // 1. Normality test + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendLine("_res1 = stats.shapiro(vp_df1)"); + code.appendLine("_res2 = stats.shapiro(vp_df2)"); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':[_res1.statistic,_res2.statistic],'p-value':[_res1.pvalue,_res2.pvalue]},"); + code.appendLine(" index=[['Normality test (Shapiro-Wilk)' for i in range(2)],['Variable1','Variable2']]))"); + code.appendLine(""); + // 2. Equal Variance test + code.appendLine("# Equal Variance test (Levene)"); + code.appendLine("display(Markdown('### Equal Variance test (Levene)'))"); + code.appendLine("_res = stats.levene(vp_df1, vp_df2, center='mean')"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue}, index=['Equal Variance test (Levene)']))"); + code.appendLine(""); + // 3. Independent two-sample Statistics + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendLine("display(pd.DataFrame(data={'N':[vp_df1.size,vp_df2.size],"); + code.appendLine(" 'Mean':[vp_df1.mean(),vp_df2.mean()],"); + code.appendLine(" 'Std. Deviation':[vp_df1.std(),vp_df2.std()],"); + code.appendLine(" 'Std. Error mean':[vp_df1.std()/np.sqrt(vp_df1.size),"); + code.appendLine(" vp_df2.std()/np.sqrt(vp_df2.size )]},"); + code.appendLine(" index=[['Statistics' for i in range(2)],['Variable1','Variable2']]))"); + code.appendLine(""); + // 4. Independent two-sample t-test + code.appendLine("# Independent two-sample t-test"); + code.appendFormatLine("_res1 = stats.ttest_ind(vp_df1, vp_df2, equal_var=True, alternative='{0}')", alterHypo); + code.appendFormatLine("_res2 = stats.ttest_ind(vp_df1, vp_df2, equal_var=False, alternative='{0}')", alterHypo); + code.appendLine("display(Markdown('### Independent two-sample t-test'))"); + code.appendFormatLine("display(pd.DataFrame(data={'Statistic':[_res1.statistic,_res2.statistic],'Alternative':['{0}' for i in range(2)],", alterHypo); + code.appendLine(" 'p-value':[_res1.pvalue,_res2.pvalue],"); + code.appendLine(" 'Mean difference':[vp_df1.mean()-vp_df2.mean() for i in range(2)]},"); + code.appendLine(" index=[['Independent two-sample t-test' for i in range(2)],['Equal variance' for i in range(2)],[True,False]]))"); + code.append("display(Markdown('If equal_var is False, perform Welch\\\'s t-test, which does not assume equal population variance'))"); + break; + case 'paired-sample': + // variable declaration + code.appendLine("# Paired samples t-test"); + code.appendFormatLine("vp_df = {0}.dropna().copy()", data); + code.appendLine(""); + code.appendFormatLine("try: vp_df[{0}].reset_index(drop=True, inplace=True)", pairedVariable1); + code.appendLine("except: pass"); + code.appendFormatLine("try: vp_df[{0}].reset_index(drop=True, inplace=True)", pairedVariable2); + code.appendLine("except: pass"); + code.appendLine(""); + // 1. Normality test + code.appendLine("# Normality test (Shapiro-Wilk)"); + code.appendLine("from IPython.display import display, Markdown"); + code.appendLine("from scipy import stats"); + code.appendFormatLine("_res = stats.shapiro(vp_df[{0}]-vp_df[{1}])", pairedVariable1, pairedVariable2); + code.appendLine("display(Markdown('### Normality test (Shapiro-Wilk)'))"); + code.appendLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'p-value':_res.pvalue},"); + code.appendLine(" index=['Normality test (Shapiro-Wilk): Paired differences']))"); + code.appendLine(""); + // 2. Paired samples Statistics + code.appendLine("# Statistics"); + code.appendLine("display(Markdown('### Statistics'))"); + code.appendFormatLine("display(pd.DataFrame(data={'N':[vp_df[{0}].size,vp_df[{1}].size,vp_df[{2}].size],", pairedVariable1, pairedVariable2, pairedVariable1); + code.appendFormatLine(" 'Mean':[vp_df[{0}].mean(),vp_df[{1}].mean(),(vp_df[{2}]-vp_df[{3}]).mean()],", pairedVariable1, pairedVariable2, pairedVariable1, pairedVariable2); + code.appendFormatLine(" 'Std. Deviation':[vp_df[{0}].std(),vp_df[{1}].std(),(vp_df[{2}]-vp_df[{3}]).std()],", pairedVariable1, pairedVariable2, pairedVariable1, pairedVariable2); + code.appendFormatLine(" 'Std. Error mean':[vp_df[{0}].std()/np.sqrt(vp_df[{1}].size),", pairedVariable1, pairedVariable1); + code.appendFormatLine(" vp_df[{0}].std()/np.sqrt(vp_df[{1}].size),", pairedVariable2, pairedVariable2); + code.appendFormatLine(" (vp_df[{0}]-vp_df[{1}]).std()/np.sqrt(vp_df[{2}].size)]},", pairedVariable1, pairedVariable2, pairedVariable1); + code.appendLine(" index=[['Statistics' for i in range(3)],['Variable1','Variable2','Paired differences']]))"); + code.appendLine(""); + // 3. Paired samples t-test + code.appendLine("# Paired samples t-test"); + code.appendFormatLine("_res = stats.ttest_rel(vp_df[{0}], vp_df[{1}], alternative='{2}')", pairedVariable1, pairedVariable2, alterHypo); + code.appendFormatLine("_lower, _upper = _res.confidence_interval(confidence_level={0})", confInt); + code.appendLine("display(Markdown('### Paired samples t-test'))"); + code.appendFormatLine("display(pd.DataFrame(data={'Statistic':_res.statistic,'dof':_res.df,'Alternative':'{0}',", alterHypo); + code.appendFormatLine(" 'p-value':_res.pvalue,'Mean difference':(vp_df[{0}]-vp_df[{1}]).mean(),", pairedVariable1, pairedVariable2); + code.appendFormatLine(" 'Confidence interval':{0},'Lower':_lower,'Upper':_upper},", confInt); + code.append(" index=['Paired samples t-test']))"); + break; + } + codeList.push(code.toString()); + + + return codeList; } } diff --git a/visualpython/js/m_stats/__init__.py b/visualpython/js/m_stats/__init__.py new file mode 100644 index 00000000..1e8c9919 --- /dev/null +++ b/visualpython/js/m_stats/__init__.py @@ -0,0 +1 @@ +print('Visual Python') diff --git a/visualpython/js/m_visualize/Chart.js b/visualpython/js/m_visualize/Chart.js index 4a3293c9..d979a6e1 100644 --- a/visualpython/js/m_visualize/Chart.js +++ b/visualpython/js/m_visualize/Chart.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_Const', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/com_generator', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/FileNavigation', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_Const, com_util, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { +], function(chartHTml, chartCss, com_String, com_Const, com_util, com_interface, com_generator, PopupComponent, FileNavigation, SuggestInput, DataSelector) { /** * Chart @@ -35,6 +36,7 @@ define([ this.config.dataview = false; this.config.sizeLevel = 2; this.config.checkModules = ['plt']; + this.config.docs = 'https://matplotlib.org/stable/'; this.setDefaultVariables(); this.state = { @@ -267,14 +269,16 @@ define([ $(this.wrapSelector('#vp_plImportRun')).click(function() { // generateImportCode var code = that.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Chart'); }); } @@ -1308,8 +1312,8 @@ define([ 'numerical': [ { label: 'count', method: 'count()' }, { label: 'unique count', method: 'unique()' }, - { label: 'sum', method: 'sum()' }, - { label: 'average', method: 'mean()' }, + { label: 'sum', method: 'sum(numeric_only=True)' }, + { label: 'average', method: 'mean(numeric_only=True)' }, { label: 'min', method: 'min()' }, { label: 'max', method: 'max()' } ] diff --git a/visualpython/js/m_visualize/Plotly.js b/visualpython/js/m_visualize/Plotly.js index e77db277..684840b3 100644 --- a/visualpython/js/m_visualize/Plotly.js +++ b/visualpython/js/m_visualize/Plotly.js @@ -34,6 +34,7 @@ define([ this.config.importButton = true; this.config.dataview = false; this.config.checkModules = ['px']; + this.config.docs = 'https://plotly.com/python-api-reference/index.html'; this.state = { chartType: 'scatter', @@ -450,20 +451,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_ptPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_ptPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } diff --git a/visualpython/js/m_visualize/Seaborn.js b/visualpython/js/m_visualize/Seaborn.js index 9b2b34b8..bea6cfd1 100644 --- a/visualpython/js/m_visualize/Seaborn.js +++ b/visualpython/js/m_visualize/Seaborn.js @@ -18,12 +18,13 @@ define([ 'vp_base/js/com/com_String', 'vp_base/js/com/com_generatorV2', 'vp_base/js/com/com_util', + 'vp_base/js/com/com_interface', 'vp_base/js/com/component/PopupComponent', 'vp_base/js/com/component/SuggestInput', 'vp_base/js/com/component/VarSelector2', 'vp_base/data/m_visualize/seabornLibrary', 'vp_base/js/com/component/DataSelector' -], function(chartHTml, chartCss, com_String, com_generator, com_util, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { +], function(chartHTml, chartCss, com_String, com_generator, com_util, com_interface, PopupComponent, SuggestInput, VarSelector2, SEABORN_LIBRARIES, DataSelector) { class Seaborn extends PopupComponent { _init() { @@ -31,7 +32,9 @@ define([ this.config.dataview = false; this.config.size = { width: 1064, height: 550 }; + this.config.autoScroll = false; this.config.checkModules = ['plt', 'sns']; + this.config.docs = 'https://seaborn.pydata.org/index.html'; this.state = { chartType: 'scatterplot', @@ -51,6 +54,7 @@ define([ stat: '', showValues: false, showValuesPrecision: '', + errorbar: '', sortBy: 'y', sortType: '', sortHue: '', @@ -198,7 +202,9 @@ define([ $(that.wrapSelector('#kde')).closest('.sb-option').show(); $(that.wrapSelector('#stat')).closest('.sb-option').show(); } else if (chartType == 'barplot') { + $(that.wrapSelector('#orient')).closest('.sb-option').show(); $(that.wrapSelector('#showValues')).closest('.sb-option').show(); + $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); if (that.state.setXY === false) { if (that.state.x !== '' && that.state.y !== '') { $(that.wrapSelector('#sortBy')).closest('.sb-option').show(); @@ -219,6 +225,8 @@ define([ } } else if (chartType == 'heatmap') { $(that.wrapSelector('#annot')).closest('.sb-option').show(); + } else if (chartType === 'lineplot') { + $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); } }); @@ -360,29 +368,33 @@ define([ let val = $(this).val(); if (val !== '') { // enable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', false); + $(that.wrapSelector('#xticks_label')).attr('readonly', false); } else { // disable xticks_label - $(that.wrapSelector('#xticks_label')).prop('readonly', true); + $(that.wrapSelector('#xticks_label')).attr('readonly', true); } }); $(this.wrapSelector('#yticks')).on('change', function() { let val = $(this).val(); if (val !== '') { // enable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', false); + $(that.wrapSelector('#yticks_label')).attr('readonly', false); } else { // disable yticks_label - $(that.wrapSelector('#yticks_label')).prop('readonly', true); + $(that.wrapSelector('#yticks_label')).attr('readonly', true); } }); // axes - ticks label: inform user to type location option to use label - $(this.wrapSelector('#xticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#xticks')).focus(); + $(this.wrapSelector('#xticks_label')).on('click', function() { + if ($(that.wrapSelector('#xticks')).val() === '') { + $(that.wrapSelector('#xticks')).focus(); + } }); - $(this.wrapSelector('#yticks_label[readonly]')).on('click', function() { - $(that.wrapSelector('#yticks')).focus(); + $(this.wrapSelector('#yticks_label')).on('click', function() { + if ($(that.wrapSelector('#yticks')).val() === '') { + $(that.wrapSelector('#yticks')).focus(); + } }); // preview refresh @@ -521,6 +533,15 @@ define([ }); $(page).find('#sampleCount').html(sampleCountTag.toString()); + // set errorbar list + var vpErrorbarSuggest = new SuggestInput(); + vpErrorbarSuggest.setComponentID('errorbar'); + vpErrorbarSuggest.addClass('vp-input vp-state'); + vpErrorbarSuggest.setPlaceholder("('ci', 95)"); + vpErrorbarSuggest.setValue(this.state.errorbar); + vpErrorbarSuggest.setSuggestList(["None", "'ci'", "'pi'", "'sd'", "'se'"]); + $(page).find('#errorbar').replaceWith(vpErrorbarSuggest.toTagString()); + // data options depend on chart type $(page).find('.sb-option').hide(); if (this.state.chartType == 'histplot') { @@ -528,7 +549,9 @@ define([ $(page).find('#kde').closest('.sb-option').show(); $(page).find('#stat').closest('.sb-option').show(); } else if (this.state.chartType == 'barplot') { + $(page).find('#orient').closest('.sb-option').show(); $(page).find('#showValues').closest('.sb-option').show(); + $(page).find('#errorbar').closest('.sb-option').show(); if (this.state.setXY === false) { if (this.state.x !== '' && this.state.y !== '') { $(page).find('#sortBy').closest('.sb-option').show(); @@ -549,6 +572,8 @@ define([ } } else if (this.state.chartType == 'heatmap') { $(page).find('#annot').closest('.sb-option').show(); + } else if (this.state.chartType === 'lineplot') { + $(page).find('#errorbar').closest('.sb-option').show(); } //================================================================ @@ -765,14 +790,16 @@ define([ handleInnerOk() { // generateImportCode var code = this.generateImportCode(); + // DEPRECATED: no longer save to block as default // create block and run it - $('#vp_wrapper').trigger({ - type: 'create_option_page', - blockType: 'block', - menuId: 'lgExe_code', - menuState: { taskState: { code: code } }, - afterAction: 'run' - }); + // $('#vp_wrapper').trigger({ + // type: 'create_option_page', + // blockType: 'block', + // menuId: 'lgExe_code', + // menuState: { taskState: { code: code } }, + // afterAction: 'run' + // }); + com_interface.insertCell('code', code, true, 'Visualization > Seaborn'); this.closeInnerPopup(); @@ -809,20 +836,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#chartPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue, msg.content.detail); } $(that.wrapSelector('#chartPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } @@ -866,8 +893,8 @@ define([ generateCode(preview=false) { let { - chartType, data, x, y, setXY, hue, kde, stat, - showValues, showValuesPrecision, + chartType, data, x, y, setXY, hue, orient, kde, stat, + showValues, showValuesPrecision, errorbar, sortType, sortBy, sortHue, sortHueText, userOption='', x_limit_from, x_limit_to, y_limit_from, y_limit_to, @@ -932,6 +959,12 @@ define([ if (showValues === true && chartType === 'barplot') { // etcOptionCode.push('ci=None'); // changed to errorbar after 0.12 version etcOptionCode.push('errorbar=None'); + } else { + if (chartType === 'barplot' || chartType === 'lineplot') { + if (errorbar !== '') { + etcOptionCode.push(com_util.formatString("errorbar={0}", errorbar)); + } + } } if (setXY === false && sortType !== '') { let sortCode = ''; @@ -944,10 +977,10 @@ define([ } if (chartType === 'barplot' && sortX !== '' && sortY !== '') { if (hue !== '' && sortHue !== '') { - sortCode = com_util.formatString("{0}[{1}[{2}]=={3}].groupby({4})[{5}].mean().sort_values({6}).index" + sortCode = com_util.formatString("{0}[{1}[{2}]=={3}].groupby({4})[{5}].mean(numeric_only=True).sort_values({6}).index" , state.data, state.data, state.hue, com_util.convertToStr(sortHue, sortHueText), sortX, sortY, sortTypeStr); } else { - sortCode = com_util.formatString("{0}.groupby({1})[{2}].mean().sort_values({3}).index", state.data, sortX, sortY, sortTypeStr); + sortCode = com_util.formatString("{0}.groupby({1})[{2}].mean(numeric_only=True).sort_values({3}).index", state.data, sortX, sortY, sortTypeStr); } } else if (chartType === 'countplot' && (sortX !== '' || sortY !== '')) { let countVar = sortX === ''? sortY: sortX; diff --git a/visualpython/js/m_visualize/WordCloud.js b/visualpython/js/m_visualize/WordCloud.js index b361ed65..96ee22a1 100644 --- a/visualpython/js/m_visualize/WordCloud.js +++ b/visualpython/js/m_visualize/WordCloud.js @@ -32,6 +32,7 @@ define([ this.config.importButton = true; this.config.dataview = false; this.config.checkModules = ['Counter', 'plt', 'WordCloud']; + this.config.docs = 'https://amueller.github.io/word_cloud/references.html'; this.state = { data: '', @@ -264,20 +265,20 @@ define([ } } else { var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_wcPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); } }).catch(function(resultObj) { let { msg } = resultObj; var errorContent = ''; - if (msg.content.ename) { + if (msg.content && msg.content.ename) { errorContent = com_util.templateForErrorBox(msg.content.ename, msg.content.evalue); } $(that.wrapSelector('#vp_wcPreview')).html(errorContent); - vpLog.display(VP_LOG_TYPE.ERROR, msg.content.ename, msg.content.evalue, msg.content); + vpLog.display(VP_LOG_TYPE.ERROR, msg.content?.ename, msg.content?.evalue, msg.content); }); }); } diff --git a/visualpython/js/menu/MenuFrame.js b/visualpython/js/menu/MenuFrame.js index f727ce0a..386916a3 100644 --- a/visualpython/js/menu/MenuFrame.js +++ b/visualpython/js/menu/MenuFrame.js @@ -23,13 +23,14 @@ define([ '../com/component/Component', '../com/component/SuggestInput', '../com/component/InnerFuncViewer', + '../com/component/PackageManager', __VP_RAW_LOADER__('../../data/libraries.json'), // INTEGRATION: text! to raw-loader './MenuGroup', './MenuItem', './TaskBar' -], function(menuFrameHtml, menuFrameCss, com_Config, com_Const, com_util, com_interface, Component, SuggestInput, InnerFuncViewer, +], function(menuFrameHtml, menuFrameCss, com_Config, com_Const, com_util, com_interface, Component, SuggestInput, InnerFuncViewer, PackageManager, librariesJson, MenuGroup, MenuItem, TaskBar) { 'use strict'; @@ -114,6 +115,11 @@ define([ $(this.wrapSelector('#vp_versionUpdater')).on('click', function() { vpConfig.checkVpVersion(); }); + // Click package manager + $(this.wrapSelector('#vp_packageManager')).on('click', function() { + let packageManager = new PackageManager(); + packageManager.open(); + }); } _unbindResizable() { @@ -182,7 +188,7 @@ define([ getMenuLibraries() { var libraries = {}; // LAB: webpack5 load json object by default - if (vpConfig.extensionType === 'lab') { + if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { libraries = librariesJson; } else { libraries = JSON.parse(librariesJson); @@ -199,19 +205,19 @@ define([ return {}; } - getMenuLibrary(menuId, libraries=this.menuLibraries) { + getMenuLibrary(menuId, libraries=this.menuLibrariesFlatten) { for (var i=0; i < libraries.length; i++) { var item = libraries[i]; if (item) { if (item.id === menuId) { return item; } - if (item.type === 'package') { - var result = this.getMenuLibrary(menuId, item.item); - if (result) { - return result; - } - } + // if (item.type === 'package') { + // var result = this.getMenuLibrary(menuId, item.item); + // if (result) { + // return result; + // } + // } } } return null; @@ -246,7 +252,12 @@ define([ var that = this; var body = group.getBody(); var item = group.getItem(); + var state = group.getState(); item && item.forEach(child => { + // remember parent to navigate its parent menu + var category = state.category?(state.category + ' > ' + state.name):state.name; + child['category'] = category; + if (child.type == 'package') { // packages : MenuGroup var menuGroup = new MenuGroup($(body), child); @@ -292,7 +303,7 @@ define([ }); let functionList = this.menuLibrariesFlatten.map(menu => { - return { label: menu.name, value: menu.name, ...menu } + return { label: menu.name, value: menu.name, dtype: menu.category, ...menu } }); // render searchbox let searchBox = new SuggestInput(); diff --git a/visualpython/js/menu/MenuItem.js b/visualpython/js/menu/MenuItem.js index 22343b2e..8cebdd73 100644 --- a/visualpython/js/menu/MenuItem.js +++ b/visualpython/js/menu/MenuItem.js @@ -148,6 +148,10 @@ define([ // LAB: img to url // page.appendFormatLine('

', com_Const.IMAGE_PATH + apps.icon); page.appendFormatLine('

', id); + // Exception for title alignment + if (id === 'stats_studentstTest') { + name = "Student's
t-test"; + } page.appendFormatLine('

{0}

', name); page.append(''); } else { diff --git a/visualpython/js/menu/TaskItem.js b/visualpython/js/menu/TaskItem.js index dcb63181..c8e2be64 100644 --- a/visualpython/js/menu/TaskItem.js +++ b/visualpython/js/menu/TaskItem.js @@ -39,10 +39,7 @@ define([ if (isOpen) { // hide task if it's already opened // open task - $('#vp_wrapper').trigger({ - type: 'close_option_page', - component: that.state.task - }); + that.state.task.hide(); } else { // open task $('#vp_wrapper').trigger({ diff --git a/visualpython/lib/codemirror/lib/codemirror.css b/visualpython/lib/codemirror/lib/codemirror.css index f4d5718a..0a80f68d 100644 --- a/visualpython/lib/codemirror/lib/codemirror.css +++ b/visualpython/lib/codemirror/lib/codemirror.css @@ -3,7 +3,7 @@ .CodeMirror { /* Set height, width, borders, and global font properties here */ font-family: monospace; - height: 300px; + /* height: 300px; */ color: black; direction: ltr; } diff --git a/visualpython/python/pandasCommand.py b/visualpython/python/pandasCommand.py index e418c1ff..4f8301c6 100644 --- a/visualpython/python/pandasCommand.py +++ b/visualpython/python/pandasCommand.py @@ -11,7 +11,7 @@ # from IPython.core.display is deprecated since IPython 7.14 from IPython.display import display -def _vp_get_rows_list(df): +def _vp_get_rows_list(df, start_idx=0): """ Get Rows List with Detail Information """ @@ -19,7 +19,8 @@ def _vp_get_rows_list(df): indexType = str(df.index.dtype) # make dict for rows info for i, r in enumerate(df.index): - rInfo = { 'label': r, 'value': r, 'location': i } + rInfo = { 'label': r, 'value': r, 'location': start_idx + i } + # value if type(r).__name__ == 'str': rInfo['value'] = "'{}'".format(r) @@ -37,7 +38,7 @@ def _vp_get_columns_list(df): """ colInfo = { 'name': list(df.columns.names), 'level': df.columns.nlevels, 'list': [] } for i, c in enumerate(df.columns): - cInfo = { 'label': c, 'value': c, 'dtype': str(df[c].dtype), 'array': str(df[c].array), 'location': i } + cInfo = { 'label': c, 'value': c, 'dtype': str(df[c].dtype), 'is_numeric': _vp_pd.api.types.is_numeric_dtype(df[c]), 'array': str(df[c].array), 'location': i } # value if type(c).__name__ == 'list' or type(c).__name__ == 'tuple': cInfo['label'] = list(c) diff --git a/visualpython/python/userCommand.py b/visualpython/python/userCommand.py index 6383f560..78034078 100644 --- a/visualpython/python/userCommand.py +++ b/visualpython/python/userCommand.py @@ -7,6 +7,8 @@ import pandas as _vp_pd import numpy as _vp_np import matplotlib.pyplot as _vp_plt +import scipy.stats as _vp_stats +import statsmodels.api as _vp_sm import fitz import nltk nltk.download('punkt') @@ -46,23 +48,48 @@ def vp_pdf_get_sentence(fname_lst): ###### # Visual Python: Data Analysis > Frame ###### -def vp_drop_outlier(df, col, weight=1.5): - sr = df[col] - - q25 = _vp_np.percentile(sr.values, 25) - q75 = _vp_np.percentile(sr.values, 75) - - iqr = q75 - q25 - iqr_w = iqr * weight - - val_l = q25 - iqr_w - val_h = q75 + iqr_w - - outlier_index = sr[(sr < val_l) | (sr > val_h)].index - - df_res = df.drop(outlier_index).copy() - - return df_res +def vp_fill_outlier(df, col_lst, fill_type='iqr', fill_value_lst=[], weight=1.5): + dfr = df.copy() + for idx, col in enumerate(col_lst): + sr = dfr[col] + q25 = _vp_np.percentile(sr.values, 25) + q75 = _vp_np.percentile(sr.values, 75) + iqr = q75 - q25 + iqr_w = iqr * weight + val_l = q25 - iqr_w + val_h = q75 + iqr_w + if fill_type == 'mean': + f_val = sr[~((sr < val_l) | (sr > val_h))].mean() + elif fill_type == 'median': + f_val = sr[~((sr < val_l) | (sr > val_h))].median() + elif fill_type == 'value': + f_val = fill_value_lst[idx] + elif fill_type == 'NA': + f_val = _vp_np.nan + if fill_type == 'iqr': + dfr.loc[(sr < val_l), col] = val_l + dfr.loc[(sr > val_h), col] = val_h + else: + dfr.loc[(sr < val_l) | (sr > val_h), col] = f_val + return dfr +###### +# Visual Python: Data Analysis > Frame +###### +def vp_drop_outlier(df, col_lst, weight=1.5): + dfr = df.copy() + outlier_index_lst = [] + for idx, col in enumerate(col_lst): + sr = dfr[col] + q25 = _vp_np.percentile(sr.values, 25) + q75 = _vp_np.percentile(sr.values, 75) + iqr = q75 - q25 + iqr_w = iqr * weight + val_l = q25 - iqr_w + val_h = q75 + iqr_w + outlier_index_lst += sr[(sr < val_l) | (sr > val_h)].index.to_list() + outlier_index_lst = list(set(outlier_index_lst)) + dfr.drop(outlier_index_lst, inplace=True) + return dfr ###### # Visual Python: Machine Learning > Model Info ###### @@ -73,7 +100,7 @@ def vp_create_feature_importances(model, X_train=None, sort=False): feature_names = [ 'X{}'.format(i) for i in range(len(model.feature_importances_)) ] df_i = _vp_pd.DataFrame(model.feature_importances_, index=feature_names, columns=['Feature_importance']) - df_i['Percentage'] = 100 * (df_i['Feature_importance'] / df_i['Feature_importance'].max()) + df_i['Percentage'] = 100 * df_i['Feature_importance'] if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) df_i = df_i.round(2) @@ -96,6 +123,41 @@ def vp_plot_feature_importances(model, X_train=None, sort=False, top_count=0): _vp_plt.show() ###### +# Visual Python: Machine Learning > Model Info +###### +def vp_create_permutation_importances(model, X_train, y_train, scoring=None, sort=False): + from sklearn.inspection import permutation_importance + if isinstance(X_train, _vp_pd.core.frame.DataFrame): + feature_names = X_train.columns + else: + feature_names = [ 'X{}'.format(i) for i in range(X_train.shape[1]) ] + + imp = permutation_importance(model, X_train, y_train, scoring=scoring) + + df_i = _vp_pd.DataFrame(imp['importances_mean'], index=feature_names, columns=['Feature_importance']) + df_i['Percentage'] = 100 * df_i['Feature_importance'] + if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) + df_i = df_i.round(2) + + return df_i +###### +# Visual Python: Machine Learning > Model Info +###### +def vp_plot_permutation_importances(model, X_train, y_train, scoring=None, sort=False, top_count=0): + df_i = vp_create_permutation_importances(model, X_train, y_train, scoring=scoring, sort=sort) + + if sort: + if top_count > 0: + df_i['Percentage'].sort_values().tail(top_count).plot(kind='barh') + else: + df_i['Percentage'].sort_values().plot(kind='barh') + else: + df_i['Percentage'].plot(kind='barh') + _vp_plt.xlabel('Feature importance Percentage') + _vp_plt.ylabel('Features') + + _vp_plt.show() +###### # Visual Python: Visualization > Seaborn ###### def vp_seaborn_show_values(axs, precision=1, space=0.01): @@ -134,4 +196,121 @@ def _single(ax): for idx, ax in _vp_np.ndenumerate(axs): _single(ax) else: - _single(axs) \ No newline at end of file + _single(axs) +###### +# Visual Python: Statistics > Correlation Analysis +###### +def vp_confidence_interval_corr(x, y, method='pearson', alpha=0.05): + try: x=_vp_pd.Series(x); y=_vp_pd.Series(y) + except: return _vp_np.nan + + corr_func = {'pearson':_vp_stats.pearsonr,'spearman':_vp_stats.spearmanr,'kendall':_vp_stats.kendalltau} + se_diff = {'pearson':3,'spearman':3,'kendall':4} + se_func = {'pearson': lambda corr: 1, + 'spearman':lambda corr: 1 + corr ** 2 / 2., + 'kendall': lambda corr: .437 } + + corr, pvalue = corr_func[method](x,y) + + z = _vp_np.log((1 + corr) / (1 - corr)) / 2 + se = _vp_np.sqrt(se_func[method](corr) / (x.size - se_diff[method])) + + z_lower = z - _vp_stats.norm.ppf(1 - alpha / 2.) * se + z_upper = z + _vp_stats.norm.ppf(1 - alpha / 2.) * se + + corr_lower = (_vp_np.exp(2 * z_lower) - 1) / (_vp_np.exp(2 * z_lower) + 1) + corr_upper = (_vp_np.exp(2 * z_upper) - 1) / (_vp_np.exp(2 * z_upper) + 1) + + return corr, pvalue, corr_lower, corr_upper +###### +# Visual Python: Statistics > Reliability Analysis +###### +def vp_cronbach_alpha(data): + _corr = data.corr() + _N = data.shape[1] + _rs = _vp_np.array([]) + for i, col in enumerate(_corr.columns): + _sum = _corr[col][i+1:].values + _rs = _vp_np.append(_sum, _rs) + _mean = _vp_np.mean(_rs) + + return (_N*_mean)/(1+(_N-1)*_mean) +###### +# Visual Python: Statistics > ANOVA +###### +def vp_confidence_interval(var, confidence_level=0.95): + try: sr = _vp_pd.Series(var) + except: return _vp_np.nan + return _vp_stats.t.interval(confidence_level, df=sr.count()-1, loc=sr.mean(), scale=sr.std() / _vp_np.sqrt(sr.count()) ) +###### +# Visual Python: Statistics > ANOVA +###### +def vp_sem(var): + try: sr = _vp_pd.Series(var) + except: return _vp_np.nan + return sr.std() / _vp_np.sqrt(sr.count()) +###### +# Visual Python: Statistics > Regression - Multiple linear regression > Method: Stepwise +###### +def vp_stepwise_select(df_x, df_y, alpha=0.05): + select_list = list() + while len(df_x.columns) > 0: + col_list = list(set(df_x.columns)-set(select_list)) + sr_pval = _vp_pd.Series(index=col_list) + for col in col_list: + result = _vp_sm.OLS(df_y, _vp_sm.add_constant(df_x[select_list+[col]])).fit() + sr_pval[col] = result.pvalues[col] + best_pval = sr_pval.min() + if best_pval < alpha: + select_list.append(sr_pval.idxmin()) + while len(select_list) > 0: + result = _vp_sm.OLS(df_y, _vp_sm.add_constant(df_x[select_list])).fit() + sr_pval2 = result.pvalues.iloc[1:] + worst_pval = sr_pval2.max() + if worst_pval > alpha: + select_list.remove(sr_pval2.idxmax()) + else: + break + else: + break + return select_list +###### +# Visual Python: Statistics > Regression - Multiple linear regression > Method: Backward +###### +def vp_backward_select(df_x, df_y, alpha=0.05): + select_list=list(df_x.columns) + while True: + result = _vp_sm.OLS(df_y, _vp_sm.add_constant(_vp_pd.DataFrame(df_x[select_list]))).fit() + sr_pval = result.pvalues.iloc[1:] + worst_pval = sr_pval.max() + if worst_pval > alpha: + select_list.remove(sr_pval.idxmax()) + else: + break + return select_list +###### +# Visual Python: Statistics > Regression - Multiple linear regression > Method: Forward +###### +def vp_forward_select(df_x, df_y, alpha=0.05): + select_list = list() + while True: + col_list = list(set(df_x.columns)-set(select_list)) + sr_pval = _vp_pd.Series(index=col_list) + for col in col_list: + result = _vp_sm.OLS(df_y, _vp_sm.add_constant(_vp_pd.DataFrame(df_x[select_list+[col]]))).fit() + sr_pval[col] = result.pvalues[col] + best_pval = sr_pval.min() + if best_pval < alpha: + select_list.append(sr_pval.idxmin()) + else: + break + + return select_list +###### +# Visual Python: Statistics > Regression - Mediated linear regression +###### +def vp_sobel(a, b, sea, seb): + z = (a * b) / ( (a**2)*(seb**2) + (b**2)*(sea**2) )**0.5 + one_pvalue = _vp_stats.norm.sf(abs(z)) + two_pvalue = _vp_stats.norm.sf(abs(z))*2 + return z, one_pvalue, two_pvalue \ No newline at end of file diff --git a/visualpython/python/variableCommand.py b/visualpython/python/variableCommand.py index 3195908e..6a0df33c 100644 --- a/visualpython/python/variableCommand.py +++ b/visualpython/python/variableCommand.py @@ -125,4 +125,22 @@ def _vp_check_module_loaded(fname_list): result.append(True) else: result.append(False) - return result \ No newline at end of file + return result + +def _vp_check_package_list(pack_list): + """ + Check package info : name, version, path + """ + import importlib as _vp_ilib + import warnings + _pack_info = {} + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + for pack in pack_list: + try: + _vp_pack = _vp_ilib.import_module(pack) + _vp_ilib.reload(_vp_pack) + _pack_info[pack] = { 'name': _vp_pack.__name__, 'installed': True, 'version': _vp_pack.__version__, 'path': _vp_pack.__path__ } + except: + _pack_info[pack] = { 'name': pack, 'installed': False } + return _pack_info \ No newline at end of file diff --git a/visualpython/visualpython.yaml b/visualpython/visualpython.yaml index 11ae71f3..a4e3d5bb 100644 --- a/visualpython/visualpython.yaml +++ b/visualpython/visualpython.yaml @@ -5,28 +5,4 @@ Link: README.md Icon: img/logo.png Main: visualpython.js Compatibility: 4.x, 5.x, 6.x -Parameters: -- name: vpcfg.run_code_without_asking - description: Run code without asking - input_type: checkbox - default: false -- name: vpcfg.change_task_without_asking - description: Change task without asking - input_type: checkbox - default: false -- name: vpcfg.code_insert_position - description: Code insert position - input_type: text - default: below -- name: vpcfg.api_list_sort_by - description: Api list sorting by... - input_type: text - default: frequency -- name: vpcfg.default_variable_for_required - description: Default variable for required input - input_type: checkbox - default: false -- name: vpcfg.auto_import_package - description: Automatically import packages - input_type: checkbox - default: false +Parameters: \ No newline at end of file