Options

diff --git a/visualpython/html/m_ml/modelInfo.html b/visualpython/html/m_ml/modelInfo.html index 690efb8b..02badc27 100644 --- a/visualpython/html/m_ml/modelInfo.html +++ b/visualpython/html/m_ml/modelInfo.html @@ -38,7 +38,7 @@

Options

diff --git a/visualpython/html/m_stats/probDist.html b/visualpython/html/m_stats/probDist.html index 0bab3ef5..e1a54fa7 100644 --- a/visualpython/html/m_stats/probDist.html +++ b/visualpython/html/m_stats/probDist.html @@ -20,8 +20,8 @@

Action diff --git a/visualpython/html/m_visualize/seaborn.html b/visualpython/html/m_visualize/seaborn.html index d39d2d1b..08e99bd7 100644 --- a/visualpython/html/m_visualize/seaborn.html +++ b/visualpython/html/m_visualize/seaborn.html @@ -100,6 +100,14 @@ Bins

+ Orient + +

Kde ' , 'vp_fileNavigationInput', 'New File Name', this.state.fileName); page.appendFormatLine(''); page.appendFormatLine('', 'vp-filenavi-btn', 'select', 'Select'); @@ -430,6 +460,13 @@ define([ that.handleSelectFile(filePath, fileName); }); + // bind file extension change event + $(this.wrapSelector('#vp_fileNavigationExt')).on('change', function() { + let ext = $(this).val(); + that.selectedExt = ext; + + that.renderFileList(); + }); // bind save cancel event $(this.wrapSelector('.vp-filenavi-btn')).on('click', function() { let menu = $(this).data('menu'); @@ -477,10 +514,10 @@ define([ let that = this; /** Implement after rendering */ // if save mode - if (this.state.type == 'save') { - // render saving box - this.renderSaveBox(); - } + // if (this.state.type == 'save') { + // render saving box + this.renderSaveBox(); + // } // get current path this.getCurrentDirectory().then(function(currentPath) { @@ -527,20 +564,21 @@ define([ //============================================================================ // Set selection result //============================================================================ - if (this.state.type == 'save') { - // add as saving file - this.setSelectedFile(fileInput, pathInput); - } else { - // Manage result using finish function - let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented - let status = true; - let error = null; - vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); - this.state.finish(filesPath, status, error); + this.setSelectedFile(fileInput, pathInput); + // if (this.state.type == 'save') { + // // add as saving file + // this.setSelectedFile(fileInput, pathInput); + // } else { + // // Manage result using finish function + // let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented + // let status = true; + // let error = null; + // vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); + // this.state.finish(filesPath, status, error); - // remove and close file navigation - this.close(); - } + // // remove and close file navigation + // this.close(); + // } } getCurrentDirectory() { return vpKernel.getCurrentDirectory(); @@ -588,31 +626,6 @@ define([ return a - b; }); - /** - * Filter file/dir which included in this.state.extensions - */ - if (Array.isArray(that.state.extensions) && that.state.extensions.length > 0 && that.state.extensions.toString() !== '') { - filtered_varList = filtered_varList.filter((data, index) => { - if (index == 0) { - return true; - } - - if (data.type && data.type == 'dir') { - // if file, just show - return true; - } else if (data.name) { - var extension = data.name.substring(data.name.lastIndexOf('.') + 1); - if (that.state.extensions.includes(extension)) { - return true; - } else { - return false; - } - } else { - return false; - } - }); - } - vpLog.display(VP_LOG_TYPE.DEVELOP, 'FileNavigation - getFileList: ', filtered_varList); var { currentDirStr, currentRelativePathStr } = that.splitPathStrAndSetStack(dirObj, filtered_varList); diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index a31fe8af..ae109fd9 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -143,7 +143,6 @@ define([ description: 'Transform labels to normalized encoding.' } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -159,6 +158,32 @@ define([ } } } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -407,10 +432,11 @@ define([ 'fit': { name: 'fit', label: 'Fit', - code: '${model}.fit(${fit_featureData})', + code: '${model}.fit(${fit_featureData}${fit_targetData})', description: 'Run fit with all sets of parameters.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } ] }, 'predict': { @@ -419,7 +445,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Call predict on the estimator with the best found parameters.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -479,16 +505,30 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['input'], usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', @@ -584,6 +624,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index 016eacc0..d9d3d9fa 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -112,7 +112,7 @@ define([ }); break; case 'variable': - this._getVariableList(this.type, function(dataList) { + this._getVariableList(this.varType, function(dataList) { that._executeCallback(dataList); }); break; @@ -367,9 +367,9 @@ define([ let infoStr = ''; if (mode === 'columns') { if (data.isNumeric === true) { - iconStr = ''; + iconStr = ''; } else { - iconStr = ''; + iconStr = ''; } } else if (mode === 'variable') { infoStr = ` | ${data.type}`; @@ -401,9 +401,9 @@ define([ let infoStr = ''; if (mode === 'columns') { if (data.isNumeric === true) { - iconStr = ''; + iconStr = ''; } else { - iconStr = ''; + iconStr = ''; } } else if (mode === 'variable') { infoStr = ` | ${data.type}`; diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js index 00afe14c..a808315e 100644 --- a/visualpython/js/com/component/PackageManager.js +++ b/visualpython/js/com/component/PackageManager.js @@ -47,6 +47,7 @@ define([ this.packageLibTemplate = { 'numpy': { pipName: 'numpy' }, 'pandas': { pipName: 'pandas' }, + 'pyarrow': { pipName: 'pyarrow' }, 'matplotlib': { pipName: 'matplotlib' }, 'seaborn': { pipName: 'seaborn' }, 'plotly': { pipName: 'plotly' }, diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index c1159b5f..60b37260 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -487,22 +487,7 @@ define([ // set run button vpConfig.getData('runType', 'vpcfg').then(function(data) { vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); - if (data == undefined || data == null || data === '') { - data = 'run'; // default = run - } - that.config.runType = data; - $(that.wrapSelector(`.vp-popup-run-type[value="${data}"]`)).prop('checked', true); - $(that.wrapSelector('.vp-popup-run-button')).attr('data-type', data); - let runTitle = 'Run code'; - switch (data) { - case 'run-save': - runTitle = 'Save to block & Run code'; - break; - case 'add': - runTitle = 'Add code to cell'; - break; - } - $(that.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + that._setDefaultRunType(data); }); evt.stopPropagation(); break; @@ -527,24 +512,15 @@ define([ // Click detail radio $(this.wrapSelector('.vp-popup-run-type')).on('click', function(evt) { let runType = $(that.wrapSelector('.vp-popup-run-type[name=runType]:checked')).val(); - that.config.runType = runType; // save as vpConfig vpConfig.setData({runType: runType}, 'vpcfg'); - $(that.wrapSelector('.vp-popup-run-button')).attr('data-type', runType); - let runTitle = 'Run code'; - switch (runType) { - case 'run-save': - runTitle = 'Save to block & Run code'; - break; - case 'add': - runTitle = 'Add code to cell'; - break; - } - $(that.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + that._setDefaultRunType(runType); }); // Click detail buttons $(this.wrapSelector('.vp-popup-detail-action-button')).on('click', function(evt) { var btnType = $(this).data('type'); + vpConfig.setData({runType: btnType}, 'vpcfg'); + that._setDefaultRunType(btnType); // run, run-save, add switch(btnType) { // case 'apply': // that.save(); @@ -749,22 +725,7 @@ define([ let that = this; vpConfig.getData('runType', 'vpcfg').then(function(data) { vpLog.display(VP_LOG_TYPE.DEVELOP, 'Runtype get data', data); - if (data == undefined || data == null || data === '') { - data = 'run'; // default = run - } - that.config.runType = data; - $(that.wrapSelector(`.vp-popup-run-type[value="${data}"]`)).prop('checked', true); - $(that.wrapSelector('.vp-popup-run-button')).attr('data-type', data); - let runTitle = 'Run code'; - switch (data) { - case 'run-save': - runTitle = 'Save to block & Run code'; - break; - case 'add': - runTitle = 'Add code to cell'; - break; - } - $(that.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + that._setDefaultRunType(data); }); } @@ -911,6 +872,25 @@ define([ vpLog.display(VP_LOG_TYPE.DEVELOP, 'savedState', that.state); } + _setDefaultRunType(runType) { + if (runType == undefined || runType == null || runType === '') { + runType = 'run'; // default = run + } + this.config.runType = runType; + $(this.wrapSelector(`.vp-popup-run-type[value="${runType}"]`)).prop('checked', true); + $(this.wrapSelector('.vp-popup-run-button')).attr('data-type', runType); + let runTitle = 'Run code'; // when runType is 'run' + switch (runType) { + case 'run-save': + runTitle = 'Save to block & Run code'; + break; + case 'add': + runTitle = 'Add code to cell'; + break; + } + $(this.wrapSelector('.vp-popup-run-button')).prop('title', runTitle); + } + _getTagValue(tag) { let id = tag.id; let tagName = $(tag).prop('tagName'); // returns with UpperCase @@ -1348,7 +1328,7 @@ define([ $(this.wrapSelector('.vp-inner-popup-box')).show(); // focus on first input - $(this.wrapSelector('.vp-inner-popup-box input:not(:disabled):visible:first')).focus(); + $(this.wrapSelector('.vp-inner-popup-box input:not(:readonly):not(:disabled):visible:first')).focus(); // disable Jupyter key com_interface.disableOtherShortcut(); } diff --git a/visualpython/js/com/lib/__init__.py b/visualpython/js/com/lib/__init__.py deleted file mode 100644 index 1e8c9919..00000000 --- a/visualpython/js/com/lib/__init__.py +++ /dev/null @@ -1 +0,0 @@ -print('Visual Python') diff --git a/visualpython/js/com/template/__init__.py b/visualpython/js/com/template/__init__.py deleted file mode 100644 index 1e8c9919..00000000 --- a/visualpython/js/com/template/__init__.py +++ /dev/null @@ -1 +0,0 @@ -print('Visual Python') diff --git a/visualpython/js/loadVisualpython.js b/visualpython/js/loadVisualpython.js index d7f57aa5..206f2d94 100644 --- a/visualpython/js/loadVisualpython.js +++ b/visualpython/js/loadVisualpython.js @@ -287,8 +287,10 @@ define([ } else if (vpConfig.extensionType === 'lab' || vpConfig.extensionType === 'lite') { // LAB: if widget is ready or changed, ready for lab kernel connected, and restart vp vpLab.shell._currentChanged.connect(function(s1, value) { - var { newValue } = value; - vpLog.display(VP_LOG_TYPE.DEVELOP, 'jupyterlab shell currently changed', s1, value); + // var { newValue } = value; + // LAB 4.x: get currentWidget + var newValue = s1.currentWidget; + vpLog.display(VP_LOG_TYPE.DEVELOP, 'jupyterlab shell currently changed', s1, newValue); // kernel restart for notebook and console if (newValue && newValue.sessionContext) { vpConfig.hideProtector(); diff --git a/visualpython/js/m_apps/Bind.js b/visualpython/js/m_apps/Bind.js index 4a015029..8d5e42af 100644 --- a/visualpython/js/m_apps/Bind.js +++ b/visualpython/js/m_apps/Bind.js @@ -339,7 +339,7 @@ define([ renderDataPage(renderedText, isHtml = true) { var tag = new com_String(); - tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area + tag.appendFormatLine('

', 'vp_rendered_html'); // 'rendered_html' style from jupyter output area if (isHtml) { tag.appendLine(renderedText); } else { diff --git a/visualpython/js/m_apps/File.js b/visualpython/js/m_apps/File.js index ce93fc74..6a04a91c 100644 --- a/visualpython/js/m_apps/File.js +++ b/visualpython/js/m_apps/File.js @@ -35,16 +35,17 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 1; + this.config.sizeLevel = 2; this.config.checkModules = ['pd']; this.fileExtensions = { - 'csv': 'csv', - 'excel': 'xlsx', - 'json': 'json', - 'pickle': '', - 'sas': '', // xport or sas7bdat - 'spss': '' + 'csv': ['csv', 'tsv', 'txt'], + 'excel': ['xlsx', 'xls'], + 'json': ['json'], + 'pickle': [], + 'sas': [], // xport or sas7bdat + 'spss': [], + 'parquet': ['parquet'] } this.package = { @@ -68,7 +69,8 @@ define([ } this.state = { - fileExtension: 'csv', + fileType: 'csv', + fileExtension: ['csv'], selectedFile: '', selectedPath: '', vp_fileioType: 'Read', @@ -90,7 +92,8 @@ define([ 'json': 'pd_readJson', 'pickle': 'pd_readPickle', 'sas': 'pd_readSas', - 'spss': 'pd_readSpss' + 'spss': 'pd_readSpss', + 'parquet':'pd_readParquet' }, selectedType: 'csv', package: null, @@ -104,7 +107,8 @@ define([ 'csv': 'pd_toCsv', 'excel': 'pd_toExcel', 'json': 'pd_toJson', - 'pickle': 'pd_toPickle' + 'pickle': 'pd_toPickle', + 'parquet':'pd_toParquet' }, selectedType: 'csv', package: null, @@ -185,6 +189,11 @@ define([ } } + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#fileReadAs')); + } + _bindEvent() { super._bindEvent(); /** Implement binding events */ @@ -194,38 +203,74 @@ define([ that.state['vp_fileioType'] = pageType; $(that.wrapSelector('.vp-fileio-box')).hide(); $(that.wrapSelector('#vp_file' + pageType)).show(); - + + if (pageType === 'Read' && that.fileState[pageType].selectedType === 'spss') { + // show install button + that.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); + } else { + that.hideInstallButton(); + $(that.wrapSelector('.vp-spss-note')).remove(); + } + + //set fileExtensions that.fileResultState = { ...that.fileState[pageType].fileResultState }; }); + + // fileReadAs change Event, Use PyArrow + $(document).on('change', this.wrapSelector('#fileReadAs'), function() { + let isChecked = $(this).prop('checked'); + var fileioType = that.state.vp_fileioType; + var prefix = '#vp_file' + fileioType + ' '; + var selectedType = that.fileState[fileioType]['selectedType']; + var fileioTypePrefix = fileioType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } + let fileId = that.fileState[fileioType].fileTypeId[selectedType]; + + if (isChecked) { // pyArrow + fileId = "pa_" + fileioTypePrefix + selectedType[0].toUpperCase() + selectedType.slice(1); + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pa_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); + } else { // pandas + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + if (that.state.fileType != 'parquet'){ // parquet has no options area + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').show(); + } + } + + let pdLib = pandasLibrary.PANDAS_FUNCTION; + let thisPkg = JSON.parse(JSON.stringify(pdLib[fileId])); + + that.fileState[fileioType].package = thisPkg; + }); + } _bindEventByType(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + + var fileioTypePrefix = pageType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } // select file type $(this.wrapSelector(prefix + '#fileType')).change(function() { - var value = $(this).val(); - that.fileState[pageType].selectedType = value; + var fileType = $(this).val(); + that.fileState[pageType].selectedType = fileType; // reload that.renderPage(pageType); that._bindEventByType(pageType); - - if (value === 'spss') { - // show install button - that.showInstallButton(); - // show install note below File type selection - $(` - NOTE: - pyreadstat package is required to read spss file. - `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); - } else { - that.hideInstallButton(); - } }); // open file navigation @@ -237,8 +282,8 @@ define([ } let extensionList = []; - if (that.state.fileExtension !== '') { - extensionList = [ that.state.fileExtension ]; + if (that.state.fileExtension && that.state.fileExtension.length > 0) { + extensionList = that.state.fileExtension; } let fileNavi = new FileNavigation({ @@ -327,7 +372,7 @@ define([ renderPage(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + // clear $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).html(''); $(this.wrapSelector(prefix + '#vp_optionBox table tbody')).html(''); @@ -344,7 +389,7 @@ define([ ...this.fileState[pageType].fileResultState }; - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { // hide additional option box $(this.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); } else { @@ -355,7 +400,7 @@ define([ if (selectedType == 'json') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path_or_buf'); } - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path'); } } @@ -365,11 +410,21 @@ define([ // pdGen.vp_showInterfaceOnPage(this.wrapSelector('#vp_file' + pageType), thisPkg); pdGen.vp_showInterfaceOnPage(this, thisPkg, this.state, parent=('#vp_file' + pageType)); + // pyarrow can r/w parquet, csv and only read json. + if ((pageType == 'Read' && selectedType == 'json') || selectedType == 'parquet'|| selectedType == 'csv') { + // add checkbox 'Use PyArrow', next to File Type + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' Use PyArrow')) + ); + } else { + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(`File Type`)) + .append($(' ')) + ); + } + // prepend file type selector - $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( - $('').append($(`File Type`)) - .append($('')) - ); var fileTypeList = Object.keys(fileTypeObj); fileTypeList.forEach(type => { $(this.wrapSelector(prefix + '#fileType')).append( @@ -390,9 +445,9 @@ define([ .append($('')) ) } - + $(this.wrapSelector(prefix + '#fileType')).val(selectedType); - + // add file navigation button if (pageType == 'Write') { if (selectedType == 'json') { @@ -400,7 +455,7 @@ define([ com_util.formatString('

' , 'vp-file-browser-button') ); - } else if (selectedType == 'pickle') { + } else if (selectedType == 'pickle' || selectedType == 'parquet') { $(prefix + '#path').parent().html( com_util.formatString('

' , 'vp-file-browser-button') @@ -419,6 +474,18 @@ define([ , 'vp-file-browser-button') ); } + + if (pageType === 'Read' && selectedType === 'spss') { + // show install button + this.showInstallButton(); + // show install note below File type selection + $(` + NOTE: + pyreadstat package is required to read spss file. + `).insertAfter($(this.wrapSelector('#fileType')).closest('tr')); + } else { + this.hideInstallButton(); + } // encoding suggest input $(this.wrapSelector('#encoding')).replaceWith(function() { @@ -431,6 +498,18 @@ define([ suggestInput.setPlaceholder('encoding option'); return suggestInput.toTagString(); }); + + // seperator suggest input + $(this.wrapSelector('#sep')).replaceWith(function() { + // seperator list : + var sepList = [',', '|', '\\t', '\\n', ':', ';', '-', '_', '&', '/', '\\']; + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('sep'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return sepList; }); + suggestInput.setPlaceholder('Input seperator'); + return suggestInput.toTagString(); + }); } render() { @@ -493,7 +572,6 @@ define([ var result = pdGen.vp_codeGenerator(this, thisPkg, this.state, userOption.toString(), parent='#vp_fileWrite'); sbCode.append(result); } - return sbCode.toString(); } diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index 47cf0883..438dfc79 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -87,7 +87,8 @@ define([ { id: 'add_row', label: 'Add row', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.ADD_ROW }, { id: 'delete', label: 'Delete', selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP }, { id: 'rename', label: 'Rename', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RENAME }, - { id: 'asType', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'as_type', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'to_datetime', label: 'To Datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, { id: 'replace', label: 'Replace', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.REPLACE }, { id: 'discretize', label: 'Discretize', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, numeric_only: true, menuType: FRAME_EDIT_TYPE.DISCRETIZE } ] @@ -618,6 +619,7 @@ define([ case FRAME_EDIT_TYPE.RENAME: case FRAME_EDIT_TYPE.REPLACE: case FRAME_EDIT_TYPE.AS_TYPE: + case FRAME_EDIT_TYPE.TO_DATETIME: case FRAME_EDIT_TYPE.DISCRETIZE: case FRAME_EDIT_TYPE.DATA_SHIFT: case FRAME_EDIT_TYPE.SORT_INDEX: @@ -722,6 +724,10 @@ define([ $(this.wrapSelector('.vp-inner-popup-fillvalue')).focus(); return; } + } else if (type === FRAME_EDIT_TYPE.TO_DATETIME) { + if (content.newcol === '') { + $(this.wrapSelector('.vp-inner-popup-todt-new-col')).focus(); + } } // run check modules for outliers and load codes if (type === FRAME_EDIT_TYPE.FILL_OUT) { @@ -1106,6 +1112,70 @@ define([ $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); } }); + } else if (menuType === FRAME_EDIT_TYPE.TO_DATETIME) { + // bind event for selecting format + $(this.wrapSelector('.vp-inner-popup-todt-format')).on('change', function() { + let format = $(this).val(); + if (format === 'auto') { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(''); + } + + if (format === 'typing') { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', true); + } + }); + + // Add column set event + $(this.wrapSelector('.vp-inner-popup-todt-addcol')).on('click', function() { + let dateTypeList = [ // df[col].dt[{dateType}] + { label: 'Year', value: 'year' }, + { label: 'Month', value: 'month' }, + { label: 'Day', value: 'day' }, + { label: 'Date', value: 'date' }, + { label: 'DayOfWeek', value: 'dayofweek' }, + { label: 'DayOfYear', value: 'dayofyear' }, + { label: 'DaysInMonth', value: 'daysinmonth' }, + { label: 'Quarter', value: 'quarter' }, + { label: 'Time', value: 'time' }, + { label: 'Hour', value: 'hour' }, + { label: 'Minute', value: 'minute' }, + { label: 'Second', value: 'second' }, + { label: 'Nanosecond', value: 'nanosecond' }, + ]; + let dateTypeOptionTag = new com_String(); + dateTypeList.forEach(opt => { + dateTypeOptionTag.appendFormat('', opt.value, opt.label); + }); + + let addColItemTag = $(``); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-content')).append(addColItemTag); + $(addColItemTag)[0].scrollIntoView(); + + // bind event for selecting date type option + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).off('change'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).on('change', function() { + let dateTypeVal = $(this).val(); + $(this).parent().find('.vp-inner-popup-todt-addcol-colname').val(dateTypeVal); + }); + + // bind event for deleting + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).off('click'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).on('click', function() { + // delete item + $(this).closest('.vp-inner-popup-todt-addcol-item').remove(); + }); + }); } } @@ -2113,6 +2183,74 @@ define([ return content.toString(); } + renderToDatetime() { + var content = new com_String(); + let formatList = [ + { label: 'Auto', value: 'auto' }, + { label: '%Y/%m/%d', value: '%Y/%m/%d' }, + { label: '%Y-%m-%d', value: '%Y-%m-%d' }, + { label: '%y/%m/%d', value: '%y/%m/%d' }, + { label: '%y-%m-%d', value: '%y-%m-%d' }, + { label: '%d/%m/%Y', value: '%d/%m/%Y' }, + { label: '%d-%m-%Y', value: '%d-%m-%Y' }, + { label: '%d/%m/%y', value: '%d/%m/%y' }, + { label: '%d-%m-%y', value: '%d-%m-%y' }, + { label: 'Typing', value: 'typing' } + ]; + let formatOptionTag = new com_String(); + formatList.forEach(opt => { + formatOptionTag.appendFormat('', opt.value, opt.label); + }); + + content.appendFormat(` + `, this.state.selected[0].label, formatOptionTag.toString(), this.state.selected[0].label); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + renderFillNAPage() { var content = new com_String(); content.appendFormatLine('

', 'vp-inner-popup-fillna-page'); @@ -2454,6 +2592,11 @@ define([ title = 'Convert type'; content = this.renderAsType(); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + title = 'Convert to Datetime'; + size = { width: 500, height: 450 }; + content = this.renderToDatetime(); + break; case FRAME_EDIT_TYPE.FILL_NA: title = 'Fill NA'; content = this.renderFillNAPage(); @@ -2823,6 +2966,22 @@ define([ } }); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + content['format'] = $(this.wrapSelector('.vp-inner-popup-todt-format')).val(); + content['format_typing'] = $(this.wrapSelector('.vp-inner-popup-todt-format-typing')).val(); + content['dayfirst'] = $(this.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(); + content['newcol'] = $(this.wrapSelector('.vp-inner-popup-todt-new-col')).val(); + var colList = []; + var addcolItemTags = $(this.wrapSelector('.vp-inner-popup-todt-addcol-item')); + addcolItemTags && addcolItemTags.each((idx, tag) => { + let colName = $(tag).find('.vp-inner-popup-todt-addcol-colname').val(); + let dateType = $(tag).find('.vp-inner-popup-todt-addcol-type').val(); + if (colName !== '' && dateType !== '') { + colList.push({ colName: colName, dateType: dateType }); + } + }); + content['collist'] = colList; + break; case FRAME_EDIT_TYPE.DISCRETIZE: content['input'] = $(this.wrapSelector('.vp-inner-popup-input')).val(); content['inputastext'] = $(this.wrapSelector('.vp-inner-popup-inputastext')).prop('checked'); @@ -3343,6 +3502,31 @@ define([ }); code.appendFormat("{0} = {1}.astype({{2}})", tempObj, tempObj, astypeStr.toString()); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + code.appendFormat("{0}['{1}'] = pd.to_datetime({2}[{3}]", tempObj, content['newcol'], tempObj, selectedName); + let optionList = []; + if (content['format'] === 'auto') { + if (content['dayfirst'] !== '') { + optionList.push(`dayfirst=${content['dayfirst']}`); + } + } else if (content['format'] === 'typing') { + if (content['format_typing'] !== '') { + optionList.push(`format='${content['format_typing']}'`); + } + } else { + optionList.push(`format='${content['format']}'`); + } + if (optionList.length > 0) { + code.appendFormat(', {0}', optionList.join(', ')); + } + code.append(')'); + if (content['collist'].length > 0) { + content['collist'].forEach(obj => { + code.appendLine(); + code.appendFormat("{0}['{1}'] = {2}['{3}'].dt.{4}", tempObj, obj.colName, tempObj, content['newcol'], obj.dateType); + }); + } + break; case FRAME_EDIT_TYPE.DISCRETIZE: let newColumn = com_util.convertToStr(content['input'], content['inputastext']); let method = content['type']; @@ -3437,7 +3621,7 @@ define([ var indexList = data.index; var dataList = data.data; - columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value } }); + columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value, isNumeric: col.is_numeric } }); indexList = indexList.map(idx => { return { label: idx, code: idx } }); if (!more) { @@ -3453,6 +3637,12 @@ define([ while (colIdx < columnList.length) { let col = columnList[colIdx]; let colCode = col.code.slice(0, colLevIdx + 1).join(','); + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } let nextCol = columnList[colIdx + 1]; if (nextCol && nextCol.code.slice(0, colLevIdx + 1).join(',') === colCode) { colSpan++; @@ -3467,8 +3657,8 @@ define([ } else { colClass = VP_FE_TABLE_COLUMN_GROUP; } - table.appendFormatLine('{8}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, col.label[colLevIdx]); + table.appendFormatLine('{8}{9}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, colIcon, col.label[colLevIdx]); colSpan = 1; } colIdx++; @@ -3487,12 +3677,18 @@ define([ table.appendLine(''); columnList && columnList.forEach(col => { var colCode = col.code; + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } var colClass = ''; if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code).includes(colCode)) { colClass = 'selected'; } - table.appendFormatLine('{6}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, col.label); + table.appendFormatLine('{6}{7}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, colIcon, col.label); }); // // add column table.appendFormatLine('

', VP_FE_ADD_COLUMN, 'vp-icon-plus'); @@ -3713,6 +3909,7 @@ define([ DROP: 3, RENAME: 2, AS_TYPE: 10, + TO_DATETIME: 19, REPLACE: 9, DISCRETIZE: 15, diff --git a/visualpython/js/m_apps/Groupby.js b/visualpython/js/m_apps/Groupby.js index a88dcfce..7b61d561 100644 --- a/visualpython/js/m_apps/Groupby.js +++ b/visualpython/js/m_apps/Groupby.js @@ -643,7 +643,7 @@ define([ page.appendFormatLine('

', 'vp-gb-method-user'); page.appendFormatLine('{0}', 'User option'); page.appendFormatLine('', 'Type user method'); - page.appendFormatLine('{0}' + page.appendFormatLine('{0}' , 'Text'); page.appendLine(''); page.appendLine('

'); diff --git a/visualpython/js/m_apps/Import.js b/visualpython/js/m_apps/Import.js index 38b6aec6..bc0321d5 100644 --- a/visualpython/js/m_apps/Import.js +++ b/visualpython/js/m_apps/Import.js @@ -22,26 +22,27 @@ define([ const importTemplates = { 'data-analysis': [ - { i0: 'numpy', i1: 'np', type: 'module'}, - { i0: 'pandas', i1: 'pd', type: 'module'}, + { i0: 'numpy', i1: 'np', type: 'module', checked: 'checked'}, + { i0: 'pandas', i1: 'pd', type: 'module', checked: 'checked'}, { i0: 'matplotlib.pyplot', i1: 'plt', type: 'module' , include: [ '%matplotlib inline' - ] + ], checked: 'checked' }, - { i0: 'seaborn', i1: 'sns', type: 'module'}, + { i0: 'seaborn', i1: 'sns', type: 'module', checked: 'checked'}, { i0: 'plotly.express', i1: 'px', type: 'module' , include: [ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' - ], checked: false - } + ], checked: '' + }, + { i0: 'pyarrow', i1: 'pa', type: 'module', checked: ''}, ], 'machine-learning': [ - { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function' }, - { i0: 'sklearn', i1: 'metrics', type: 'function' } + { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function', checked: 'checked' }, + { i0: 'sklearn', i1: 'metrics', type: 'function', checked: 'checked' } ] } @@ -174,9 +175,9 @@ define([ let that = this; libraries && libraries.forEach((lib, idx) => { if (lib.type == 'function') { - page.appendLine(that.templateForFunction(idx, lib.i0, lib.i1, lib.checked)); + page.appendLine(that.templateForFunction(idx, lib.i0, lib.i1, (lib.checked === 'checked' || lib.checked === true))); } else { - page.appendLine(that.templateForModule(idx, lib.i0, lib.i1, lib.checked)); + page.appendLine(that.templateForModule(idx, lib.i0, lib.i1, (lib.checked === 'checked' || lib.checked === true))); } }); page.appendLine(''); @@ -245,7 +246,7 @@ define([ if (pacI0 == "") { continue; } - if (pacChecked) { + if (pacChecked === true) { if (sbCode.toString().trim().length > 0) { sbCode.appendLine(); } @@ -269,7 +270,7 @@ define([ } } - importMeta.push({ i0: pacI0, i1: pacI1, type: pacType, checked: pacChecked }); + importMeta.push({ i0: pacI0, i1: pacI1, type: pacType, checked: (pacChecked?'checked':'') }); } this.state.importMeta = importMeta; diff --git a/visualpython/js/m_apps/Instance.js b/visualpython/js/m_apps/Instance.js index 83a8a4ea..1095ef31 100644 --- a/visualpython/js/m_apps/Instance.js +++ b/visualpython/js/m_apps/Instance.js @@ -263,7 +263,8 @@ define([ let cmObj = this.getCodemirror('vp_instanceVariable'); let rightCode = (cmObj && cmObj.cm)?cmObj.cm.getValue():''; if (leftCode && leftCode != '') { - sbCode.appendFormat('{0} = {1}', leftCode, rightCode); + sbCode.appendFormatLine('{0} = {1}', leftCode, rightCode); + sbCode.append(leftCode); // show allocation (from version 2.4.10) } else { sbCode.appendFormat('{0}', rightCode); } diff --git a/visualpython/js/m_library/m_pandas/readFile.js b/visualpython/js/m_library/m_pandas/readFile.js index 296d3509..c9144db9 100644 --- a/visualpython/js/m_library/m_pandas/readFile.js +++ b/visualpython/js/m_library/m_pandas/readFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -60,7 +61,8 @@ define([ 'csv': 'pd_readCsv', 'excel': 'pd_readExcel', 'json': 'pd_readJson', - 'pickle': 'pd_readPickle' + 'pickle': 'pd_readPickle', + 'parquet': 'pd_readParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_library/m_pandas/toFile.js b/visualpython/js/m_library/m_pandas/toFile.js index 41a58ab3..08bb45c8 100644 --- a/visualpython/js/m_library/m_pandas/toFile.js +++ b/visualpython/js/m_library/m_pandas/toFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -60,7 +61,8 @@ define([ 'csv': 'pd_toCsv', 'excel': 'pd_toExcel', 'json': 'pd_toJson', - 'pickle': 'pd_toPickle' + 'pickle': 'pd_toPickle', + 'parquet': 'pd_toParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index e01e7201..10b9625f 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -57,13 +57,13 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], - 'ETC': ['prep-simple-imputer', 'make-column-transformer'] + 'ETC': ['prep-simple-imputer', 'prep-smote', 'make-column-transformer'] } this.mctEstimator = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], } diff --git a/visualpython/js/m_ml/FitPredict.js b/visualpython/js/m_ml/FitPredict.js index 22055a30..cdd979b9 100644 --- a/visualpython/js/m_ml/FitPredict.js +++ b/visualpython/js/m_ml/FitPredict.js @@ -427,7 +427,6 @@ define([ description: 'Transform labels to normalized encoding.' } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -443,6 +442,32 @@ define([ } } } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 1d450a0a..8eea2b53 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -409,16 +409,30 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['input'], usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', @@ -514,6 +528,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js index 0db5f041..6dfc1989 100644 --- a/visualpython/js/m_ml/Pipeline.js +++ b/visualpython/js/m_ml/Pipeline.js @@ -63,6 +63,9 @@ define([ - Fit - Transform - Predict + - Fit and Predict + - Fit and Transform + - Fit and Resample */ this.templateList = { 'data-prep': { @@ -73,9 +76,10 @@ define([ * ml_* is pre-defined app * pp_* is defined only for Pipeline */ - { name: 'ml_dataPrep', label: 'Data Prep', useApp: true }, + { name: 'ml_dataPrep', label: 'Data Prep', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_resample'] }, { name: 'pp_fit', label: 'Fit' }, - { name: 'pp_transform', label: 'Transform' } + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_resample', label: 'Fit and Resample' } ] }, 'regression': { @@ -286,7 +290,7 @@ define([ that.state.modelTypeName = modelTypeName; // show fit / predict / transform depends on model selection - let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform']; + let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform', 'fit_resample']; let actions = that.modelEditor.getAction(modelTypeName); defaultActions.forEach(actKey => { if (actions[actKey] === undefined) { @@ -308,6 +312,10 @@ define([ } else { $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); } + } else if (actKey === 'fit_resample') { + // for SMOTE: show fit_resample only + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); } } $(that.wrapSelector('.vp-pp-item')).removeClass('vp-last-visible'); @@ -580,6 +588,9 @@ define([ case 'pp_fit_transform': tag = this.templateForOptionPage(actions['fit_transform']); break; + case 'pp_fit_resample': + tag = this.templateForOptionPage(actions['fit_resample']); + break; } $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(`

${tag}

@@ -680,6 +691,9 @@ define([ case 'pp_fit_transform': actObj = actions['fit_transform']; break; + case 'pp_fit_resample': + actObj = actions['fit_resample']; + break; } let code = new com_String(); @@ -721,7 +735,7 @@ define([ code.appendLine(); code.appendLine(); } - if (useApp) { + if (useApp === true) { let appCode = app.generateCode(); if (appCode instanceof Array) { appCode = appCode.join('\n'); diff --git a/visualpython/js/m_ml/dataSplit.js b/visualpython/js/m_ml/dataSplit.js index bfa2aac2..a8c9417f 100644 --- a/visualpython/js/m_ml/dataSplit.js +++ b/visualpython/js/m_ml/dataSplit.js @@ -175,7 +175,7 @@ define([ options.appendFormat(', shuffle={0}', shuffle); } if (shuffle != 'False' && stratify && stratify != '') { - options.appendFormat(', startify={0}', stratify); + options.appendFormat(', stratify={0}', stratify); } let code = new com_String(); diff --git a/visualpython/js/m_stats/ProbDist.js b/visualpython/js/m_stats/ProbDist.js index ecd629ae..245cd9f6 100644 --- a/visualpython/js/m_stats/ProbDist.js +++ b/visualpython/js/m_stats/ProbDist.js @@ -86,14 +86,24 @@ define([ // discrete option $(that.wrapSelector('.vp-pd-display-option.dist')).show(); - // set size to 100 - $(that.wrapSelector('#size')).val(100); - that.state.size = 100; - - // hide continuous action - if (that.state.action === 'stats-to-pvalue' || that.state.action === 'pvalue-to-stats') { - $(that.wrapSelector('#action')).val('random-number'); - $(that.wrapSelector('#action')).trigger('change'); + // set size to 1000 + $(that.wrapSelector('#size')).val(1000); + that.state.size = 1000; + + // hide distribution plot for multinomial + if (distType === 'multinomial') { + $(that.wrapSelector('.vp-pd-display-option.dist-plot')).hide(); + // hide other actions + if (that.state.action !== 'random-number') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } + } else { + // hide continuous action + if (that.state.action === 'stats-to-pvalue' || that.state.action === 'pvalue-to-stats') { + $(that.wrapSelector('#action')).val('random-number'); + $(that.wrapSelector('#action')).trigger('change'); + } } } else { // continuous option diff --git a/visualpython/js/m_visualize/Seaborn.js b/visualpython/js/m_visualize/Seaborn.js index d172d998..bea6cfd1 100644 --- a/visualpython/js/m_visualize/Seaborn.js +++ b/visualpython/js/m_visualize/Seaborn.js @@ -32,6 +32,7 @@ define([ this.config.dataview = false; this.config.size = { width: 1064, height: 550 }; + this.config.autoScroll = false; this.config.checkModules = ['plt', 'sns']; this.config.docs = 'https://seaborn.pydata.org/index.html'; @@ -201,6 +202,7 @@ define([ $(that.wrapSelector('#kde')).closest('.sb-option').show(); $(that.wrapSelector('#stat')).closest('.sb-option').show(); } else if (chartType == 'barplot') { + $(that.wrapSelector('#orient')).closest('.sb-option').show(); $(that.wrapSelector('#showValues')).closest('.sb-option').show(); $(that.wrapSelector('#errorbar')).closest('.sb-option').show(); if (that.state.setXY === false) { @@ -547,6 +549,7 @@ define([ $(page).find('#kde').closest('.sb-option').show(); $(page).find('#stat').closest('.sb-option').show(); } else if (this.state.chartType == 'barplot') { + $(page).find('#orient').closest('.sb-option').show(); $(page).find('#showValues').closest('.sb-option').show(); $(page).find('#errorbar').closest('.sb-option').show(); if (this.state.setXY === false) { @@ -890,7 +893,7 @@ define([ generateCode(preview=false) { let { - chartType, data, x, y, setXY, hue, kde, stat, + chartType, data, x, y, setXY, hue, orient, kde, stat, showValues, showValuesPrecision, errorbar, sortType, sortBy, sortHue, sortHueText, userOption='', diff --git a/visualpython/lib/codemirror/lib/codemirror.css b/visualpython/lib/codemirror/lib/codemirror.css index f4d5718a..0a80f68d 100644 --- a/visualpython/lib/codemirror/lib/codemirror.css +++ b/visualpython/lib/codemirror/lib/codemirror.css @@ -3,7 +3,7 @@ .CodeMirror { /* Set height, width, borders, and global font properties here */ font-family: monospace; - height: 300px; + /* height: 300px; */ color: black; direction: ltr; } diff --git a/visualpython/python/userCommand.py b/visualpython/python/userCommand.py index 586eea36..cac0152f 100644 --- a/visualpython/python/userCommand.py +++ b/visualpython/python/userCommand.py @@ -100,7 +100,7 @@ def vp_create_feature_importances(model, X_train=None, sort=False): feature_names = [ 'X{}'.format(i) for i in range(len(model.feature_importances_)) ] df_i = _vp_pd.DataFrame(model.feature_importances_, index=feature_names, columns=['Feature_importance']) - df_i['Percentage'] = 100 * (df_i['Feature_importance'] / df_i['Feature_importance'].max()) + df_i['Percentage'] = 100 * df_i['Feature_importance'] if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) df_i = df_i.round(2) @@ -123,6 +123,41 @@ def vp_plot_feature_importances(model, X_train=None, sort=False, top_count=0): _vp_plt.show() ###### +# Visual Python: Machine Learning > Model Info +###### +def vp_create_permutation_importances(model, X_train, y_train, scoring=None, sort=False): + from sklearn.inspection import permutation_importance + if isinstance(X_train, _vp_pd.core.frame.DataFrame): + feature_names = X_train.columns + else: + feature_names = [ 'X{}'.format(i) for i in range(len(model.feature_importances_)) ] + + imp = permutation_importance(model, X_train, y_train, scoring) + + df_i = _vp_pd.DataFrame(imp['importances_mean'], index=feature_names, columns=['Feature_importance']) + df_i['Percentage'] = 100 * df_i['Feature_importance'] + if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) + df_i = df_i.round(2) + + return df_i +###### +# Visual Python: Machine Learning > Model Info +###### +def vp_plot_permutation_importances(model, X_train, y_train, scoring=None, sort=False, top_count=0): + df_i = vp_create_permutation_importances(model, X_train, y_train, scoring, sort) + + if sort: + if top_count > 0: + df_i['Percentage'].sort_values().tail(top_count).plot(kind='barh') + else: + df_i['Percentage'].sort_values().plot(kind='barh') + else: + df_i['Percentage'].plot(kind='barh') + _vp_plt.xlabel('Feature importance Percentage') + _vp_plt.ylabel('Features') + + _vp_plt.show() +###### # Visual Python: Visualization > Seaborn ###### def vp_seaborn_show_values(axs, precision=1, space=0.01):