From 7f10ea778f794f84587f2baf5b7b7a506136aa8b Mon Sep 17 00:00:00 2001 From: Minku Koo Date: Sun, 27 Aug 2023 02:22:27 +0900 Subject: [PATCH 01/37] Fixed #233 - Update File, Import - Add pyarrow at import app - Add file format Parquet at file read and write --- visualpython/js/com/com_Config.js | 10 ++++++++-- visualpython/js/m_apps/Import.js | 3 ++- visualpython/js/m_library/m_pandas/readFile.js | 6 ++++-- visualpython/js/m_library/m_pandas/toFile.js | 6 ++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/visualpython/js/com/com_Config.js b/visualpython/js/com/com_Config.js index bbc6fb83..91fbb022 100644 --- a/visualpython/js/com/com_Config.js +++ b/visualpython/js/com/com_Config.js @@ -75,7 +75,8 @@ define([ 'import matplotlib.pyplot as plt', '%matplotlib inline', 'import seaborn as sns', - 'import plotly.express as px' + 'import plotly.express as px', + 'import pyarrow as pa' ], 'matplotlib customizing': [ 'import matplotlib.pyplot as plt', @@ -132,7 +133,8 @@ define([ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' ] - } + }, + { library: 'pyarrow', alias:'pa' }, ] } @@ -208,6 +210,10 @@ define([ 'statsmodels.api': { code: 'import statsmodels.api as sm', type: 'package' + }, + 'pyarrow': { + code: 'import pyarrow as pa', + type: 'package' } } diff --git a/visualpython/js/m_apps/Import.js b/visualpython/js/m_apps/Import.js index 38b6aec6..645114d8 100644 --- a/visualpython/js/m_apps/Import.js +++ b/visualpython/js/m_apps/Import.js @@ -37,7 +37,8 @@ define([ 'from plotly.offline import init_notebook_mode', 'init_notebook_mode(connected=True)' ], checked: false - } + }, + { i0: 'pyarrow', i1: 'pa', type: 'module', checked: false}, ], 'machine-learning': [ { i0: 'sklearn.model_selection', i1: 'train_test_split', type: 'function' }, diff --git a/visualpython/js/m_library/m_pandas/readFile.js b/visualpython/js/m_library/m_pandas/readFile.js index 296d3509..c9144db9 100644 --- a/visualpython/js/m_library/m_pandas/readFile.js +++ b/visualpython/js/m_library/m_pandas/readFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -60,7 +61,8 @@ define([ 'csv': 'pd_readCsv', 'excel': 'pd_readExcel', 'json': 'pd_readJson', - 'pickle': 'pd_readPickle' + 'pickle': 'pd_readPickle', + 'parquet': 'pd_readParquet' }, selectedType: 'csv', package: null diff --git a/visualpython/js/m_library/m_pandas/toFile.js b/visualpython/js/m_library/m_pandas/toFile.js index 41a58ab3..08bb45c8 100644 --- a/visualpython/js/m_library/m_pandas/toFile.js +++ b/visualpython/js/m_library/m_pandas/toFile.js @@ -48,7 +48,8 @@ define([ 'csv': 'csv', 'excel': 'xlsx', 'json': 'json', - 'pickle': '' + 'pickle': '', + 'parquet': 'parquet' } this.dataPath = 'https://raw.githubusercontent.com/visualpython/visualpython/main/visualpython/data/sample_csv/'; this.fileResultState = { @@ -60,7 +61,8 @@ define([ 'csv': 'pd_toCsv', 'excel': 'pd_toExcel', 'json': 'pd_toJson', - 'pickle': 'pd_toPickle' + 'pickle': 'pd_toPickle', + 'parquet': 'pd_toParquet' }, selectedType: 'csv', package: null From 9b4f24aac81e266f14556129d6f9832af72dadd6 Mon Sep 17 00:00:00 2001 From: Minku Koo Date: Sun, 27 Aug 2023 22:37:50 +0900 Subject: [PATCH 02/37] Fixed #233 - Add 'Use PyArrow' checkbox - Add 'Use PyArrow' checkbox - When checked 'Use PyArrow', change the generated code that used pyarrow. - Add parquet to file type --- visualpython/data/m_library/pandasLibrary.js | 183 +++++++++++++++++++ visualpython/js/m_apps/File.js | 88 +++++++-- 2 files changed, 255 insertions(+), 16 deletions(-) diff --git a/visualpython/data/m_library/pandasLibrary.js b/visualpython/data/m_library/pandasLibrary.js index b18ed0c9..008b8af5 100644 --- a/visualpython/data/m_library/pandasLibrary.js +++ b/visualpython/data/m_library/pandasLibrary.js @@ -6825,6 +6825,189 @@ define([ }, ] }, + // *** + "pd_toParquet": { + "name": "To Parquet", + "library": "pandas", + "description": "DataFrame/Series to Parquet file", + "code": "${i0}.to_parquet(${path}${etc})", + "options": [ + { + "name": "i0", + "label": "DataFrame", + "required": true, + "component": [ + "data_select" + ], + "var_type": [ + "DataFrame", + "Series" + ] + }, + { + "name": "path", + "label": "File path/variable", + "required": true, + "type": "text" + } + ] + }, + "pd_readParquet": { + "name": "Read Parquet", + "library": "pandas", + "description": "Parquet to pandas object", + "code": "${o0} = pd.read_parquet(${i0}${etc})", + "options": [ + { + "name": "i0", + "label": "File path/object", + "required": true, + "type": "text", + "component": [ + "file" + ] + }, + { + "name": "o0", + "label": "Allocate to", + "output": true, + "component": [ + "input" + ], + "value": "vp_df" + }, + ] + }, + "pa_readCsv": { + "name": "Read Csv as pyarrow", + "library": "pyarrow", + "description": "Csv to pandas object", + "code": "${o0} = pa.csv.read_csv(${i0}${etc}).to_pandas()", + "options": [ + { + "name": "i0", + "label": "File path/object", + "required": true, + "type": "text", + "component": [ + "file" + ] + }, + { + "name": "o0", + "label": "Allocate to", + "output": true, + "component": [ + "input" + ], + "value": "vp_df" + } + ] + }, + "pa_toCsv": { + "name": "To Csv as pyarrow", + "library": "pyarrow", + "description": "DataFrame/Series to csv file", + "code": "pa.csv.write_csv(${i0}, ${path})", + "options": [ + { + "name": "i0", + "label": "DataFrame", + "required": true, + "component": [ + "data_select" + ], + "var_type": [ + "DataFrame", + "Series" + ] + }, + { + "name": "path", + "label": "File path/variable", + "required": true, + "type": "text" + } + ] + }, + "pa_readJson": { + "name": "Read Json as pyarrow", + "library": "pyarrow", + "description": "Json to pyarrow object", + "code": "${o0} = pa.json.read_json(${i0}${etc}).to_pandas()", + "options": [ + { + "name": "i0", + "label": "File path/object", + "required": true, + "type": "text", + "component": [ + "file" + ] + }, + { + "name": "o0", + "label": "Allocate to", + "output": true, + "component": [ + "input" + ], + "value": "vp_df" + } + ] + }, + "pa_readParquet": { + "name": "Read Parquet as pyarrow", + "library": "pyarrow", + "description": "Parquet to pandas object", + "code": "${o0} = pa.parquet.read_table(${i0}${etc}).to_pandas()", + "options": [ + { + "name": "i0", + "label": "File path/object", + "required": true, + "type": "text", + "component": [ + "file" + ] + }, + { + "name": "o0", + "label": "Allocate to", + "output": true, + "component": [ + "input" + ], + "value": "vp_df" + } + ] + }, + "pa_toParquet": { + "name": "To Parquet as pyarrow", + "library": "pyarrow", + "description": "DataFrame/Series to Parquet file", + "code": "pa.parquet.write_table(${i0}, ${path})", + "options": [ + { + "name": "i0", + "label": "DataFrame", + "required": true, + "component": [ + "data_select" + ], + "var_type": [ + "DataFrame", + "Series" + ] + }, + { + "name": "path", + "label": "File path/variable", + "required": true, + "type": "text" + } + ] + }, } return { diff --git a/visualpython/js/m_apps/File.js b/visualpython/js/m_apps/File.js index ce93fc74..2fe1454e 100644 --- a/visualpython/js/m_apps/File.js +++ b/visualpython/js/m_apps/File.js @@ -44,7 +44,8 @@ define([ 'json': 'json', 'pickle': '', 'sas': '', // xport or sas7bdat - 'spss': '' + 'spss': '', + 'parquet':'parquet' } this.package = { @@ -90,7 +91,8 @@ define([ 'json': 'pd_readJson', 'pickle': 'pd_readPickle', 'sas': 'pd_readSas', - 'spss': 'pd_readSpss' + 'spss': 'pd_readSpss', + 'parquet':'pd_readParquet' }, selectedType: 'csv', package: null, @@ -104,7 +106,8 @@ define([ 'csv': 'pd_toCsv', 'excel': 'pd_toExcel', 'json': 'pd_toJson', - 'pickle': 'pd_toPickle' + 'pickle': 'pd_toPickle', + 'parquet':'pd_toParquet' }, selectedType: 'csv', package: null, @@ -194,22 +197,64 @@ define([ that.state['vp_fileioType'] = pageType; $(that.wrapSelector('.vp-fileio-box')).hide(); $(that.wrapSelector('#vp_file' + pageType)).show(); - + + //set fileExtensions that.fileResultState = { ...that.fileState[pageType].fileResultState }; }); + + // fileReadAs change Event, Use PyArrow + $(document).on('change', this.wrapSelector('#fileReadAs'), function() { + let isChecked = $(this).prop('checked'); + var fileioType = that.state.vp_fileioType; + var prefix = '#vp_file' + fileioType + ' '; + var selectedFileFormat = that.fileState[fileioType].selectedType; + var fileioTypePrefix = fileioType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } + + if(isChecked){ // pyArrow + that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pa_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); + } + else{ // pandas + that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + if (that.state.fileExtension != 'parquet'){ // parquet has no options area + $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').show(); + } + } + + var fileTypeObj = that.fileState[fileioType]['fileTypeId']; + var selectedType = that.fileState[fileioType]['selectedType']; + let fileId = fileTypeObj[selectedType]; + let pdLib = pandasLibrary.PANDAS_FUNCTION; + let thisPkg = JSON.parse(JSON.stringify(pdLib[fileId])); + + that.fileState[fileioType].package = thisPkg; + }); + } _bindEventByType(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + + var fileioTypePrefix = pageType.toLowerCase(); + if(fileioTypePrefix == 'write'){ + fileioTypePrefix = "to"; + } + var selectedFileFormat = that.fileState[pageType].selectedType; // select file type $(this.wrapSelector(prefix + '#fileType')).change(function() { var value = $(this).val(); that.fileState[pageType].selectedType = value; + + // Whenever change the file type, change to default pandas + that.fileState[pageType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + // reload that.renderPage(pageType); @@ -327,7 +372,7 @@ define([ renderPage(pageType) { var that = this; var prefix = '#vp_file' + pageType + ' '; - + // clear $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).html(''); $(this.wrapSelector(prefix + '#vp_optionBox table tbody')).html(''); @@ -344,7 +389,7 @@ define([ ...this.fileState[pageType].fileResultState }; - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { // hide additional option box $(this.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); } else { @@ -355,7 +400,7 @@ define([ if (selectedType == 'json') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path_or_buf'); } - if (selectedType == 'pickle') { + if (selectedType == 'pickle' || selectedType == 'parquet') { this.fileResultState.pathInputId = this.wrapSelector(prefix + '#path'); } } @@ -365,11 +410,22 @@ define([ // pdGen.vp_showInterfaceOnPage(this.wrapSelector('#vp_file' + pageType), thisPkg); pdGen.vp_showInterfaceOnPage(this, thisPkg, this.state, parent=('#vp_file' + pageType)); + // pyarrow can r/w parquet, csv and only read json. + if ((pageType == 'Read' && selectedType == 'json') || selectedType == 'parquet'|| selectedType == 'csv') { + // add checkbox 'Use PyArrow', next to File Type + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(``)) + .append($(' ')) + ); + } + else{ + $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( + $('').append($(``)) + .append($(' ')) + ); + } + // prepend file type selector - $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( - $('').append($(``)) - .append($('')) - ); var fileTypeList = Object.keys(fileTypeObj); fileTypeList.forEach(type => { $(this.wrapSelector(prefix + '#fileType')).append( @@ -377,6 +433,7 @@ define([ ); }); + // prepend user option let hasAllocateTo = $(this.wrapSelector(prefix + '#o0')).length > 0; if (hasAllocateTo) { @@ -390,9 +447,9 @@ define([ .append($('')) ) } - + $(this.wrapSelector(prefix + '#fileType')).val(selectedType); - + // add file navigation button if (pageType == 'Write') { if (selectedType == 'json') { @@ -400,7 +457,7 @@ define([ com_util.formatString('
' , 'vp-file-browser-button') ); - } else if (selectedType == 'pickle') { + } else if (selectedType == 'pickle' || selectedType == 'parquet') { $(prefix + '#path').parent().html( com_util.formatString('
' , 'vp-file-browser-button') @@ -493,7 +550,6 @@ define([ var result = pdGen.vp_codeGenerator(this, thisPkg, this.state, userOption.toString(), parent='#vp_fileWrite'); sbCode.append(result); } - return sbCode.toString(); } From 1bd07bd9773fb0891a4f2020fad7846ede68ccb9 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:37:12 +0900 Subject: [PATCH 03/37] Move SMOTE position to ETC --- visualpython/js/com/com_Config.js | 4 ++-- visualpython/js/m_ml/DataPrep.js | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/visualpython/js/com/com_Config.js b/visualpython/js/com/com_Config.js index 2a43b6b4..3b136011 100644 --- a/visualpython/js/com/com_Config.js +++ b/visualpython/js/com/com_Config.js @@ -1109,11 +1109,11 @@ define([ Config.ML_DATA_DICT = { 'Data Preparation': [ /** Encoding */ - 'OneHotEncoder', 'LabelEncoder', 'OrdinalEncoder', 'TargetEncoder', 'SMOTE', + 'OneHotEncoder', 'LabelEncoder', 'OrdinalEncoder', 'TargetEncoder', /** Scaling */ 'StandardScaler', 'RobustScaler', 'MinMaxScaler', 'Normalizer', 'FunctionTransformer', 'PolynomialFeatures', 'KBinsDiscretizer', /** ETC */ - 'SimpleImputer', 'ColumnTransformer' + 'SimpleImputer', 'SMOTE', 'ColumnTransformer' ], 'Regression': [ 'LinearRegression', 'Ridge', 'Lasso', 'ElasticNet', 'SVR', 'DecisionTreeRegressor', 'RandomForestRegressor', 'GradientBoostingRegressor', 'XGBRegressor', 'LGBMRegressor', 'CatBoostRegressor', diff --git a/visualpython/js/m_ml/DataPrep.js b/visualpython/js/m_ml/DataPrep.js index e01e7201..10b9625f 100644 --- a/visualpython/js/m_ml/DataPrep.js +++ b/visualpython/js/m_ml/DataPrep.js @@ -57,13 +57,13 @@ define([ this.modelConfig = ML_LIBRARIES; this.modelTypeList = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], - 'ETC': ['prep-simple-imputer', 'make-column-transformer'] + 'ETC': ['prep-simple-imputer', 'prep-smote', 'make-column-transformer'] } this.mctEstimator = { - 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target', 'prep-smote'], + 'Encoding': ['prep-onehot', 'prep-label', 'prep-ordinal', 'prep-target'], 'Scaling': ['prep-standard', 'prep-robust', 'prep-minmax', 'prep-normalizer', 'prep-func-trsfrm-log', 'prep-func-trsfrm-exp', 'prep-poly-feat', 'prep-kbins-discretizer'], } From cf2e0d6f4a6eba2fa09f7e76483ae29c361926e1 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:37:32 +0900 Subject: [PATCH 04/37] Edit to display output on Instance --- visualpython/js/m_apps/Instance.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/visualpython/js/m_apps/Instance.js b/visualpython/js/m_apps/Instance.js index 83a8a4ea..1095ef31 100644 --- a/visualpython/js/m_apps/Instance.js +++ b/visualpython/js/m_apps/Instance.js @@ -263,7 +263,8 @@ define([ let cmObj = this.getCodemirror('vp_instanceVariable'); let rightCode = (cmObj && cmObj.cm)?cmObj.cm.getValue():''; if (leftCode && leftCode != '') { - sbCode.appendFormat('{0} = {1}', leftCode, rightCode); + sbCode.appendFormatLine('{0} = {1}', leftCode, rightCode); + sbCode.append(leftCode); // show allocation (from version 2.4.10) } else { sbCode.appendFormat('{0}', rightCode); } From 7416969c7c626e11cbecce1dd661ee7d3cd372bb Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:37:52 +0900 Subject: [PATCH 05/37] Set default user method's input to non-text --- visualpython/js/m_apps/Groupby.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualpython/js/m_apps/Groupby.js b/visualpython/js/m_apps/Groupby.js index a88dcfce..7b61d561 100644 --- a/visualpython/js/m_apps/Groupby.js +++ b/visualpython/js/m_apps/Groupby.js @@ -643,7 +643,7 @@ define([ page.appendFormatLine('
', 'vp-gb-method-user'); page.appendFormatLine('', 'User option'); page.appendFormatLine('', 'Type user method'); - page.appendFormatLine('' + page.appendFormatLine('' , 'Text'); page.appendLine(''); page.appendLine('
'); From 616ddb946d36a57eaf0e5f854c81c7ec4ec8dfda Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:38:10 +0900 Subject: [PATCH 06/37] Add option to SMOTE --- visualpython/data/m_ml/mlLibrary.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/visualpython/data/m_ml/mlLibrary.js b/visualpython/data/m_ml/mlLibrary.js index e5bf2205..5f574e33 100644 --- a/visualpython/data/m_ml/mlLibrary.js +++ b/visualpython/data/m_ml/mlLibrary.js @@ -175,11 +175,12 @@ define([ name: 'SMOTE', install: '!pip install imblearn', import: 'from imblearn.over_sampling import SMOTE', - code: 'SMOTE(${random_state}${k_neighbors}${etc})', + code: 'SMOTE(${random_state}${k_neighbors}${sampling_strategy}${etc})', returnType: 'SMOTE', options: [ { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, - { name: 'k_neighbors', component: ['input_number'], default: 5, usePair: true } + { name: 'k_neighbors', component: ['input_number'], default: 5, usePair: true }, + { name: 'sampling_strategy', component: ['input'], placeholder: "'auto'", usePair: true } ] }, /** Data Preparation - Scaling */ From 1e5a67b77c883988e0a57963e57fcfa1cd6d4fac Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:38:26 +0900 Subject: [PATCH 07/37] Fix MultiSelector bug on variable mode --- visualpython/js/com/component/MultiSelector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index 016eacc0..b8f36ba2 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -112,7 +112,7 @@ define([ }); break; case 'variable': - this._getVariableList(this.type, function(dataList) { + this._getVariableList(this.varType, function(dataList) { that._executeCallback(dataList); }); break; From e46aa1e06c22021dba77d47f3658fccacd05ba72 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:38:41 +0900 Subject: [PATCH 08/37] Edit SMOTE actions --- visualpython/html/m_ml/fitPredict.html | 2 +- visualpython/js/m_ml/FitPredict.js | 27 +++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/visualpython/html/m_ml/fitPredict.html b/visualpython/html/m_ml/fitPredict.html index 4b9498c9..c444d2d8 100644 --- a/visualpython/html/m_ml/fitPredict.html +++ b/visualpython/html/m_ml/fitPredict.html @@ -38,7 +38,7 @@
Options
-
+
diff --git a/visualpython/js/m_ml/FitPredict.js b/visualpython/js/m_ml/FitPredict.js index 22055a30..cdd979b9 100644 --- a/visualpython/js/m_ml/FitPredict.js +++ b/visualpython/js/m_ml/FitPredict.js @@ -427,7 +427,6 @@ define([ description: 'Transform labels to normalized encoding.' } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -443,6 +442,32 @@ define([ } } } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { From ffec38cd1471981d1058dbad9aeb7a6f0541544b Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Wed, 13 Sep 2023 17:38:52 +0900 Subject: [PATCH 09/37] Add SMOTE model info --- visualpython/html/m_ml/modelInfo.html | 2 +- visualpython/js/m_ml/ModelInfo.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/visualpython/html/m_ml/modelInfo.html b/visualpython/html/m_ml/modelInfo.html index 690efb8b..02badc27 100644 --- a/visualpython/html/m_ml/modelInfo.html +++ b/visualpython/html/m_ml/modelInfo.html @@ -38,7 +38,7 @@
Options
-
+
diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 1d450a0a..1a530783 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -514,6 +514,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] From 8001e7c0c96e8475b04dc78a0bad23d7111bd83c Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 18 Sep 2023 09:11:12 +0900 Subject: [PATCH 10/37] Fix feature importances code --- visualpython/python/userCommand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualpython/python/userCommand.py b/visualpython/python/userCommand.py index 586eea36..4e5be7cd 100644 --- a/visualpython/python/userCommand.py +++ b/visualpython/python/userCommand.py @@ -100,7 +100,7 @@ def vp_create_feature_importances(model, X_train=None, sort=False): feature_names = [ 'X{}'.format(i) for i in range(len(model.feature_importances_)) ] df_i = _vp_pd.DataFrame(model.feature_importances_, index=feature_names, columns=['Feature_importance']) - df_i['Percentage'] = 100 * (df_i['Feature_importance'] / df_i['Feature_importance'].max()) + df_i['Percentage'] = 100 * df_i['Feature_importance'] if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) df_i = df_i.round(2) From 112792c093eac21073c749cdceccac41606c8540 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 18 Sep 2023 09:12:31 +0900 Subject: [PATCH 11/37] Edit permutation importances and add plot --- visualpython/js/com/component/ModelEditor.js | 18 ++++++++-- visualpython/js/m_ml/ModelInfo.js | 18 ++++++++-- visualpython/python/userCommand.py | 35 ++++++++++++++++++++ 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index a31fe8af..d1fe6ced 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -479,16 +479,30 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['input'], usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', diff --git a/visualpython/js/m_ml/ModelInfo.js b/visualpython/js/m_ml/ModelInfo.js index 1a530783..8eea2b53 100644 --- a/visualpython/js/m_ml/ModelInfo.js +++ b/visualpython/js/m_ml/ModelInfo.js @@ -409,16 +409,30 @@ define([ name: 'permutation_importance', label: 'Permutation importance', import: 'from sklearn.inspection import permutation_importance', - code: '${importance_allocate} = permutation_importance(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${random_state}${etc})', + code: '${importance_allocate} = vp_create_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort})', description: 'Permutation importance for feature evaluation.', options: [ { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, { name: 'scoring', component: ['input'], usePair: true }, - { name: 'random_state', component: ['input_number'], placeholder: '123', usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, { name: 'importance_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'importances' } ] }, + 'plot_permutation_importance': { + name: 'plot_permutation_importance', + label: 'Plot permutation importance', + import: 'from sklearn.inspection import permutation_importance', + code: 'vp_plot_permutation_importances(${model}, ${importance_featureData}, ${importance_targetData}${scoring}${sort}${top_count})', + description: 'Permutation importance for feature evaluation.', + options: [ + { name: 'importance_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'importance_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' }, + { name: 'scoring', component: ['input'], usePair: true }, + { name: 'sort', label: 'Sort data', component: ['bool_checkbox'], value: true, usePair: true }, + { name: 'top_count', label: 'Top count', component: ['input_number'], min: 0, usePair: true } + ] + }, 'feature_importances': { name: 'feature_importances', label: 'Feature importances', diff --git a/visualpython/python/userCommand.py b/visualpython/python/userCommand.py index 4e5be7cd..cac0152f 100644 --- a/visualpython/python/userCommand.py +++ b/visualpython/python/userCommand.py @@ -123,6 +123,41 @@ def vp_plot_feature_importances(model, X_train=None, sort=False, top_count=0): _vp_plt.show() ###### +# Visual Python: Machine Learning > Model Info +###### +def vp_create_permutation_importances(model, X_train, y_train, scoring=None, sort=False): + from sklearn.inspection import permutation_importance + if isinstance(X_train, _vp_pd.core.frame.DataFrame): + feature_names = X_train.columns + else: + feature_names = [ 'X{}'.format(i) for i in range(len(model.feature_importances_)) ] + + imp = permutation_importance(model, X_train, y_train, scoring) + + df_i = _vp_pd.DataFrame(imp['importances_mean'], index=feature_names, columns=['Feature_importance']) + df_i['Percentage'] = 100 * df_i['Feature_importance'] + if sort: df_i.sort_values(by='Feature_importance', ascending=False, inplace=True) + df_i = df_i.round(2) + + return df_i +###### +# Visual Python: Machine Learning > Model Info +###### +def vp_plot_permutation_importances(model, X_train, y_train, scoring=None, sort=False, top_count=0): + df_i = vp_create_permutation_importances(model, X_train, y_train, scoring, sort) + + if sort: + if top_count > 0: + df_i['Percentage'].sort_values().tail(top_count).plot(kind='barh') + else: + df_i['Percentage'].sort_values().plot(kind='barh') + else: + df_i['Percentage'].plot(kind='barh') + _vp_plt.xlabel('Feature importance Percentage') + _vp_plt.ylabel('Features') + + _vp_plt.show() +###### # Visual Python: Visualization > Seaborn ###### def vp_seaborn_show_values(axs, precision=1, space=0.01): From 97457ffe5c4d8f70cd2f45047495472b413b0a48 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 18 Sep 2023 09:13:16 +0900 Subject: [PATCH 12/37] Edit to consider SMOTE --- visualpython/js/com/component/ModelEditor.js | 47 ++++++++++++++++++-- visualpython/js/m_ml/Pipeline.js | 20 +++++++-- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/visualpython/js/com/component/ModelEditor.js b/visualpython/js/com/component/ModelEditor.js index d1fe6ced..ae109fd9 100644 --- a/visualpython/js/com/component/ModelEditor.js +++ b/visualpython/js/com/component/ModelEditor.js @@ -143,7 +143,6 @@ define([ description: 'Transform labels to normalized encoding.' } } - if (modelType != 'ColumnTransformer') { actions = { ...actions, @@ -159,6 +158,32 @@ define([ } } } + if (modelType === 'SMOTE') { + actions = { + 'fit': { + name: 'fit', + label: 'Fit', + code: '${model}.fit(${fit_featureData}, ${fit_targetData})', + description: 'Check inputs and statistics of the sampler.', + options: [ + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + }, + 'fit_resample': { + name: 'fit_resample', + label: 'Fit and resample', + code: '${fit_res_allocateX}, ${fit_res_allocatey} = ${model}.fit_resample(${fit_res_featureData}, ${fit_res_targetData})', + description: 'Resample the dataset.', + options: [ + { name: 'fit_res_allocateX', label: 'Allocate feature', component: ['input'], placeholder: 'New variable', value: 'X_res' }, + { name: 'fit_res_allocatey', label: 'Allocate target', component: ['input'], placeholder: 'New variable', value: 'y_res' }, + { name: 'fit_res_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_res_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train' } + ] + } + } + } break; case 'Regression': actions = { @@ -407,10 +432,11 @@ define([ 'fit': { name: 'fit', label: 'Fit', - code: '${model}.fit(${fit_featureData})', + code: '${model}.fit(${fit_featureData}${fit_targetData})', description: 'Run fit with all sets of parameters.', options: [ - { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' } + { name: 'fit_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_train' }, + { name: 'fit_targetData', label: 'Target Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'y_train', usePair: true, pairKey: 'y' } ] }, 'predict': { @@ -419,7 +445,7 @@ define([ code: '${pred_allocate} = ${model}.predict(${pred_featureData})', description: 'Call predict on the estimator with the best found parameters.', options: [ - { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X' }, + { name: 'pred_featureData', label: 'Feature Data', component: ['data_select'], var_type: ['DataFrame', 'Series', 'ndarray', 'list', 'dict'], value: 'X_test' }, { name: 'pred_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'pred' } ] }, @@ -598,6 +624,19 @@ define([ } } } + if (modelType === 'SMOTE') { + infos = { + 'get_feature_names_out': { + name: 'get_feature_names_out', + label: 'Get feature names', + code: '${feature_names_allocate} = ${model}.get_feature_names_out()', + description: 'Get output feature names for transformation.', + options: [ + { name: 'feature_names_allocate', label: 'Allocate to', component: ['input'], placeholder: 'New variable', value: 'features' } + ] + } + } + } infos = { ...infos, 'get_params': defaultInfos['get_params'] diff --git a/visualpython/js/m_ml/Pipeline.js b/visualpython/js/m_ml/Pipeline.js index 0db5f041..7dbab1d7 100644 --- a/visualpython/js/m_ml/Pipeline.js +++ b/visualpython/js/m_ml/Pipeline.js @@ -63,6 +63,9 @@ define([ - Fit - Transform - Predict + - Fit and Predict + - Fit and Transform + - Fit and Resample */ this.templateList = { 'data-prep': { @@ -73,9 +76,10 @@ define([ * ml_* is pre-defined app * pp_* is defined only for Pipeline */ - { name: 'ml_dataPrep', label: 'Data Prep', useApp: true }, + { name: 'ml_dataPrep', label: 'Data Prep', useApp: true, child: ['pp_fit', 'pp_transform', 'pp_fit_resample'] }, { name: 'pp_fit', label: 'Fit' }, - { name: 'pp_transform', label: 'Transform' } + { name: 'pp_transform', label: 'Transform' }, + { name: 'pp_fit_resample', label: 'Fit and Resample' } ] }, 'regression': { @@ -286,7 +290,7 @@ define([ that.state.modelTypeName = modelTypeName; // show fit / predict / transform depends on model selection - let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform']; + let defaultActions = ['fit', 'predict', 'transform', 'fit_predict', 'fit_transform', 'fit_resample']; let actions = that.modelEditor.getAction(modelTypeName); defaultActions.forEach(actKey => { if (actions[actKey] === undefined) { @@ -308,6 +312,10 @@ define([ } else { $(that.wrapSelector(`.vp-pp-item[data-name="pp_${actKey}"]`)).hide(); } + } else if (actKey === 'fit_resample') { + // for SMOTE: show fit_resample only + $(that.wrapSelector(`.vp-pp-item[data-name="pp_fit"]`)).hide(); + $(that.wrapSelector(`.vp-pp-item[data-name="pp_transform"]`)).hide(); } } $(that.wrapSelector('.vp-pp-item')).removeClass('vp-last-visible'); @@ -580,6 +588,9 @@ define([ case 'pp_fit_transform': tag = this.templateForOptionPage(actions['fit_transform']); break; + case 'pp_fit_resample': + tag = this.templateForOptionPage(actions['fit_resample']); + break; } $(this.wrapSelector(`.vp-pp-step-page[data-name="${appId}"]`)).html(`
${tag}
@@ -680,6 +691,9 @@ define([ case 'pp_fit_transform': actObj = actions['fit_transform']; break; + case 'pp_fit_resample': + actObj = actions['fit_resample']; + break; } let code = new com_String(); From 36c75a8d831d65390ea2b9c33b8c0a4cd391a0ca Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 18 Sep 2023 09:14:10 +0900 Subject: [PATCH 13/37] Edit FileNavigation to get use multi-extensions --- .../js/com/component/FileNavigation.js | 103 ++++++++++-------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/visualpython/js/com/component/FileNavigation.js b/visualpython/js/com/component/FileNavigation.js index a5190aff..506ffc0a 100644 --- a/visualpython/js/com/component/FileNavigation.js +++ b/visualpython/js/com/component/FileNavigation.js @@ -84,6 +84,10 @@ define([ this.pathStackPointer = -1; this.pathStack = []; this.currentFileList = []; + this.selectedExt = ''; + if (this.state.extensions.length > 0) { + this.selectedExt = this.state.extensions[0]; + } this.pathState = { parentPath: '', @@ -280,6 +284,32 @@ define([ // clear body $(this.wrapSelector('.fileNavigationPage-body')).html(''); + /** + * Filter file/dir which included in this.state.extensions + */ + if (Array.isArray(this.state.extensions) && this.state.extensions.length > 0 && this.state.extensions.toString() !== '') { + fileList = fileList.filter((data, index) => { + if (index == 0) { + return true; + } + + if (data.type && data.type == 'dir') { + // if directory, just show + return true; + } else if (data.name) { + var extension = data.name.substring(data.name.lastIndexOf('.') + 1); + // if (that.state.extensions.includes(extension)) { + if (that.selectedExt === '' || extension === that.selectedExt) { + return true; + } else { + return false; + } + } else { + return false; + } + }); + } + // render file items let dirArr = []; let fileArr = []; @@ -411,12 +441,12 @@ define([ page.appendFormatLine('' , 'vp_fileNavigationInput', 'New File Name', this.state.fileName); page.appendFormatLine(''); page.appendFormatLine('', 'vp-filenavi-btn', 'select', 'Select'); @@ -430,6 +460,13 @@ define([ that.handleSelectFile(filePath, fileName); }); + // bind file extension change event + $(this.wrapSelector('#vp_fileNavigationExt')).on('change', function() { + let ext = $(this).val(); + that.selectedExt = ext; + + that.renderFileList(); + }); // bind save cancel event $(this.wrapSelector('.vp-filenavi-btn')).on('click', function() { let menu = $(this).data('menu'); @@ -477,10 +514,10 @@ define([ let that = this; /** Implement after rendering */ // if save mode - if (this.state.type == 'save') { - // render saving box - this.renderSaveBox(); - } + // if (this.state.type == 'save') { + // render saving box + this.renderSaveBox(); + // } // get current path this.getCurrentDirectory().then(function(currentPath) { @@ -527,20 +564,21 @@ define([ //============================================================================ // Set selection result //============================================================================ - if (this.state.type == 'save') { - // add as saving file - this.setSelectedFile(fileInput, pathInput); - } else { - // Manage result using finish function - let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented - let status = true; - let error = null; - vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); - this.state.finish(filesPath, status, error); + this.setSelectedFile(fileInput, pathInput); + // if (this.state.type == 'save') { + // // add as saving file + // this.setSelectedFile(fileInput, pathInput); + // } else { + // // Manage result using finish function + // let filesPath = [{ file: fileInput, path: pathInput }]; //FIXME: fix it if multiple selection implemented + // let status = true; + // let error = null; + // vpLog.display(VP_LOG_TYPE.DEVELOP, 'fileNavigation finished', filesPath, status, error); + // this.state.finish(filesPath, status, error); - // remove and close file navigation - this.close(); - } + // // remove and close file navigation + // this.close(); + // } } getCurrentDirectory() { return vpKernel.getCurrentDirectory(); @@ -588,31 +626,6 @@ define([ return a - b; }); - /** - * Filter file/dir which included in this.state.extensions - */ - if (Array.isArray(that.state.extensions) && that.state.extensions.length > 0 && that.state.extensions.toString() !== '') { - filtered_varList = filtered_varList.filter((data, index) => { - if (index == 0) { - return true; - } - - if (data.type && data.type == 'dir') { - // if file, just show - return true; - } else if (data.name) { - var extension = data.name.substring(data.name.lastIndexOf('.') + 1); - if (that.state.extensions.includes(extension)) { - return true; - } else { - return false; - } - } else { - return false; - } - }); - } - vpLog.display(VP_LOG_TYPE.DEVELOP, 'FileNavigation - getFileList: ', filtered_varList); var { currentDirStr, currentRelativePathStr } = that.splitPathStrAndSetStack(dirObj, filtered_varList); From 20ba6f3fc976c7765aa1f22111ccbc63b3a9270f Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 18 Sep 2023 09:15:53 +0900 Subject: [PATCH 14/37] Edit File to use multi-extensions --- .../js/com/component/PackageManager.js | 1 + visualpython/js/m_apps/File.js | 108 +++++++++++------- 2 files changed, 66 insertions(+), 43 deletions(-) diff --git a/visualpython/js/com/component/PackageManager.js b/visualpython/js/com/component/PackageManager.js index 00afe14c..a808315e 100644 --- a/visualpython/js/com/component/PackageManager.js +++ b/visualpython/js/com/component/PackageManager.js @@ -47,6 +47,7 @@ define([ this.packageLibTemplate = { 'numpy': { pipName: 'numpy' }, 'pandas': { pipName: 'pandas' }, + 'pyarrow': { pipName: 'pyarrow' }, 'matplotlib': { pipName: 'matplotlib' }, 'seaborn': { pipName: 'seaborn' }, 'plotly': { pipName: 'plotly' }, diff --git a/visualpython/js/m_apps/File.js b/visualpython/js/m_apps/File.js index 2fe1454e..2f88eef3 100644 --- a/visualpython/js/m_apps/File.js +++ b/visualpython/js/m_apps/File.js @@ -35,17 +35,17 @@ define([ super._init(); /** Write codes executed before rendering */ this.config.dataview = false; - this.config.sizeLevel = 1; + this.config.sizeLevel = 2; this.config.checkModules = ['pd']; this.fileExtensions = { - 'csv': 'csv', - 'excel': 'xlsx', - 'json': 'json', - 'pickle': '', - 'sas': '', // xport or sas7bdat - 'spss': '', - 'parquet':'parquet' + 'csv': ['csv', 'tsv'], + 'excel': ['xlsx', 'xls'], + 'json': ['json'], + 'pickle': [], + 'sas': [], // xport or sas7bdat + 'spss': [], + 'parquet': ['parquet'] } this.package = { @@ -69,7 +69,8 @@ define([ } this.state = { - fileExtension: 'csv', + fileType: 'csv', + fileExtension: ['csv'], selectedFile: '', selectedPath: '', vp_fileioType: 'Read', @@ -188,6 +189,11 @@ define([ } } + _unbindEvent() { + super._unbindEvent(); + $(document).off('change', this.wrapSelector('#fileReadAs')); + } + _bindEvent() { super._bindEvent(); /** Implement binding events */ @@ -198,6 +204,19 @@ define([ $(that.wrapSelector('.vp-fileio-box')).hide(); $(that.wrapSelector('#vp_file' + pageType)).show(); + if (pageType === 'Read' && that.fileState[pageType].selectedType === 'spss') { + // show install button + that.showInstallButton(); + // show install note below File type selection + $(` + + + `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); + } else { + that.hideInstallButton(); + $(that.wrapSelector('.vp-spss-note')).remove(); + } + //set fileExtensions that.fileResultState = { @@ -210,26 +229,24 @@ define([ let isChecked = $(this).prop('checked'); var fileioType = that.state.vp_fileioType; var prefix = '#vp_file' + fileioType + ' '; - var selectedFileFormat = that.fileState[fileioType].selectedType; + var selectedType = that.fileState[fileioType]['selectedType']; var fileioTypePrefix = fileioType.toLowerCase(); if(fileioTypePrefix == 'write'){ fileioTypePrefix = "to"; } + let fileId = that.fileState[fileioType].fileTypeId[selectedType]; - if(isChecked){ // pyArrow - that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pa_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + if (isChecked) { // pyArrow + fileId = "pa_" + fileioTypePrefix + selectedType[0].toUpperCase() + selectedType.slice(1); + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pa_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').hide(); - } - else{ // pandas - that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); - if (that.state.fileExtension != 'parquet'){ // parquet has no options area + } else { // pandas + // that.fileState[fileioType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); + if (that.state.fileType != 'parquet'){ // parquet has no options area $(that.wrapSelector(prefix + '#vp_optionBox')).closest('.vp-accordian-container').show(); } } - var fileTypeObj = that.fileState[fileioType]['fileTypeId']; - var selectedType = that.fileState[fileioType]['selectedType']; - let fileId = fileTypeObj[selectedType]; let pdLib = pandasLibrary.PANDAS_FUNCTION; let thisPkg = JSON.parse(JSON.stringify(pdLib[fileId])); @@ -246,31 +263,14 @@ define([ if(fileioTypePrefix == 'write'){ fileioTypePrefix = "to"; } - var selectedFileFormat = that.fileState[pageType].selectedType; // select file type $(this.wrapSelector(prefix + '#fileType')).change(function() { - var value = $(this).val(); - that.fileState[pageType].selectedType = value; - - // Whenever change the file type, change to default pandas - that.fileState[pageType].fileTypeId[that.state.fileExtension] = "pd_" + fileioTypePrefix + selectedFileFormat[0].toUpperCase() + selectedFileFormat.slice(1); - + var fileType = $(this).val(); + that.fileState[pageType].selectedType = fileType; // reload that.renderPage(pageType); that._bindEventByType(pageType); - - if (value === 'spss') { - // show install button - that.showInstallButton(); - // show install note below File type selection - $(` - - - `).insertAfter($(that.wrapSelector('#fileType')).closest('tr')); - } else { - that.hideInstallButton(); - } }); // open file navigation @@ -282,8 +282,8 @@ define([ } let extensionList = []; - if (that.state.fileExtension !== '') { - extensionList = [ that.state.fileExtension ]; + if (that.state.fileExtension && that.state.fileExtension.length > 0) { + extensionList = that.state.fileExtension; } let fileNavi = new FileNavigation({ @@ -417,8 +417,7 @@ define([ $('').append($(``)) .append($(' ')) ); - } - else{ + } else { $(this.wrapSelector(prefix + '#vp_inputOutputBox table tbody')).prepend( $('').append($(``)) .append($(' ')) @@ -433,7 +432,6 @@ define([ ); }); - // prepend user option let hasAllocateTo = $(this.wrapSelector(prefix + '#o0')).length > 0; if (hasAllocateTo) { @@ -476,6 +474,18 @@ define([ , 'vp-file-browser-button') ); } + + if (pageType === 'Read' && selectedType === 'spss') { + // show install button + this.showInstallButton(); + // show install note below File type selection + $(` + + + `).insertAfter($(this.wrapSelector('#fileType')).closest('tr')); + } else { + this.hideInstallButton(); + } // encoding suggest input $(this.wrapSelector('#encoding')).replaceWith(function() { @@ -488,6 +498,18 @@ define([ suggestInput.setPlaceholder('encoding option'); return suggestInput.toTagString(); }); + + // seperator suggest input + $(this.wrapSelector('#sep')).replaceWith(function() { + // seperator list : + var sepList = [',', '|', '\\t', '\\n', ':', ';', '-', '_', '&', '/', '\\']; + var suggestInput = new SuggestInput(); + suggestInput.setComponentID('sep'); + suggestInput.addClass('vp-input vp-state'); + suggestInput.setSuggestList(function() { return sepList; }); + suggestInput.setPlaceholder('Input seperator'); + return suggestInput.toTagString(); + }); } render() { From 1b8cc36876ebef7a57d51ce2d8ed86df6cadfb4e Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 18 Sep 2023 09:16:26 +0900 Subject: [PATCH 15/37] Add orient option to barplot on Seaborn --- visualpython/data/m_visualize/seabornLibrary.js | 3 ++- visualpython/html/m_visualize/seaborn.html | 8 ++++++++ visualpython/js/m_visualize/Seaborn.js | 5 ++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/visualpython/data/m_visualize/seabornLibrary.js b/visualpython/data/m_visualize/seabornLibrary.js index a5b185c8..f0a91faf 100644 --- a/visualpython/data/m_visualize/seabornLibrary.js +++ b/visualpython/data/m_visualize/seabornLibrary.js @@ -148,13 +148,14 @@ define([ }, 'barplot': { name: 'Bar Plot', - code: '${allocateTo} = sns.barplot(${data}${x}${y}${hue}${etc})', + code: '${allocateTo} = sns.barplot(${data}${x}${y}${hue}${orient}${etc})', description: 'Show point estimates and confidence intervals as rectangular bars.', options: [ { name: 'data', component: ['var_select'], var_type: ['DataFrame', 'Series', 'list'], usePair: true }, { name: 'x', component: ['col_select'], usePair: true }, { name: 'y', component: ['col_select'], usePair: true }, { name: 'hue', component: ['col_select'], usePair: true }, + { name: 'orient', component: ['option_select'], usePair: true }, { name: 'allocateTo', label: 'Allocate To', component: ['input'], usePair: true } ] }, diff --git a/visualpython/html/m_visualize/seaborn.html b/visualpython/html/m_visualize/seaborn.html index d39d2d1b..08e99bd7 100644 --- a/visualpython/html/m_visualize/seaborn.html +++ b/visualpython/html/m_visualize/seaborn.html @@ -100,6 +100,14 @@
+
+ + +
- - + + From 6d79f5abbd877456def8be0817dea9e2249db097 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Thu, 21 Sep 2023 17:45:38 +0900 Subject: [PATCH 25/37] Add hover title on multiselector columns --- visualpython/js/com/component/MultiSelector.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/visualpython/js/com/component/MultiSelector.js b/visualpython/js/com/component/MultiSelector.js index b8f36ba2..d9d3d9fa 100644 --- a/visualpython/js/com/component/MultiSelector.js +++ b/visualpython/js/com/component/MultiSelector.js @@ -367,9 +367,9 @@ define([ let infoStr = ''; if (mode === 'columns') { if (data.isNumeric === true) { - iconStr = ''; + iconStr = ''; } else { - iconStr = ''; + iconStr = ''; } } else if (mode === 'variable') { infoStr = ` | ${data.type}`; @@ -401,9 +401,9 @@ define([ let infoStr = ''; if (mode === 'columns') { if (data.isNumeric === true) { - iconStr = ''; + iconStr = ''; } else { - iconStr = ''; + iconStr = ''; } } else if (mode === 'variable') { infoStr = ` | ${data.type}`; From 795c2f5219529ea9f0517bad028e611f1a6d7b36 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Thu, 21 Sep 2023 17:45:55 +0900 Subject: [PATCH 26/37] Add txt extension --- visualpython/js/m_apps/File.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualpython/js/m_apps/File.js b/visualpython/js/m_apps/File.js index 2f88eef3..6a04a91c 100644 --- a/visualpython/js/m_apps/File.js +++ b/visualpython/js/m_apps/File.js @@ -39,7 +39,7 @@ define([ this.config.checkModules = ['pd']; this.fileExtensions = { - 'csv': ['csv', 'tsv'], + 'csv': ['csv', 'tsv', 'txt'], 'excel': ['xlsx', 'xls'], 'json': ['json'], 'pickle': [], From 5a235184e26e9065c0bd61d2273a0ced93e69094 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Thu, 21 Sep 2023 17:46:23 +0900 Subject: [PATCH 27/37] Edit PopupComponent to focus on limited tags --- visualpython/js/com/component/PopupComponent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualpython/js/com/component/PopupComponent.js b/visualpython/js/com/component/PopupComponent.js index 311a6b0b..60b37260 100644 --- a/visualpython/js/com/component/PopupComponent.js +++ b/visualpython/js/com/component/PopupComponent.js @@ -1328,7 +1328,7 @@ define([ $(this.wrapSelector('.vp-inner-popup-box')).show(); // focus on first input - $(this.wrapSelector('.vp-inner-popup-box input:not(:disabled):visible:first')).focus(); + $(this.wrapSelector('.vp-inner-popup-box input:not(:readonly):not(:disabled):visible:first')).focus(); // disable Jupyter key com_interface.disableOtherShortcut(); } From a7e732813cb5ca904f8fbc96b2e326c05d2be640 Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Thu, 21 Sep 2023 17:46:48 +0900 Subject: [PATCH 28/37] Add to_datetime to Frame app --- visualpython/css/m_apps/frame.css | 17 +++ visualpython/js/m_apps/Frame.js | 209 +++++++++++++++++++++++++++++- 2 files changed, 220 insertions(+), 6 deletions(-) diff --git a/visualpython/css/m_apps/frame.css b/visualpython/css/m_apps/frame.css index 0731b958..8661f4af 100644 --- a/visualpython/css/m_apps/frame.css +++ b/visualpython/css/m_apps/frame.css @@ -145,6 +145,7 @@ top: 0; background-color: var(--vp-background-color); border-bottom: 1px solid var(--vp-border-gray-color); + border-right: 1px solid var(--vp-border-gray-color); text-align: right; text-overflow: ellipsis; @@ -168,6 +169,10 @@ /* background: var(--vp-light-gray-color); */ /* background: rgba(66, 165, 245, 0.2); */ } +.vp-fe-table-column-isnumeric { + float: left; + margin-right: 5px; +} /* Row Hover */ .vp-fe-table tbody tr:hover { @@ -302,6 +307,18 @@ float: right; display: inline-block; } +/* to datetime */ +.vp-inner-popup-todt-addcol-content { + display: grid; + row-gap: 5px; + max-height: 105px; +} +.vp-inner-popup-todt-addcol-head, +.vp-inner-popup-todt-addcol-item { + display: grid; + grid-template-columns: 160px 160px auto; + column-gap: 5px; +} /* UDF Editor - CodeMirror */ .vp-fr-subset-box { diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index 47cf0883..6452ddf6 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -87,7 +87,8 @@ define([ { id: 'add_row', label: 'Add row', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.ADD_ROW }, { id: 'delete', label: 'Delete', selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP }, { id: 'rename', label: 'Rename', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RENAME }, - { id: 'asType', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'as_type', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, + { id: 'to_datetime', label: 'To datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, { id: 'replace', label: 'Replace', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.REPLACE }, { id: 'discretize', label: 'Discretize', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, numeric_only: true, menuType: FRAME_EDIT_TYPE.DISCRETIZE } ] @@ -618,6 +619,7 @@ define([ case FRAME_EDIT_TYPE.RENAME: case FRAME_EDIT_TYPE.REPLACE: case FRAME_EDIT_TYPE.AS_TYPE: + case FRAME_EDIT_TYPE.TO_DATETIME: case FRAME_EDIT_TYPE.DISCRETIZE: case FRAME_EDIT_TYPE.DATA_SHIFT: case FRAME_EDIT_TYPE.SORT_INDEX: @@ -1106,6 +1108,73 @@ define([ $(that.wrapSelector('.vp-inner-popup-fill-row')).hide(); } }); + } else if (menuType === FRAME_EDIT_TYPE.TO_DATETIME) { + // bind event for selecting format + $(this.wrapSelector('.vp-inner-popup-todt-format')).on('change', function() { + let format = $(this).val(); + if (format === 'auto') { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).prop('disabled', true); + $(that.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(''); + } + + if (format === 'typing') { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', false); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-format-typing')).prop('disabled', true); + } + }); + + // bind event for checking add column + $(this.wrapSelector('.vp-inner-popup-todt-use-addcol')).on('change', function() { + let checked = $(this).prop('checked'); + if (checked === true) { + $(that.wrapSelector('.vp-inner-popup-todt-addcol-box')).show(); + } else { + $(that.wrapSelector('.vp-inner-popup-todt-addcol-box')).hide(); + } + }); + + // Add column set event + $(this.wrapSelector('.vp-inner-popup-todt-addcol')).on('click', function() { + let dateTypeList = [ // df[col].dt[{dateType}] + { label: 'Year', value: 'year' }, + { label: 'Month', value: 'month' }, + { label: 'Day', value: 'day' }, + { label: 'Date', value: 'date' }, + { label: 'DayOfWeek', value: 'dayofweek' }, + { label: 'DayOfYear', value: 'dayofyear' }, + { label: 'DaysInMonth', value: 'daysinmonth' }, + { label: 'Quarter', value: 'quarter' }, + { label: 'Time', value: 'time' }, + { label: 'Hour', value: 'hour' }, + { label: 'Minute', value: 'minute' }, + { label: 'Second', value: 'second' }, + { label: 'Nanosecond', value: 'nanosecond' }, + ]; + let dateTypeOptionTag = new com_String(); + dateTypeList.forEach(opt => { + dateTypeOptionTag.appendFormat('', opt.value, opt.label); + }); + + let addColItemTag = $(`
+ + + +
`); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-content')).append(addColItemTag); + $(addColItemTag)[0].scrollIntoView(); + + // bind event for deleting + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).off('click'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).on('click', function() { + // delete item + $(this).closest('.vp-inner-popup-todt-addcol-item').remove(); + }); + }); } } @@ -2113,6 +2182,75 @@ define([ return content.toString(); } + renderToDatetime() { + var content = new com_String(); + let formatList = [ + { label: 'Auto', value: 'auto' }, + { label: 'Year', value: '%Y' }, + { label: 'Month', value: '%m' }, + { label: 'Day', value: '%d' }, + { label: 'Day Of Week', value: '%w' }, + { label: '%Y/%m/%d', value: '%Y/%m/%d' }, + { label: '%Y-%m-%d', value: '%Y-%m-%d' }, + { label: '%d/%m/%Y', value: '%d/%m/%Y' }, + { label: '%d-%m-%Y', value: '%d-%m-%Y' }, + { label: 'Typing', value: 'typing' }, + ]; + let formatOptionTag = new com_String(); + formatList.forEach(opt => { + formatOptionTag.appendFormat('', opt.value, opt.label); + }); + + content.appendFormat(`
+
+ + +
+
+ +
+ + +
+
+
+ + +
+
+
+ +
+ +
+ `, this.state.selected[0].label, formatOptionTag.toString(), ); + + // set content + $(this.wrapSelector('.vp-inner-popup-body')).html(content.toString()); + return content.toString(); + } + renderFillNAPage() { var content = new com_String(); content.appendFormatLine('
', 'vp-inner-popup-fillna-page'); @@ -2454,6 +2592,11 @@ define([ title = 'Convert type'; content = this.renderAsType(); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + title = 'Convert to datetime'; + size = { width: 500, height: 450 }; + content = this.renderToDatetime(); + break; case FRAME_EDIT_TYPE.FILL_NA: title = 'Fill NA'; content = this.renderFillNAPage(); @@ -2823,6 +2966,22 @@ define([ } }); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + content['format'] = $(this.wrapSelector('.vp-inner-popup-todt-format')).val(); + content['format_typing'] = $(this.wrapSelector('.vp-inner-popup-todt-format-typing')).val(); + content['dayfirst'] = $(this.wrapSelector('.vp-inner-popup-todt-dayfirst')).val(); + content['use_addcol'] = $(this.wrapSelector('.vp-inner-popup-todt-use-addcol')).prop('checked'); + var colList = []; + var addcolItemTags = $(this.wrapSelector('.vp-inner-popup-todt-addcol-item')); + addcolItemTags && addcolItemTags.each((idx, tag) => { + let colName = $(tag).find('.vp-inner-popup-todt-addcol-colname').val(); + let dateType = $(tag).find('.vp-inner-popup-todt-addcol-type').val(); + if (colName !== '' && dateType !== '') { + colList.push({ colName: colName, dateType: dateType }); + } + }); + content['collist'] = colList; + break; case FRAME_EDIT_TYPE.DISCRETIZE: content['input'] = $(this.wrapSelector('.vp-inner-popup-input')).val(); content['inputastext'] = $(this.wrapSelector('.vp-inner-popup-inputastext')).prop('checked'); @@ -3343,6 +3502,31 @@ define([ }); code.appendFormat("{0} = {1}.astype({{2}})", tempObj, tempObj, astypeStr.toString()); break; + case FRAME_EDIT_TYPE.TO_DATETIME: + code.appendFormat("{0}[{1}] = pd.to_datetime({2}[{3}]", tempObj, selectedName, tempObj, selectedName); + let optionList = []; + if (content['format'] === 'auto') { + if (content['dayfirst'] !== '') { + optionList.push(`dayfirst=${content['dayfirst']}`); + } + } else if (content['format'] === 'typing') { + if (content['format_typing'] !== '') { + optionList.push(`format='${content['format_typing']}'`); + } + } else { + optionList.push(`format='${content['format']}'`); + } + if (optionList.length > 0) { + code.appendFormat(', {0}', optionList.join(', ')); + } + code.append(')'); + if (content['use_addcol'] === true && content['collist'].length > 0) { + content['collist'].forEach(obj => { + code.appendLine(); + code.appendFormat("{0}['{1}'] = {2}[{3}].dt.{4}", tempObj, obj.colName, tempObj, selectedName, obj.dateType); + }); + } + break; case FRAME_EDIT_TYPE.DISCRETIZE: let newColumn = com_util.convertToStr(content['input'], content['inputastext']); let method = content['type']; @@ -3437,7 +3621,7 @@ define([ var indexList = data.index; var dataList = data.data; - columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value } }); + columnList = columnList.map(col => { return { label: col.label, type: col.dtype, code: col.value, isNumeric: col.is_numeric } }); indexList = indexList.map(idx => { return { label: idx, code: idx } }); if (!more) { @@ -3453,6 +3637,12 @@ define([ while (colIdx < columnList.length) { let col = columnList[colIdx]; let colCode = col.code.slice(0, colLevIdx + 1).join(','); + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } let nextCol = columnList[colIdx + 1]; if (nextCol && nextCol.code.slice(0, colLevIdx + 1).join(',') === colCode) { colSpan++; @@ -3467,8 +3657,8 @@ define([ } else { colClass = VP_FE_TABLE_COLUMN_GROUP; } - table.appendFormatLine('{8}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, col.label[colLevIdx]); + table.appendFormatLine('{8}{9}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label[colLevIdx-1], col.label[colLevIdx], colClass, selected, colSpan, colIcon, col.label[colLevIdx]); colSpan = 1; } colIdx++; @@ -3487,12 +3677,18 @@ define([ table.appendLine(''); columnList && columnList.forEach(col => { var colCode = col.code; + var colIcon = ''; + if (col.isNumeric === true) { + colIcon = ''; + } else { + colIcon = ''; + } var colClass = ''; if (that.state.axis == FRAME_AXIS.COLUMN && that.state.selected.map(col=>col.code).includes(colCode)) { colClass = 'selected'; } - table.appendFormatLine('{6}' - , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, col.label); + table.appendFormatLine('{6}{7}' + , colCode, FRAME_AXIS.COLUMN, col.type, col.label, VP_FE_TABLE_COLUMN, colClass, colIcon, col.label); }); // // add column table.appendFormatLine('
', VP_FE_ADD_COLUMN, 'vp-icon-plus'); @@ -3713,6 +3909,7 @@ define([ DROP: 3, RENAME: 2, AS_TYPE: 10, + TO_DATETIME: 19, REPLACE: 9, DISCRETIZE: 15, From c95a6bec9690965f51055afadb3beb9d9af66ffe Mon Sep 17 00:00:00 2001 From: minjk-bl Date: Mon, 25 Sep 2023 16:34:11 +0900 Subject: [PATCH 29/37] Edit to_datetime ui and functions on Frame app --- visualpython/css/m_apps/frame.css | 4 +- visualpython/js/m_apps/Frame.js | 66 +++++++++++++++---------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/visualpython/css/m_apps/frame.css b/visualpython/css/m_apps/frame.css index 8661f4af..360fbadb 100644 --- a/visualpython/css/m_apps/frame.css +++ b/visualpython/css/m_apps/frame.css @@ -170,8 +170,10 @@ /* background: rgba(66, 165, 245, 0.2); */ } .vp-fe-table-column-isnumeric { - float: left; margin-right: 5px; + vertical-align: middle; + height: 15px; + line-height: 15px; } /* Row Hover */ diff --git a/visualpython/js/m_apps/Frame.js b/visualpython/js/m_apps/Frame.js index 6452ddf6..438dfc79 100644 --- a/visualpython/js/m_apps/Frame.js +++ b/visualpython/js/m_apps/Frame.js @@ -88,7 +88,7 @@ define([ { id: 'delete', label: 'Delete', selection: FRAME_SELECT_TYPE.MULTI, menuType: FRAME_EDIT_TYPE.DROP }, { id: 'rename', label: 'Rename', selection: FRAME_SELECT_TYPE.NONE, menuType: FRAME_EDIT_TYPE.RENAME }, { id: 'as_type', label: 'As type', selection: FRAME_SELECT_TYPE.NONE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.AS_TYPE }, - { id: 'to_datetime', label: 'To datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, + { id: 'to_datetime', label: 'To Datetime', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.TO_DATETIME }, { id: 'replace', label: 'Replace', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, menuType: FRAME_EDIT_TYPE.REPLACE }, { id: 'discretize', label: 'Discretize', selection: FRAME_SELECT_TYPE.SINGLE, axis: FRAME_AXIS.COLUMN, numeric_only: true, menuType: FRAME_EDIT_TYPE.DISCRETIZE } ] @@ -724,6 +724,10 @@ define([ $(this.wrapSelector('.vp-inner-popup-fillvalue')).focus(); return; } + } else if (type === FRAME_EDIT_TYPE.TO_DATETIME) { + if (content.newcol === '') { + $(this.wrapSelector('.vp-inner-popup-todt-new-col')).focus(); + } } // run check modules for outliers and load codes if (type === FRAME_EDIT_TYPE.FILL_OUT) { @@ -1126,16 +1130,6 @@ define([ } }); - // bind event for checking add column - $(this.wrapSelector('.vp-inner-popup-todt-use-addcol')).on('change', function() { - let checked = $(this).prop('checked'); - if (checked === true) { - $(that.wrapSelector('.vp-inner-popup-todt-addcol-box')).show(); - } else { - $(that.wrapSelector('.vp-inner-popup-todt-addcol-box')).hide(); - } - }); - // Add column set event $(this.wrapSelector('.vp-inner-popup-todt-addcol')).on('click', function() { let dateTypeList = [ // df[col].dt[{dateType}] @@ -1159,7 +1153,7 @@ define([ }); let addColItemTag = $(`
- + @@ -1167,6 +1161,13 @@ define([
`); $(that.wrapSelector('.vp-inner-popup-todt-addcol-content')).append(addColItemTag); $(addColItemTag)[0].scrollIntoView(); + + // bind event for selecting date type option + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).off('change'); + $(that.wrapSelector('.vp-inner-popup-todt-addcol-type')).on('change', function() { + let dateTypeVal = $(this).val(); + $(this).parent().find('.vp-inner-popup-todt-addcol-colname').val(dateTypeVal); + }); // bind event for deleting $(that.wrapSelector('.vp-inner-popup-todt-addcol-del')).off('click'); @@ -2186,15 +2187,15 @@ define([ var content = new com_String(); let formatList = [ { label: 'Auto', value: 'auto' }, - { label: 'Year', value: '%Y' }, - { label: 'Month', value: '%m' }, - { label: 'Day', value: '%d' }, - { label: 'Day Of Week', value: '%w' }, { label: '%Y/%m/%d', value: '%Y/%m/%d' }, { label: '%Y-%m-%d', value: '%Y-%m-%d' }, + { label: '%y/%m/%d', value: '%y/%m/%d' }, + { label: '%y-%m-%d', value: '%y-%m-%d' }, { label: '%d/%m/%Y', value: '%d/%m/%Y' }, { label: '%d-%m-%Y', value: '%d-%m-%Y' }, - { label: 'Typing', value: 'typing' }, + { label: '%d/%m/%y', value: '%d/%m/%y' }, + { label: '%d-%m-%y', value: '%d-%m-%y' }, + { label: 'Typing', value: 'typing' } ]; let formatOptionTag = new com_String(); formatList.forEach(opt => { @@ -2224,15 +2225,14 @@ define([
-
- +
+ +
-