Skip to content

Commit 1134d44

Browse files
minjk-blminjk-bl
authored
Merge pull request #238 from Minku-Koo/devops
Fixed #233 - Using parquet and pyArrow, when R/W files
2 parents ee84a90 + 9b4f24a commit 1134d44

File tree

6 files changed

+273
-23
lines changed

6 files changed

+273
-23
lines changed

‎visualpython/data/m_library/pandasLibrary.js

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6880,6 +6880,189 @@ define([
68806880
},
68816881
]
68826882
},
6883+
// ***
6884+
"pd_toParquet": {
6885+
"name": "To Parquet",
6886+
"library": "pandas",
6887+
"description": "DataFrame/Series to Parquet file",
6888+
"code": "${i0}.to_parquet(${path}${etc})",
6889+
"options": [
6890+
{
6891+
"name": "i0",
6892+
"label": "DataFrame",
6893+
"required": true,
6894+
"component": [
6895+
"data_select"
6896+
],
6897+
"var_type": [
6898+
"DataFrame",
6899+
"Series"
6900+
]
6901+
},
6902+
{
6903+
"name": "path",
6904+
"label": "File path/variable",
6905+
"required": true,
6906+
"type": "text"
6907+
}
6908+
]
6909+
},
6910+
"pd_readParquet": {
6911+
"name": "Read Parquet",
6912+
"library": "pandas",
6913+
"description": "Parquet to pandas object",
6914+
"code": "${o0} = pd.read_parquet(${i0}${etc})",
6915+
"options": [
6916+
{
6917+
"name": "i0",
6918+
"label": "File path/object",
6919+
"required": true,
6920+
"type": "text",
6921+
"component": [
6922+
"file"
6923+
]
6924+
},
6925+
{
6926+
"name": "o0",
6927+
"label": "Allocate to",
6928+
"output": true,
6929+
"component": [
6930+
"input"
6931+
],
6932+
"value": "vp_df"
6933+
},
6934+
]
6935+
},
6936+
"pa_readCsv": {
6937+
"name": "Read Csv as pyarrow",
6938+
"library": "pyarrow",
6939+
"description": "Csv to pandas object",
6940+
"code": "${o0} = pa.csv.read_csv(${i0}${etc}).to_pandas()",
6941+
"options": [
6942+
{
6943+
"name": "i0",
6944+
"label": "File path/object",
6945+
"required": true,
6946+
"type": "text",
6947+
"component": [
6948+
"file"
6949+
]
6950+
},
6951+
{
6952+
"name": "o0",
6953+
"label": "Allocate to",
6954+
"output": true,
6955+
"component": [
6956+
"input"
6957+
],
6958+
"value": "vp_df"
6959+
}
6960+
]
6961+
},
6962+
"pa_toCsv": {
6963+
"name": "To Csv as pyarrow",
6964+
"library": "pyarrow",
6965+
"description": "DataFrame/Series to csv file",
6966+
"code": "pa.csv.write_csv(${i0}, ${path})",
6967+
"options": [
6968+
{
6969+
"name": "i0",
6970+
"label": "DataFrame",
6971+
"required": true,
6972+
"component": [
6973+
"data_select"
6974+
],
6975+
"var_type": [
6976+
"DataFrame",
6977+
"Series"
6978+
]
6979+
},
6980+
{
6981+
"name": "path",
6982+
"label": "File path/variable",
6983+
"required": true,
6984+
"type": "text"
6985+
}
6986+
]
6987+
},
6988+
"pa_readJson": {
6989+
"name": "Read Json as pyarrow",
6990+
"library": "pyarrow",
6991+
"description": "Json to pyarrow object",
6992+
"code": "${o0} = pa.json.read_json(${i0}${etc}).to_pandas()",
6993+
"options": [
6994+
{
6995+
"name": "i0",
6996+
"label": "File path/object",
6997+
"required": true,
6998+
"type": "text",
6999+
"component": [
7000+
"file"
7001+
]
7002+
},
7003+
{
7004+
"name": "o0",
7005+
"label": "Allocate to",
7006+
"output": true,
7007+
"component": [
7008+
"input"
7009+
],
7010+
"value": "vp_df"
7011+
}
7012+
]
7013+
},
7014+
"pa_readParquet": {
7015+
"name": "Read Parquet as pyarrow",
7016+
"library": "pyarrow",
7017+
"description": "Parquet to pandas object",
7018+
"code": "${o0} = pa.parquet.read_table(${i0}${etc}).to_pandas()",
7019+
"options": [
7020+
{
7021+
"name": "i0",
7022+
"label": "File path/object",
7023+
"required": true,
7024+
"type": "text",
7025+
"component": [
7026+
"file"
7027+
]
7028+
},
7029+
{
7030+
"name": "o0",
7031+
"label": "Allocate to",
7032+
"output": true,
7033+
"component": [
7034+
"input"
7035+
],
7036+
"value": "vp_df"
7037+
}
7038+
]
7039+
},
7040+
"pa_toParquet": {
7041+
"name": "To Parquet as pyarrow",
7042+
"library": "pyarrow",
7043+
"description": "DataFrame/Series to Parquet file",
7044+
"code": "pa.parquet.write_table(${i0}, ${path})",
7045+
"options": [
7046+
{
7047+
"name": "i0",
7048+
"label": "DataFrame",
7049+
"required": true,
7050+
"component": [
7051+
"data_select"
7052+
],
7053+
"var_type": [
7054+
"DataFrame",
7055+
"Series"
7056+
]
7057+
},
7058+
{
7059+
"name": "path",
7060+
"label": "File path/variable",
7061+
"required": true,
7062+
"type": "text"
7063+
}
7064+
]
7065+
},
68837066
}
68847067

68857068
return {

‎visualpython/js/com/com_Config.js

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,8 @@ define([
7575
'import matplotlib.pyplot as plt',
7676
'%matplotlib inline',
7777
'import seaborn as sns',
78-
'import plotly.express as px'
78+
'import plotly.express as px',
79+
'import pyarrow as pa'
7980
],
8081
'matplotlib customizing': [
8182
'import matplotlib.pyplot as plt',
@@ -132,7 +133,8 @@ define([
132133
'from plotly.offline import init_notebook_mode',
133134
'init_notebook_mode(connected=True)'
134135
]
135-
}
136+
},
137+
{ library: 'pyarrow', alias:'pa' },
136138
]
137139
}
138140

@@ -208,6 +210,10 @@ define([
208210
'sm': {
209211
code: 'import statsmodels.api as sm',
210212
type: 'package'
213+
},
214+
'pyarrow': {
215+
code: 'import pyarrow as pa',
216+
type: 'package'
211217
}
212218
}
213219

0 commit comments

Comments
 (0)