import os, sys
import numpy as np
import pandas as pd
from glob import glob
### combine parameter
print("Combine Parameters")
all_parameters = glob('*.csv')
### Combine ###
Combine_data = pd.DataFrame()
for file in all_parameters:
print(file)
data = pd.read_csv(file)
Combine_data = pd.concat([Combine_data, data], ignore_index=True)
for file in all_parameters:
os.remove(file)
Combine_data.to_csv("combine_param.csv", index=False)
# --------------------------------------------------------------------------
### Remove Duplicates ###
if not Combine_data.empty:
print("Remove Duplicates")
initial_length = len(Combine_data)
print('Total Rows -', initial_length)
#Drop Dublicate
Combine_data.drop_duplicates(inplace=True)
final_length = len(Combine_data)
print('Dublicates Rows -', initial_length - final_length)
# ---------------------------------------------------------------------------
# remove false rows
if not Combine_data.empty:
print("\nRemove False Parameter")
print(Combine_data.columns.to_series().reset_index(drop=True))
col_index = int(input('Column True/False Index : '))
col_name = Combine_data.columns.to_list()[col_index]
initial_length = len(Combine_data)
print('Total Rows -', initial_length)
#Drop Dublicate
Combine_data = Combine_data[Combine_data[col_name] == True]
final_length = len(Combine_data)
print('Falsed Rows -', initial_length - final_length)
# ---------------------------------------------------------------------------
if not Combine_data.empty:
output_path = ''
if not os.path.isdir(output_path) and output_path != '':
os.mkdir(output_path)
bn_data = Combine_data[Combine_data['index'] == 'BANKNIFTY']
nf_data = Combine_data[Combine_data['index'] == 'NIFTY']
fn_data = Combine_data[Combine_data['index'] == 'FINNIFTY']
print('Total Rows -', len(Combine_data))
print('BANKNIFTY Rows -', len(bn_data))
print('NIFTY Rows-', len(nf_data))
print('FINNIFTY Rows-', len(fn_data))
# Split BANKNIFTY data
if not bn_data.empty:
no_of_param = int(input("BANKNIFTY Parameter No : "))
for idx, data in enumerate(np.array_split(bn_data, no_of_param)):
data.to_csv(f"{output_path}bn_{idx}.csv", index=False)
# Split NIFTY data
if not nf_data.empty:
no_of_param = int(input("NIFTY Parameter No : "))
for idx, data in enumerate(np.array_split(nf_data, no_of_param)):
data.to_csv(f"{output_path}nf_{idx}.csv", index=False)
# Split FINNIFTY data
if not fn_data.empty:
no_of_param = int(input("FINNIFTY Parameter No : "))
for idx, data in enumerate(np.array_split(fn_data, no_of_param)):
data.to_csv(f"{output_path}fn_{idx}.csv", index=False)
if input("create py files y/n ? : ") == "y":
import os
import sys
### convert into py files
os.system('jupyter nbconvert --to script *.ipynb')
def prepend_line(file_name, line, name):
""" Insert given string as a new line at the beginning of a file """
# define name of temporary dummy file
dummy_file = file_name + '.bak'
# open original file in read mode and dummy file in write mode
with open(file_name, 'r') as read_obj, open(dummy_file, 'w') as write_obj:
# Write given line to the dummy file
write_obj.write(line + '\n')
# Read lines from original file one by one and append them to the dummy
file
for line in read_obj:
write_obj.write(line)
os.rename(dummy_file, name)
def code_script(name):
code_script = f"""import os
import sys
code_name = sys.argv[0].split('\\\\')[-1].replace('.py', '')
import ctypes
ctypes.windll.kernel32.SetConsoleTitleW(code_name)
print(code_name)
parameter_path = '{name}'
"""
return code_script
parameters = glob(f'{output_path}*.csv')
parameters = list(set(parameters) - set(["combine_param.csv"]))
code = glob("*.ipynb")[0].replace('ipynb','py')
import fileinput
tempFile = open(code, 'r+')
for line in fileinput.input(code):
tempFile.write(line.replace('parameter_path =', '# parameter_path ='))
tempFile.close()
for file in parameters:
prepend_line(code, code_script(file), file.replace('csv', 'py'))
os.remove(code)
