import tensorflow as tf import keras import sklearn from sklearn import linear_model from sklearn.utils import shuffle import pandas as pd import numpy as np from sklearn import datasets from sklearn.metrics import accuracy_score,precision_score,recall_score,matthews_corrcoef from sklearn.tree import DecisionTreeClassifier, export_text import os from statistics import mean test_new = ["SPY","TSLA","MMM","ABT","ABBV","ABMD","ACN","ATVI","ADBE","AMD","AAP","AES","AFL","A","APD","AKAM","ALK","ALB","ARE","ALXN","ALGN","ALLE","LNT","ALL","GOOGL","GOOG","MO","AMZN","AMCR","AEE","AAL","AEP","AXP","AIG","AMT","AWK","AMP","ABC","AME","AMGN","APH","ADI","ANSS","ANTM","AON","AOS","APA","AIV","AAPL","AMAT","APTV","ADM","ANET","AJG","AIZ","T","ATO","ADSK","ADP","AZO","AVB","AVY","BKR","BLL","BAC","BK","BAX","BDX","BBY","BIO","BIIB","BLK","BA","BKNG","BWA","BXP","BSX","BMY","AVGO","BR","CHRW","COG","CDNS","CPB","COF","CAH","KMX","CCL","CAT","CBOE","CBRE","CDW","CE","CNC","CNP","CTL","CERN","CF","SCHW","CHTR","CVX","CMG","CB","CHD","CI","CINF","CTAS","CSCO","C","CFG","CTXS","CLX","CME","CMS","KO","CTSH","CL","CMCSA","CMA","CAG","CXO","COP","ED","STZ","COO","CPRT","GLW","CTVA","COST","COTY","CCI","CSX","CMI","CVS","DHI","DHR","DRI","DVA","DE","DAL","XRAY","DVN","DXCM","FANG","DLR","DFS","DISCA","DISCK","DISH","DG","DLTR","D","DPZ","DOV","DOW","DTE","DUK","DRE","DD","DXC","ETFC","EMN","ETN","EBAY","ECL","EIX","EW","EA","EMR","ETR","EOG","EFX","EQIX","EQR","ESS","EL","EVRG","ES","RE","EXC","EXPE","EXPD","EXR","XOM","FFIV","FB","FAST","FRT","FDX","FIS","FITB","FE","FRC","FISV","FLT","FLIR","FLS","FMC","F","FTNT","FTV","FBHS","FOXA","FOX","BEN","FCX","GPS","GRMN","IT","GD","GE","GIS","GM","GPC","GILD","GL","GPN","GS","GWW","HRB","HAL","HBI","HIG","HAS","HCA","PEAK","HSIC","HSY","HES","HPE","HLT","HFC","HOLX","HD","HON","HRL","HST","HPQ","HUM","HBAN","HII","IEX","IDXX","INFO","ITW","ILMN","INCY","IR","INTC","ICE","IBM","IP","IPG","IFF","INTU","ISRG","IVZ","IPGP","IQV","IRM","JKHY","J","JBHT","SJM","JNJ","JCI","JPM","JNPR","KSU","K","KEY","KEYS","KMB","KIM","KMI","KLAC","KSS","KHC","KR","LB","LHX","LH","LRCX","LW","LVS","LEG","LDOS","LEN","LLY","LNC","LIN","LYV","LKQ","LMT","L","LOW","LYB","MTB","MRO","MPC","MKTX","MAR","MMC","MLM","MAS","MA","MKC","MXIM","MCD","MCK","MDT","MRK","MET","MTD","MGM","MCHP","MU","MSFT","MAA","MHK","TAP","MDLZ","MNST","MCO","MS","MOS","MSI","MSCI","MYL","NDAQ","NOV","NTAP","NFLX","NWL","NEM","NWSA","NWS","NEE","NLSN","NKE","NI","NBL","NSC","NTRS","NOC","NLOK","NCLH","NRG","NUE","NVDA","NVR","ORLY","OXY","ODFL","OMC","OKE","ORCL","PCAR","PKG","PH","PAYX","PAYC","PYPL","PNR","PBCT","PEP","PKI","PRGO","PFE","PM","PSX","PNW","PXD","PNC","PPG","PPL","PFG","PG","PGR","PLD","PRU","PEG","PSA","PHM","PVH","QRVO","PWR","QCOM","DGX","RL","RJF","RTX","O","REG","REGN","RF","RSG","RMD","RHI","ROK","ROL","ROP","ROST","RCL","SPGI","CRM","SBAC","SLB","STX","SEE","SRE","NOW","SHW","SPG","SWKS","SLG","SNA","SO","LUV","SWK","SBUX","STT","STE","SYK","SIVB","SYF","SNPS","SYY","TMUS","TROW","TTWO","TPR","TGT","TEL","FTI","TDY","TFX","TXN","TXT","TMO","TIF","TJX","TSCO","TDG","TRV","TFC","TWTR","TYL","TSN","UDR","ULTA","USB","UAA","UA","UNP","UAL","UNH","UPS","URI","UHS","UNM","VFC","VLO","VAR","VTR","VRSN","VRSK","VZ","VRTX","V","VNO","VMC","WRB","WAB","WMT","WBA","DIS","WM","WAT","WEC","WFC","WELL","WST","WDC","WU","WRK","WY","WHR","WMB","WLTW","WYNN","XEL","XRX","XLNX","XYL","YUM","ZBRA","ZBH","ZION","ZTS"] def interpret(data): Datakit = data[["Forecast","Previous","R_Squared","Directional Accuracy","Test"]] Predict = "Directional Accuracy" print("Preparing Data") X = np.array(Datakit.drop([Predict],1)) y = np.array(Datakit[Predict]) X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size = 0.25) y_train=y_train.astype('int') y_test = y_test.astype('int') tree_model = DecisionTreeClassifier() print("Data has been loaded") print("Fitting model") tree_model.fit(X_train, y_train) predictions = tree_model.predict(X_test) #print(predictions) #print(X_test) #print(y_test) print(accuracy_score(y_test,predictions)) res_lst = [] for x in range(0,len(y_test)): if predictions[x] == 1: if y_test[x] == predictions[x]: res_lst.append(1) else: res_lst.append(0) #print(mean(res_lst)) #print("Done!") return mean(res_lst) def AccStat(y_test,predictions): NegRecall=[] OpenPrecision=[] for x in range(0,len(y_test)): if predictions[x] == 1: OpenPrecision.append(y_test[x]) if y_test[x] == 0: NegRecall.append(predictions[x]) Pnum = sum(OpenPrecision) Pdenom = len(OpenPrecision) Numerator = Pdenom-Pnum Denominator = Pdenom+len(NegRecall) Stat = Numerator/(Denominator-Numerator) return Stat def examine(data,Forecast,test_size=0.25): Datakit = data[["Forecast","Previous","R_Squared","Directional Accuracy","Test"]] Forecast_fixed = Forecast[["Forecast","Previous","R_Squared","Test"]] Predict = "Directional Accuracy" X = np.array(Datakit.drop([Predict],1)) y = np.array(Datakit[Predict]) X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, test_size = test_size) y_train=y_train.astype('int') y_test = y_test.astype('int') tree_model = DecisionTreeClassifier() tree_model.fit(X_train, y_train) predictions = tree_model.predict(X_test) Accuracy = accuracy_score(y_test,predictions) Recall = recall_score(y_test,predictions) Precision = precision_score(y_test,predictions) MCC = matthews_corrcoef(y_test,predictions) Test_Stat = AccStat(y_test,predictions) print(Test_Stat) #for y in range(0,len(predictions)): # print(predictions[y],y_test[y]) forecast_guess = tree_model.predict(Forecast_fixed) res_lst = [] for x in range(0,len(y_test)): if predictions[x] == 1: if y_test[x] == predictions[x]: res_lst.append(1) else: res_lst.append(0) #print(mean(res_lst)) #print("Done!") print('------') GenTree = export_text(tree_model) f = open('/home/oblanco214/CondaServer/Future/TreeLog/treetmp.txt','a') f.write(GenTree) f.close() #Result column change: mean(res_lst) --> compositeR (2020-12-09) *old* #return change: replaced compositeR with Accuracy; added Recall and Precision (2020-12-09) #return change: added custom Test_Stat (2020-12-14) return (Accuracy,forecast_guess,Recall,Precision,MCC,Test_Stat) #masterframe = pd.DataFrame(columns=["Unnamed: 0","Ticker","Forecast","Previous","Actual","R_Squared","Directional Accuracy","Test","Return"]) #for x in os.listdir("/home/oblanco214/PyServer/lotusfiles/Revised/"): # print(x) # data= pd.read_csv('/home/oblanco214/PyServer/lotusfiles/Revised/'+x) # masterframe = pd.concat([masterframe,data]) #print(masterframe) #masterframe.dropna(inplace=True) #interpret(masterframe)