import config

APIKEY=config.APIKEY

from newsapi import NewsApiClient
newsapi = NewsApiClient(api_key=APIKEY)

import pandas as pd
sources = newsapi.get_sources()
print('How many total sources are available?:',len(sources['sources']))
total_sources = len(sources['sources'])
sources['sources'][0]

id=pd.Series(name='id')
name=pd.Series(name='name')
description=pd.Series(name='description')
url=pd.Series(name='url')
category=pd.Series(name='category')
language=pd.Series(name='language')
country=pd.Series(name='country')
#column_names = ['id', 'name', 'description', 'url', 'category', 'language', 'country']

for idx,outlet in enumerate(sources['sources']):
    id[idx]=outlet['id']
    name[idx] = outlet['name']
    description[idx] = outlet['description']
    url[idx] = outlet['url']
    category[idx] = outlet['category']
    language[idx] = outlet['language']
    country[idx] = outlet['country']
sourcedf=pd.concat([id,name,description,url,category,language,country],axis=1)

print('What are the categories? ', sourcedf['category'].unique())

sourcedf= sourcedf[(sourcedf['language']=='en') & (sourcedf['country']=='us')]
sourcedf.head()

business_sources = sourcedf[sourcedf['category']=='business']
business_sources = business_sources.reset_index()
display(business_sources)
print('Number of sources: ',len(business_sources))
#business_sources['id'][0]

# Get 30 Days Ago
import datetime
from datetime import date, timedelta
print('Todays Date: ', date.today().isoformat())
print('30 Days Ago: ', (date.today()-timedelta(days=30)).isoformat())
startdate=(date.today()-timedelta(days=30)).isoformat()

# Init Variables
Source=pd.Series(name='source')
author=pd.Series(name='author')
title=pd.Series(name='title')
description=pd.Series(name='description')
url=pd.Series(name='url')
publishedAt=pd.Series(name='publishedAt')
content=pd.Series(name='content')
# Use Counter
counter=0
# Loop through each source 
for source in business_sources['id']:
    news_articles = newsapi.get_everything(q='apple',
                                          sources=source,
                                          from_param=startdate,
                                          )
    # Gather data from each article
    for idx, article in enumerate(news_articles['articles']):
        Source[counter] = article['source']['id']
        author[counter] = article['author']
        title[counter] = article['title']
        description[counter] = article['description']
        url[counter] = article['url']
        publishedAt[counter] = article['publishedAt']
        content[counter] = article['content']
        #author=pd.concat([Source,article['author']],axis=1)
        #for details in article:
        #    print(details['title'])
        counter = counter + 1
# Combine 
all_articles = pd.concat([Source, author, title, description, url, publishedAt, content],axis=1)
all_articles.head()

pd.set_option('display.max_columns',None)

from textblob import TextBlob

text = all_articles['title'][8]
blob = TextBlob(text)
sentiment = blob.sentiment

print(sentiment)

TitlePolarity = pd.Series(name='TitlePolarity')
TitleSubjectivity = pd.Series(name='TitleSubjectivity')
TitleNegative = pd.Series(name='TitleNegative')
TitleNeutral = pd.Series(name='TitleNeutral')
TitlePositive = pd.Series(name='TitlePositive')
TitleComposite = pd.Series(name='TitleComposite')
from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyzer = SentimentIntensityAnalyzer()

id = 0
for title in all_articles['title']:
    #print(id,title)
    blob = TextBlob(title)
    sentiment = blob.sentiment
    TitlePolarity[id] = sentiment.polarity
    TitleSubjectivity[id] = sentiment.subjectivity

    vs = analyzer.polarity_scores(title)
    TitleNegative[id] = vs['neg']
    TitleNeutral[id] = vs['neu']
    TitlePositive[id] = vs['pos']
    TitleComposite[id] = vs['compound']

    id=id+1

all_articles = pd.concat([all_articles, TitlePolarity, TitleSubjectivity, TitleNegative, TitleNeutral, TitlePositive, TitleComposite],axis=1)
all_articles.head()

descriptionPolarity = pd.Series(name='descriptionPolarity')
descriptionSubjectivity = pd.Series(name='descriptionSubjectivity')
descriptionNegative = pd.Series(name='descriptionNegative')
descriptionNeutral = pd.Series(name='descriptionNeutral')
descriptionPositive = pd.Series(name='descriptionPositive')
descriptionComposite = pd.Series(name='descriptionComposite')
from textblob import TextBlob
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
analyzer = SentimentIntensityAnalyzer()

id = 0
for description in all_articles['description']:
    #print(id,description)
    blob = TextBlob(description)
    sentiment = blob.sentiment
    descriptionPolarity[id] = sentiment.polarity
    descriptionSubjectivity[id] = sentiment.subjectivity

    vs = analyzer.polarity_scores(description)
    descriptionNegative[id] = vs['neg']
    descriptionNeutral[id] = vs['neu']
    descriptionPositive[id] = vs['pos']
    descriptionComposite[id] = vs['compound']

    id=id+1

all_articles = pd.concat([all_articles, descriptionPolarity, descriptionSubjectivity, descriptionNegative, descriptionNeutral, descriptionPositive, descriptionComposite],axis=1)
all_articles.head()

all_articles['Date']= pd.to_datetime(all_articles['publishedAt']).dt.floor('H')

all_articles['Date']

import yfinance as yf
import numpy as np
sdf=yf.download('aapl',start=startdate, interval="1h")
sdf=sdf.reset_index()
sdf = sdf.rename(columns={'Datetime':'Date'})
sdf['Date'] = sdf['Date'].dt.floor('H')
# Subset just Date & Close
sdf=sdf[['Date','Close']]
sdf['Return'] = np.where(sdf['Close'] > sdf['Close'].shift(1),1,0)

sdf.head()

#Merge
combineddf= pd.merge(sdf,all_articles,how='left').ffill(limit=24)
combineddf['SMA12']= combineddf['Close'].rolling(12).mean()
combineddf['SMA24']= combineddf['Close'].rolling(24).mean()
combineddf['SMA48']= combineddf['Close'].rolling(48).mean()
# Drop all unnecessary data
combineddf = combineddf.drop(columns=['author','title','description','source','url','publishedAt','content'])
combineddf.tail(10)

import matplotlib.pyplot as plt
# Visualize the Closing price
plt.figure(figsize=(12,6))# specifies size
plt.title('Close Price History',fontsize=24)
plt.plot(combineddf[['Close','SMA12','SMA24']])
plt.xlabel('Date',fontsize=12)
plt.ylabel('Close Price USD   $',fontsize=12)
plt.legend(['Close','SMA12','SMA24'])
plt.show()

# missing data
combineddf= combineddf.dropna(axis=0)

# How many days did the stock go up and down?
combineddf['Return'].value_counts()

from sklearn.model_selection import train_test_split
X = combineddf.drop(columns=['Return','Date'])
Y = combineddf[['Return']]
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.33, random_state = 0)

from sklearn.preprocessing import StandardScaler

ss_train = StandardScaler()
X_train = ss_train.fit_transform(X_train)

ss_test = StandardScaler()
X_test = ss_test.fit_transform(X_test)

from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(n_estimators=1000, max_depth=5)

# Get predictions
model.fit(X_train, Y_train.values.ravel())
preds = model.predict(X_test)

from sklearn.metrics import accuracy_score

accuracy_score(Y_test,preds)

from sklearn.linear_model import LogisticRegression
model2 = LogisticRegression()
model2.fit(X_train, Y_train.values.ravel())
preds2 = model.predict(X_test)
accuracy_score(Y_test,preds2)
#preds2

from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense, LSTM
import math

stock = combineddf[['Close']]
dataset = stock.values
training_data_len = math.ceil(len(dataset)*.8)
# Scale the data
scaler = MinMaxScaler(feature_range=(0,1))

scaled_data = scaler.fit_transform(dataset)
# Create the training dataset
train_data = scaled_data[0:training_data_len,:]

x_train=[]# dependent
y_train=[]# outcome
# Loop systematically takes 60 days as x training and the output being the next day
for i in range(60,len(train_data)):
  x_train.append(train_data[i-60:i,0])
  y_train.append(train_data[i,0])
  if i<=61:
    print('x_train :',i,x_train)
    print('y_train',y_train)
    print()

# Convert x & y training to numpy arrays
x_train, y_train = np.array(x_train), np.array(y_train)

x_train= np.reshape(x_train,(x_train.shape[0],x_train.shape[1],1))
x_train.shape

# Build the lstm
model = Sequential()
model.add(LSTM(50, return_sequences=True, input_shape= (x_train.shape[1], 1 ))) # 50 neurons
model.add(LSTM(50,return_sequences=False)) # False because its the last lstm layer
model.add(Dense(25))# Neural network layer with 25 neurons
model.add(Dense(1))# neural network layer with 1 neuron

# Compile the model
model.compile(optimizer='adam', loss='mean_squared_error')

# Train the model
model.fit(x_train, y_train, batch_size=1, epochs= 1)

# Create the test dataset
test_data = scaled_data[training_data_len-60:,:]
x_test = []
y_test = dataset[training_data_len:,:]

for i in range(60, len(test_data)):
  x_test.append(test_data[i-60:i,0])

x_test = np.array(x_test)

# Reshape
x_test = np.reshape(x_test,(x_test.shape[0],x_test.shape[1],1))
x_test.shape

# Get the model's predicted price values
predictions = model.predict(x_test)

# Inverse transform the data back to the regular prices
predictions = scaler.inverse_transform(predictions)

# Visualize the data
train = stock[0:training_data_len]
valid = stock[training_data_len:]
valid['Predictions'] = predictions

plt.figure(figsize=(16,8))
plt.title('Model',fontsize=24)
plt.xlabel('Date')
plt.ylabel('Close Price USD ($)')
plt.plot(train['Close'])
plt.plot(valid[['Close','Predictions']])
plt.legend(['Train','Validation','Predicitons'],loc='lower right')
plt.show()