import pandas as pd
import tensorflow as tf
import numpy as np

from matplotlib import pyplot as plt

#train_url = "https://github.com/PhilChodrow/PIC16b/blob/master/datasets/fake_news_train.csv?raw=true"
df = pd.read_csv('fake_news_train.csv')

test_url = "https://github.com/PhilChodrow/PIC16b/blob/master/datasets/fake_news_test.csv?raw=true"
test_df = pd.read_csv(test_url)

df

import nltk
#nltk.download('stopwords') #only need to be downloaded once
#nltk.download('punkt')  #only need to be downloaded once as well
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize 
stop = stopwords.words('english')

from nltk.tokenize.treebank import TreebankWordDetokenizer

def make_dataset(df):
    df_ = df
    
    #remove stopwords for the dataframe
    df_['title'] = df_['title'].apply(word_tokenize)# split a string into separate tokens
    df_['title'] = df['title'].apply(lambda x: [w for w in x if not w in stop] )#remove stopwords
    df_['text'] = df_['text'].apply(word_tokenize)
    df_['text'] = df_['text'].apply(lambda x: [w for w in x if not w in stop] )
    df_['text'] = df_['text'].apply(lambda x: TreebankWordDetokenizer().detokenize(x))#combine the tokens back to a complete sentence
    df_['title'] = df_['title'].apply(lambda x: TreebankWordDetokenizer().detokenize(x))

    #create tensorflow dataset with inputs (title, text) and output fake column labels and return it
    return tf.data.Dataset.from_tensor_slices(({"title": df_['title'], "text": df_['text']}, df_['fake']))

dataset = make_dataset(df)

test_dataset = make_dataset(test_df)

#shuffle the complete training set
dataset = dataset.shuffle(buffer_size = len(dataset))

#specify training and validation size: 0.7 & 0.2, respectively
train_size = int(0.7*len(dataset))
val_size   = int(0.2*len(dataset))

#pick out first 0.7 as training set, with a batch of 100
train = dataset.take(train_size).batch(100)
#pick out the next 0.2 as validation set, with a batch of 100
val   = dataset.skip(train_size).take(val_size).batch(100)
#test  = dataset.skip(train_size + val_size).batch(100) #last 0.1 of data

#check for the length of the two sets
len(train), len(val)#, len(test)

test_ds = test_dataset.batch(100)

def standardization(input_data):
    lowercase = tf.strings.lower(input_data)
    no_punctuation = tf.strings.regex_replace(lowercase,
                                  '[%s]' % re.escape(string.punctuation),'')
    return no_punctuation 

from tensorflow.keras import losses
from tensorflow.keras.layers.experimental.preprocessing import TextVectorization

import re
import string

# only the first 2000 distinct words in the whole set will be tracked
max_tokens = 2000

# only the first 20 words of each headline will be considered
sequence_length = 20

vectorize_layer = TextVectorization(
    standardize=standardization,
    max_tokens=max_tokens, # only consider this many words
    output_mode='int',
    output_sequence_length=sequence_length) 

vectorize_layer.adapt(train.map(lambda x, y: x["title"]))
vectorize_layer.adapt(train.map(lambda x, y: x["text"]))

from tensorflow import keras
from tensorflow.keras import layers

title_input = keras.Input(
    shape = (1,), 
    name = "title",
    dtype = "string"
)

text_input = keras.Input(
    shape = (1,), 
    name = "text",
    dtype = "string"
)

x = vectorize_layer(title_input)
x = layers.Embedding(max_tokens, output_dim = 5, name = "embedding_tl")(x)
x = layers.Dropout(0.2)(x)
x = layers.GlobalAveragePooling1D()(x)
x = layers.Dropout(0.2)(x)
x = layers.Dense(32, activation='relu')(x)
output = layers.Dense(2, name = "fake")(x)

model_tl = keras.Model(
    inputs = title_input,
    outputs = output
)

model_tl.summary()

model_tl.compile(optimizer = "adam",
              loss = losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy']
)

history = model_tl.fit(train, 
                    validation_data=val,
                    epochs = 50, 
                    verbose = False)

plt.plot(history.history["accuracy"], label = "training")
plt.plot(history.history["val_accuracy"], label = "validation")
plt.gca().set(xlabel = "epoch", ylabel = "accuracy")
plt.legend()

model_tl.evaluate(test_ds)

x_ = vectorize_layer(text_input)
x_ = layers.Embedding(max_tokens, output_dim = 5, name = "embedding_tx")(x_)
x_ = layers.Dropout(0.2)(x_)
x_ = layers.GlobalAveragePooling1D()(x_)
x_ = layers.Dropout(0.2)(x_)
x_ = layers.Dense(32, activation='relu')(x_)
output = layers.Dense(2, name = "fake")(x_)

model_tx= keras.Model(
    inputs = text_input,
    outputs = output
)

model_tx.compile(optimizer = "adam",
              loss = losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy']
)

model_tx.fit(train, 
                    validation_data=val,
                    epochs = 50, 
                    verbose = False)

model_tx.evaluate(test_ds)

both = layers.concatenate([x, x_], axis = 1)
output_both = layers.Dense(2, name = "fake")(both)

model_both= keras.Model(
    inputs = [title_input, text_input],
    outputs = output_both
)

model_both.summary()

model_both.compile(optimizer = "adam",
              loss = losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy']
)

model_both.fit(train, 
                    validation_data=val,
                    epochs = 50, 
                    verbose = False)

model_both.evaluate(test_ds)

weights_tl = model_both.get_layer('embedding_tl').get_weights()[0]
weights_tx = model_both.get_layer('embedding_tx').get_weights()[0]# get the weights from the embedding layer
vocab = vectorize_layer.get_vocabulary()  

vocab = vectorize_layer.get_vocabulary()  

weights_tl

from sklearn.decomposition import PCA

pca = PCA(n_components=2)
weights_tl = pca.fit_transform(weights_tl)
weights_tx = pca.fit_transform(weights_tx)

len(vocab)

embedding_tl = pd.DataFrame({
    'word' : vocab, 
    'x0'   : weights_tl[:,0],
    'x1'   : weights_tl[:,1]
})
embedding_tl

embedding_tx = pd.DataFrame({
    'word' : vocab, 
    'x0'   : weights_tx[:,0],
    'x1'   : weights_tx[:,1]
})

import plotly.express as px 
import plotly.io as pio
pio.templates.default = "plotly_white"
import plotly.express as px 

fig = px.scatter( embedding_tl, 
                 x = "x0", 
                 y = "x1", 
                 size = list(np.ones(len(embedding_tl))),
                 size_max = 2,
                 hover_name = "word")

fig.show()

from plotly.io import write_html

write_html(fig, "emb_tx.html")