%matplotlib inline

# autoreload reloads modules automatically before entering the execution of code typed at the IPython prompt.
%load_ext autoreload
%autoreload 2

import warnings
warnings.filterwarnings('ignore')

from os import path
from utils import *
import pandas as pd
from IPython.display import display, HTML
import numpy as np
from matplotlib import pyplot as plt
import inspect
import time
import copy

import random
random.seed(10)
np.random.seed(10)

# Examine the data used for training the model
imageData = path.join("datasets", "imageData500_64pixels.hdf5")
validateArchive(imageData)

# Load, shape, and normalize the data used for training the model
with h5py.File(imageData, "r") as archive:   
    trainingData = np.squeeze(archive["trainData"][:])
    testData = np.squeeze(archive["testData"][:])
    trainingLabels = np.array(archive["trainLabels"][:])
    testLabels = np.array(archive["testLabels"][:])
    archive.close()

print("Archive trainingData.shape:    ", trainingData.shape)
print("Archive trainingLabels.shape:  ", trainingLabels.shape)
print("Archive testData.shape:        ", testData.shape)
print("Archive testLabels.shape:      ", testLabels.shape)
print("\n")

# Reshape the training and test data and label matrices
trainingData = trainingData.reshape(trainingData.shape[0], -1).T
testData = testData.reshape(testData.shape[0], -1).T

print ("Flattened, normalized trainingData shape:  " + str(trainingData.shape))
print ("Flattened, normalized testData shape:      " + str(testData.shape))

# Normalization
trainingData = trainingData/255.
testData = testData/255.

# Great reference:  https://www.python-course.eu/matplotlib_multiple_figures.php

# Write a function to show multiple graphs in the same figure
def printCostGraphs(costs, keys, cols, fsize = (15,6)):
    # Figure out how many rows and columns we need
    counter = 0
    rows = np.ceil(len(costs) / cols)
    fig = plt.figure(figsize = fsize)

    # Add each of the cost graphs to the figure
    for key in keys:
        c = np.squeeze(costs[key])
        sub = fig.add_subplot(rows, cols, counter + 1)
        sub.set_title('Epoch ' + str(key))
        sub.plot(c)
        counter = counter + 1

    # Draw the figure on the page
    plt.plot()
    plt.tight_layout()
    plt.show()

# Randomize values for hyperparameters based on a given key:value dictionary
class HPicker:

    def pick(self, ranges):
        hParams = {}

        # For each parameter key:val
        for key, value in ranges.items():
            if isinstance(value, list):
                start, stop, step = value
                vals = []

                # Create a range of possible values
                while (start < stop):
                    start = round(start + step, len(str(step)))
                    vals.append(start)

                # Pick one of the possible values randomly    
                hParams[key] = random.choice(vals)
            else:
                hParams[key] = value

        return hParams     

# Create instances of each of the activations we might utilize in the model

class AbstractActivation(object):
    def activate(self, z):
        raise NotImplementedError("Requires implementation by inheriting class.")

class Sigmoid(AbstractActivation):
    def activate(z):
        return 1 / (1 + np.exp(-(z)))

class Relu(AbstractActivation):
    def activate(z):
        return  z * (z > 0)

# Create a pandas dataframe with labeled columns to record model training results
def getResultsDF(hRanges):
    columns = list(hRanges.keys())
    df = pd.DataFrame(columns = columns)

    return(df)

# Do all the heavy lifting required when running N number of models with various hyperparameter configurations
def runModels(hRanges, epochs, silent = False):

    # Var inits
    picker = HPicker()
    resultsDF = getResultsDF(hRanges)
    costs = {}
    params = {}
    epoch = 0

    print("\n*** Starting model training")

    while (epoch < epochs):

        # Get the random hyperparam values
        hparams = picker.pick(hRanges)
        hparams["Epoch"] = epoch

        # Print a summary of the model about to be trained and its params to the user
        if silent is not True:
            print("Training epoch", epoch, "with params:  LR", hparams["Learning_Rate"],
                  ", iterations", hparams["Iterations"], ", HL units", hparams["HL_Units"],
                  ", lambda", hparams["Lambda"], ", and init. multi.", hparams["Weight_Multi"])

        # Train the model its given hyperparams and record the results
        params[epoch], costs[epoch],  hparams["Descending_Graph"] = model(
            trainingData, trainingLabels, hparams["HL_Units"], hparams["Iterations"],
            hparams["Learning_Rate"], hparams["Lambda"], hparams["Weight_Multi"], False)

        # Make predictions based on the model
        trainingPreds = predict(trainingData, params[epoch], trainingLabels)
        testPreds = predict(testData, params[epoch], testLabels)

        # Record prediction results
        hparams["Train_Acc"] = trainingPreds["accuracy"]
        hparams["Test_Acc"] = testPreds["accuracy"]
        hparams["Final_Cost"] = costs[epoch][-1]

        # Add model results to the pandas dataframe
        resultsDF.loc[epoch] = list(hparams.values())
        epoch = epoch + 1

    print("*** Done!\n")

    # Sort the dataframe so it's easier to find the results we are interested in
    resultsDF = resultsDF.sort_values(by = ['Descending_Graph', 'Test_Acc'], ascending = False)

    return resultsDF, params, costs

# Define the dimensions of the model
defineDimensions(data, labels, layerSize):
    nnDims = {}

    nnDims["numberInputs"] = data.shape[0]
    nnDims["hiddenLayerSize"] = layerSize
    nnDims["numberOutputs"] = labels.shape[0]

    return nnDims;

#  Initialize model params (i.e. W and b)
def initilizeParameters(dimensionDict, multiplier):

    np.random.seed(10)  # Yes, this has to be done every time...  :(
    w1 = np.random.randn(dimensionDict["hiddenLayerSize"], dimensionDict["numberInputs"]) * multiplier

    np.random.seed(10)  # Yes, this has to be done every time...  :(
    w2 = np.random.randn(dimensionDict["numberOutputs"], dimensionDict["hiddenLayerSize"]) * multiplier

    b1 = np.zeros((dimensionDict["hiddenLayerSize"], 1))
    b2 = np.zeros((dimensionDict["numberOutputs"], 1))

    params = {
        "w1" : w1,
        "w2" : w2,
        "b1" : b1,
        "b2" : b2}

    return params

# Perform forward propogation
def forwardPropagation(data, params, activation = Sigmoid()):
    w1 = params["w1"]
    b1 = params["b1"]
    w2 = params["w2"]
    b2 = params["b2"]

    z1 = np.dot(w1, data) + b1
    a1 = np.tanh(z1)
    z2 = np.dot(w2, a1) + b2
    a2 = Sigmoid.activate(z2)

    # Sanity check the dimensions
    assert(a2.shape == (1, data.shape[1]))

    cache = {"z1": z1,
             "a1": a1,
             "z2": z2,
             "a2": a2}

    return cache

# Calculate the cost of the model (includes L2 regularization)
def calculateCost(labels, params, cache, lamb):
    # Define vars to make reading and writing the formulas easier below...
    m = labels.shape[1]
    a2 = cache["a2"]
    w1 = params["w1"]
    w2 = params["w2"]

    # Perform cost and regularization calculations
    crossEntropyCost = (-1/m) * np.sum( labels*np.log(a2) + (1-labels)*np.log(1-a2) )
    l2RegularizationCost = (1/m) * (lamb/2) * (np.sum(np.square(w1)) + np.sum(np.square(w2)))
    finalCost = crossEntropyCost + l2RegularizationCost

    return finalCost

# Perform backward propogation
def backwardPropagation(data, labels, params, cache, lamb):
    # Define and populate variables
    m = data.shape[1]
    w1 = params["w1"]
    w2 = params["w2"]
    a1 = cache["a1"]
    a2 = cache["a2"]

    # Calculate gradients
    dz2 = a2 - labels
    dw2 = (1/m) * np.dot(dz2, a1.T) + (lamb/m) * w2
    db2 = (1/m) * np.sum(dz2, axis = 1, keepdims = True)
    dz1 = np.dot(w2.T, dz2) * (1 - np.power(a1, 2))
    dw1 = (1/m) * np.dot(dz1, data.T) +  (lamb/m) * w1
    db1 = (1/m) * np.sum(dz1, axis = 1, keepdims = True)

    # Store in the gradients cache
    gradients = {"dw1": dw1,
             "db1": db1,
             "dw2": dw2,
             "db2": db2}

    return gradients

# Update the model params based on the results of the backward propogation calculations
def updateParams(params, gradients, learningRate):
    params["w1"] = params["w1"] - learningRate * gradients["dw1"]
    params["b1"] = params["b1"] - learningRate * gradients["db1"]
    params["w2"] = params["w2"] - learningRate * gradients["dw2"]
    params["b2"] = params["b2"] - learningRate * gradients["db2"]

    return params

# Define the actual neural network classification model
def model(data, labels, layerSize, numIterations, learningRate, lamb, initializeMultiplier, printCost = False, showGraph = False):

    # Init vars
    dims = defineDimensions(data, labels, layerSize)
    params = initilizeParameters(dims, initializeMultiplier)
    costs = []
    descendingGraph = True

    # For each training iteration
    for i in range(0, numIterations + 1):

        # Forward propagation
        cache = forwardPropagation(data, params)

        # Cost function
        cost = calculateCost(labels, params, cache, lamb)

        # Backward  propagation
        grads = backwardPropagation(data, labels, params, cache, lamb)

        # Gradient descent parameter update
        params = updateParams(params, grads, learningRate)

        # Print the cost every N number of iterations
        if printCost and i % 500 == 0:
            print ("Cost after iteration", str(i), "is", str(cost))

        # Record the cost every N number of iterations
        if i % 500 == 0:
            if (len(costs) != 0) and (cost > costs[-1]):
                descendingGraph = False
            costs.append(cost)

    # Print the model training cost graph
    if showGraph:
        _costs = np.squeeze(costs)
        plt.plot(_costs)
        plt.ylabel('Cost')
        plt.xlabel('Iterations (every 100)')
        plt.title("Learning rate =" + str(learningRate))
        plt.show()

    return params, costs, descendingGraph

# Utilize the model's trained params to make predictions
def predict(data, params, trueLabels):
    # Apply the training weights and the sigmoid activation to the inputs
    cache = forwardPropagation(data, params)

    # Classify anything with a probability of greater than 0.5 to a 1 (i.e. cat) classification
    predictions = (cache["a2"] > 0.5)
    accuracy = 100 - np.mean(np.abs(predictions - trueLabels)) * 100

    preds = {"predictions" : predictions, "accuracy": accuracy}

    return preds

h1 = {
    "Epoch": None,
    "HL_Units": [2, 8, 1],
    "HL_Size": 1,
    "Iterations": [750, 2000, 50],
    "Learning_Rate": [0, .01, .001],
    "Lambda": [0, 2, .1],
    "Weight_Multi": [0, 0.001, .0001],
    "Train_Acc": None,
    "Test_Acc": None,
    "Final_Cost": None,
    "Descending_Graph": True
}

r1, p1, c1 = runModels(h1, 40, True)
display(HTML(r1.to_html()))

printCostGraphs(c1, list(r1.iloc[0:8, 0]), 4, (10,20))

h3 = {
    "Epoch": None,
    "HL_Units": [2, 8, 1],
    "HL_Size": 1,
    "Iterations": [2000, 10000, 500],
    "Learning_Rate": [0, .01, .001],
    "Lambda": [0, 2, .1],
    "Weight_Multi": [0, 0.001, .0001],
    "Train_Acc": None,
    "Test_Acc": None,
    "Final_Cost": None,
    "Descending_Graph": True
}

r3, p3, c3 = runModels(h3, 40, True)
display(HTML(r3.to_html()))

printCostGraphs(c3, list(r3.iloc[0:8, 0]), 4, (10,20))

h2 = {
    "Epoch": None,
    "HL_Units": [2, 8, 1],
    "HL_Size": 1,
    "Iterations": [750, 2000, 50],
    "Learning_Rate": [0, .1, .01],
    "Lambda": [0, 2, .1],
    "Weight_Multi": [0, 0.001, .0001],
    "Train_Acc": None,
    "Test_Acc": None,
    "Final_Cost": None,
    "Descending_Graph": True
}

r2, p2, c2 = runModels(h2, 40, True)
display(HTML(r2.to_html()))

printCostGraphs(c2, list(r2.iloc[0:8, 0]), 4, (10,20))

h4 = {
    "Epoch": None,
    "HL_Units": [2, 8, 1],
    "HL_Size": 1,
    "Iterations": [2000, 10000, 500],
    "Learning_Rate": [0, .1, .01],
    "Lambda": [0, 2, .1],
    "Weight_Multi": [0, 0.001, .0001],
    "Train_Acc": None,
    "Test_Acc": None,
    "Final_Cost": None,
    "Descending_Graph": True
}

r4, p4, c4 = runModels(h4, 40, True)
display(HTML(r4.to_html()))

printCostGraphs(c4, list(r4.iloc[0:8, 0]), 4, (10,20))

%%html
<style>
  table {margin-left: 0 !important;}
</style>

# Epoch 	HL_Units 	HL_Size 	Iterations 	Learning_Rate 	Lambda 	Weight_Multi 	Train_Acc 	 Test_Acc 	Final_Cost 	Descending_Graph
# 1	6	1	1100	0.06	0.6	0.0002	83.375	73.0	0.350286	True

pLarge, cLarge, gLarge = model(trainingData, trainingLabels, 6, 1100, .06, .6, .0002, False, True)

trainingPredsLarge = predict(trainingData, pLarge, trainingLabels)
testPredsLarge = predict(testData, pLarge, testLabels)
print("\nTrain accuracy:", trainingPredsLarge["accuracy"])
print("Test accuracy:", testPredsLarge["accuracy"])

from sklearn.metrics import precision_recall_fscore_support as score

precision, recall, fscore, support = score(np.squeeze(testLabels), np.squeeze(testPredsLarge["predictions"]))

print('Precision: {}'.format(precision))
print('Recall: {}'.format(recall))
print('F-Score: {}'.format(fscore))