#!/usr/bin/env python
# coding: utf-8

# <center><img src="images/title.png" width="95%"/></center>
# <center><a href="http://bit.ly/pybay-keras">bit.ly/pybay-keras</a></center>

# Who Am I?
# -----
# 
# <center>Brian Spiering</center>

# What Do I Do?
# ------
# 
# <b><center>Professor @</center><b>
# <center><img src="images/msds_logo.png" width="28%"/></center>

# <center><img src="images/real_deep_learning.png" width="80%"/></center>

# Keras - Neural Networks for humans
# ------
# 
# <center><img src="images/keras-logo-small.jpg" width="20%"/></center>
# 
# A high-level, intuitive API for Deep Learning.
# 
# Easy to define neural networks, then automatically handles execution.
# 
# A simple, modular interface which allows focus on learning and enables fast experimentation

# Goals
# -----
# 
# - General introduction to Deep Learning
# - Overview of keras library
# - An end-to-end example in keras 

# Anti-Goals
# -----
# 
# - Understanding of Deep Learning (there will be no equations)
# - Building neural networks from scratch
# - Complete survey of keras library

# Deep Learning 101
# -----
# <center><img src="images/neural_nets.jpg" width="75%"/></center>

# Deep Learning (DL) are Neural networks (NN) with >1 hidden layer
# -------
# 
# <center><img src="images/neural-networks-layers.jpg" width="80%"/></center>

# Neural Networks are Nodes & Edges
# ------
# <center><img src="images/sum.png" width="75%"/></center>

# Nonlinear function allows learning of nonlinear relationships
# ------
# 
# <center><img src="images/function_3.png" width="80%"/></center>

# Groups of nodes all the way down
# ------
# 
# <center><img src="images/layers.png" width="75%"/></center>

# Deep Learning isn't magic, it is just very good at finding patterns
# ------
# 
# <center><img src="images/features.png" width="80%"/></center>

# Deep Learning has fewer steps than traditional Machine Learning
# ------
# 
# <center><img src="images/traditional-ml-deep-learning-2.png" width="100%"/></center>

# If you want to follow along…
# -----
# 
# GitHub repo: [bit.ly/pybay-keras](http://bit.ly/pybay-keras)
# 
# If you want to type along…
# ------
# 
# 1. Run a local Jupyter Notebook
# 1. [Binder](https://mybinder.org/v2/gh/brianspiering/keras-intro/master): In-Browser Jupyter Notebook
# 1. [Colaboratory](https://colab.research.google.com/): "Google Docs for Jupyter Notebooks"

# In[84]:


reset -fs


# In[85]:


import keras


# In[86]:


# What is the backend / execution engine?


# In[87]:


keras.backend.backend()


# <center><img src="images/tf_logo.jpg" width="70%"/></center>
# 
# "An open-source software library for Machine Intelligence"
# 
# Numerical computation using data flow graphs. 

# TensorFlow: A great backend
# ------
# A __very__ flexible architecture which allows you to do almost any numerical operation.
# 
# Then deploy the computation to CPUs or GPUs (one or more) across desktop, cloud, or mobile device. 
# <center><img src="images/tf_features.png" width="38%"/></center>

# MNIST handwritten digit database: <br> The “Hello World!” of Computer Vision
# ------
# 
# <center><img src="images/mnist-digits.png" width="80%"/></center>

# <center><img src="images/MNIST-Matrix.png" width="100%"/></center>

# <center><img src="images/MNIST_neuralnet_image.png" width="100%"/></center>

# In[88]:


# Import data


# In[89]:


from keras.datasets import mnist


# In[90]:


# Setup train and test splits


# In[91]:


(x_train, y_train), (x_test, y_test) = mnist.load_data()


# In[92]:


from random import randint
from matplotlib import pyplot

get_ipython().run_line_magic('matplotlib', 'inline')


# In[93]:


pyplot.imshow(x_train[randint(0, x_train.shape[0])], cmap='gray_r');


# Munge data
# -----
# 
# <center><img src="images/mnist_keras.png" width="75%"/></center>
# 
# Convert image matrix into vector to feed into first layer 

# In[94]:


# Munge Data
# Transform from matrix to vector, cast, and normalize


# In[95]:


image_size = 784 # 28 x 28

x_train = x_train.reshape(x_train.shape[0], image_size) # Transform from matrix to vector
x_train = x_train.astype('float32') # Cast as 32 bit integers
x_train /= 255 # Normalize inputs from 0-255 to 0.0-1.0

x_test = x_test.reshape(x_test.shape[0], image_size) # Transform from matrix to vector
x_test = x_test.astype('float32') # Cast as 32 bit integers
x_test /= 255 # Normalize inputs from 0-255 to 0.0-1.0


# In[96]:


# Convert class vectors to binary class matrices


# In[97]:


y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)


# In[98]:


# Import the most common type of neural network


# In[99]:


from keras.models import Sequential


# RTFM - https://keras.io/layers/

# In[100]:


# Define model instance


# In[101]:


model = Sequential()


# In[102]:


# Import the most common type of network layer, fully interconnected


# In[103]:


from keras.layers import Dense


# <center><img src="images/dense.png" width="55%"/></center>

# In[104]:


# Define input layer


# In[105]:


layer_input = Dense(units=512,            # Number of nodes
                    activation='sigmoid', # The nonlinearity
                    input_shape=(image_size,)) 
model.add(layer_input)


# In[106]:


# Define another layer


# In[107]:


model.add(Dense(units=512, activation='sigmoid'))


# In[108]:


# Define output layers


# In[109]:


layer_output = Dense(units=10,             # Number of digits (0-9)
                     activation='softmax') # Convert neural activation to probability of category

model.add(layer_output)


# In[110]:


# Print summary


# In[111]:


model.summary()


# In[112]:


# Yes - we compile the model to run it


# In[113]:


model.compile(loss='categorical_crossentropy', 
              optimizer='sgd',
              metrics=['accuracy'])


# In[114]:


# Train the model


# In[115]:


training = model.fit(x_train, 
                     y_train,
                     epochs=5, # Number of passes over complete dataset
                     verbose=True, 
                     validation_split=0.1)


# <center><img src="images/waiting.jpg" width="55%"/></center>

# In[116]:


# Let's see how well our model performs


# In[117]:


loss, accuracy = model.evaluate(x_test, 
                                y_test, 
                                verbose=True)
print(f"Test loss: {loss:.3}")
print(f"Test accuracy: {accuracy:.3%}")


# Keras' Other Features
# -----
# 
# - Common built-in functions (e.g., activation functions and optimitizers)
# - Convolutional neural network (CNN or ConvNet)
# - Recurrent neural network (RNN) & Long-short term memory (LSTM)
# - Pre-trained models

# Summary
# -----
# 
# - Keras is designed for human beings, not computers.
# - Easier to try out Deep Learning (focus on the __what__, not the __how__).
# - Simple to define neural networks.

# <center><img src="images/twitter.png" width="100%"/></center>

# Futher Study - Keras
# --------
# 
# - [Keras docs](https://keras.io/)
# - [Keras blog](https://blog.keras.io/)
# - Keras courses
#     - [edX](https://www.edx.org/course/deep-learning-fundamentals-with-keras)
#     - [Coursera](https://www.coursera.org/lecture/ai/keras-overview-7GfN9)

# Futher Study - Deep Learning
# --------
# 
# - Prerequisites: Linear Algebra, Probability, Machine Learning
# - [fast.ai Course](http://www.fast.ai/)
# - [Deep Learning Book](http://www.deeplearningbook.org/)

# <br>

# Bonus Material
# --------

# In[118]:


# reset -fs


# In[119]:


# from keras import *


# In[120]:


# whos


# In[121]:


# from keras.datasets import fashion_mnist


# In[122]:


# # Setup train and test splits
# (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()


# In[123]:


# from random import randint
# from matplotlib import pyplot

# %matplotlib inline


# In[124]:


# pyplot.imshow(x_train[randint(0, x_train.shape[0])], cmap='gray_r');


# In[125]:


# # Define CNN model

# # Redefine input dimensions to make sure conv works
# img_rows, img_cols = 28, 28
# x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
# x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
# input_shape = (img_rows, img_cols, 1)


# In[126]:


# import keras


# In[127]:


# # Convert class vectors to binary class matrices
# y_train = keras.utils.to_categorical(y_train, 10)
# y_test = keras.utils.to_categorical(y_test, 10)


# In[128]:


# from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D


# In[129]:


# # Define model
# model = Sequential()
# model.add(Conv2D(32, 
#              kernel_size=(3, 3),
#              activation='sigmoid',
#              input_shape=input_shape))
# model.add(Conv2D(64, (3, 3), activation='sigmoid'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Flatten())
# model.add(Dense(128, activation='sigmoid'))
# model.add(Dense(10, activation='softmax'))


# In[130]:


# model.compile(loss='categorical_crossentropy', 
#               optimizer='adam',
#               metrics=['accuracy'])


# In[131]:


# # Define training
# training = model.fit(x_train, 
#                      y_train,
#                      epochs=5,
#                      verbose=True, 
#                      validation_split=0.1)


# In[132]:


# loss, accuracy = model.evaluate(x_test, 
#                                 y_test, 
#                                 verbose=True)
# print(f"Test loss: {loss:.3}")
# print(f"Test accuracy: {accuracy:.3%}")


# What is `keras`?    
# -----
# 
# <center><img src="https://www.thevintagenews.com/wp-content/uploads/2017/08/a-drinking-horn-from-the-16th-century-known-as-the-roordahuizum-drinking-horn-on-display-in-the-frisian-museum-at-leeuwarden-640x360.jpg" width="75%"/></center>
# 
# Keras (κέρας) means horn in Greek. 

# It is a reference to a literary image from ancient Greek and Latin literature.
# 
# First found in the Odyssey, where dream spirits (Oneiroi, singular Oneiros) are divided between those who deceive men with false visions, who arrive to Earth through a gate of ivory, and those who announce a future that will come to pass, who arrive through a gate of horn. 
# 
# It's a play on the words κέρας (horn) / κραίνω (fulfill), and ἐλέφας (ivory) / ἐλεφαίρομαι (deceive).

# [Source](https://keras.io/#why-this-name-keras)

# <br>