a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
Implementing Neural Network from
Scratch.
Neural Networks are really powerful algorithms used for classification.
Dataset = Iris_Dataset
Link = http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html
(http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html)
Import required libraries
In [ ]: from sklearn import datasets #for dataset
import numpy as np #for maths
import matplotlib.pyplot as plt #for plotting
Get Dataset
In [ ]: iris = datasets.load_iris() #load the dataset
data = iris.data #get features
target = iris.target #get labels
shape = data.shape #shape of data
#convert into numpy array
data = np.array(data).reshape(shape[0],shape[1])
target = np.array(target).reshape(shape[0],1)
#print shape
print("Data Shape = {}".format(data.shape))
print("Target Shape = {}".format(target.shape))
print('Classes : {}'.format(np.unique(target)))
print('Sample data : {} , Target = {}'.format(data[70],target[70]))
Define Parameters and Hyperparameters
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 1 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
One hidden layer Neural Network.
Input Units = 4
Hidden Units = 8
Output Units = 3
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 2 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
In [ ]: #HYPERPARAMETERS
#num of target labels
num_classes = len(np.unique(target))
#define layer_neurons
input_units = 4 #neurons in input layer
hidden_units = 8 #neurons in hidden layer
output_units = 3 #neurons in output layer
#define hyper-parameters
learning_rate = 0.03
#regularization parameter
beta = 0.00001
#num of iterations
iters = 4001
Dimesions of Parameters
Shape of layer1_weights (Wxh) = (4,8)
Shape of layer1_biasess (Bh) = (8,1)
Shape of layer2_weights (Why) = (8,3)
Shape of layer2_biasess (By) = (3,1)
In [ ]: #PARAMETERS
#initialize parameters i.e weights
def initialize_parameters():
#initial values should have zero mean and 0.1 standard deviation
mean = 0 #mean of parameters
std = 0.03 #standard deviation
layer1_weights = np.random.normal(mean,std,(input_units,hidden_unit
layer1_biases = np.ones((hidden_units,1))
layer2_weights = np.random.normal(mean,std,(hidden_units,output_uni
layer2_biases = np.ones((output_units,1))
parameters = dict()
parameters['layer1_weights'] = layer1_weights
parameters['layer1_biases'] = layer1_biases
parameters['layer2_weights'] = layer2_weights
parameters['layer2_biases'] = layer2_biases
return parameters
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 3 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
Activation Function
Sigmoid
0.5
0
−6 −4 −2 0 2 4 6
In [ ]: #activation function
def sigmoid(X):
return 1/(1+np.exp((-1)*X))
#softmax function for output
def softmax(X):
exp_X = np.exp(X)
exp_X_sum = np.sum(exp_X,axis=1).reshape(-1,1)
exp_X = (exp_X/exp_X_sum)
return exp_X
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 4 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
Define Utility Functions
1. Forward Propagation
---- Logits = matmul(X,Wxh) + Bh
---- A = sigmoid(logits)
---- logits = matmul(A,Why) + By
---- output = softmax(logits)
Store output and A in cache to use it in backward propagation
2. Backward Propagation
---- Error_output = output - train_labels
---- Error_activation = (matmul(error_output,Why.T))(A)(1-A)
---- dWhy = (matmul(A.T,error_output))/m
---- dWxh = (matmul(train_dataset.T,error_activation))/m
m = len(train_dataset)
Store derivatives in derivatives dict
3. Update Parameters
---- Wxh = Wxh - learning_rate(dWxh + betaWxh)
---- Why = Why - learning_rate(dWhy + betaWhy)
4. Calculate Loss and Accuracy
---- Loss = (-1(Y log(prediction)) + (1-Y) (log(1-predictions))) + beta * (sum(Wxh^2) +
sum(Why^2)))/m
---- Accuracy = sum(Y==predictions)/m
In [ ]: #forward propagation
def forward_propagation(train_dataset,parameters):
cache = dict() #to store the intermediate values for bac
m = len(train_dataset) #number of training examples
#get the parameters
layer1_weights = parameters['layer1_weights']
layer1_biases = parameters['layer1_biases']
layer2_weights = parameters['layer2_weights']
layer2_biases = parameters['layer2_biases']
#forward prop
logits = np.matmul(train_dataset,layer1_weights) + layer1_biases
activation1 = np.array(sigmoid(logits)).reshape(m,hidden_units)
activation2 = np.array(np.matmul(activation1,layer2_weights) + laye
output = np.array(softmax(activation2)).reshape(m,num_classes)
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 5 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
#fill in the cache
cache['output'] = output
cache['activation1'] = activation1
return cache,output
#backward propagation
def backward_propagation(train_dataset,train_labels,parameters,cache
derivatives = dict() #to store the derivatives
#get stuff from cache
output = cache['output']
activation1 = cache['activation1']
#get parameters
layer1_weights = parameters['layer1_weights']
layer2_weights = parameters['layer2_weights']
#calculate errors
error_output = output - train_labels
error_activation1 = np.matmul(error_output,layer2_weights.T)
error_activation1 = np.multiply(error_activation1,activation1)
error_activation1 = np.multiply(error_activation1,1-activation1)
#calculate partial derivatives
partial_derivatives2 = np.matmul(activation1.T,error_output)/len
partial_derivatives1 = np.matmul(train_dataset.T,error_activation1
#store the derivatives
derivatives['partial_derivatives1'] = partial_derivatives1
derivatives['partial_derivatives2'] = partial_derivatives2
return derivatives
#update the parameters
def update_parameters(derivatives,parameters):
#get the parameters
layer1_weights = parameters['layer1_weights']
layer2_weights = parameters['layer2_weights']
#get the derivatives
partial_derivatives1 = derivatives['partial_derivatives1']
partial_derivatives2 = derivatives['partial_derivatives2']
#update the derivatives
layer1_weights -= (learning_rate*(partial_derivatives1 + beta*layer
layer2_weights -= (learning_rate*(partial_derivatives2 + beta*layer
#update the dict
parameters['layer1_weights'] = layer1_weights
parameters['layer2_weights'] = layer2_weights
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 6 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
return parameters
#calculate the loss and accuracy
def cal_loss_accuray(train_labels,predictions,parameters):
#get the parameters
layer1_weights = parameters['layer1_weights']
layer2_weights = parameters['layer2_weights']
#cal loss and accuracy
loss = -1*np.sum(np.multiply(np.log(predictions),train_labels) +
accuracy = np.sum(np.argmax(train_labels,axis=1)==np.argmax(predict
accuracy /= len(train_dataset)
return loss,accuracy
Train Function
1. Initialize Parameters
2. Forward Propagation
3. Backward Propagation
4. Calculate Loss and Accuracy
5. Update the parameters
Repeat the steps 2-5 for the given number of iterations
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 7 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
In [ ]: #Implementation of 3 layer Neural Network
#training function
def train(train_dataset,train_labels,iters=2):
#To store loss after every iteration.
J = []
#WEIGHTS
global layer1_weights
global layer1_biases
global layer2_weights
global layer2_biases
#initialize the parameters
parameters = initialize_parameters()
layer1_weights = parameters['layer1_weights']
layer1_biases = parameters['layer1_biases']
layer2_weights = parameters['layer2_weights']
layer2_biases = parameters['layer2_biases']
#to store final predictons after training
final_output = []
for j in range(iters):
#forward propagation
cache,output = forward_propagation(train_dataset,parameters)
#backward propagation
derivatives = backward_propagation(train_dataset,train_labels
#calculate the loss and accuracy
loss,accuracy = cal_loss_accuray(train_labels,output,parameters
#update the parameters
parameters = update_parameters(derivatives,parameters)
#append loss
J.append(loss)
#update final output
final_output = output
#print accuracy and loss
if(j%500==0):
print("Step %d"%j)
print("Loss %f"%loss)
print("Accuracy %f%%"%(accuracy*100))
return J,final_output
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 8 of 9
a-simple-neural-network-from-scratch - Jupyter Notebook 24/11/2020, 13:10
In [ ]: #shuffle the dataset
z = list(zip(data,target))
np.random.shuffle(z)
data,target = zip(*z)
#make train_dataset and train_labels
train_dataset = np.array(data).reshape(-1,4)
train_labels = np.zeros([train_dataset.shape[0],num_classes])
#one-hot encoding
for i,label in enumerate(target):
train_labels[i,label] = 1
#normalizations
for i in range(input_units):
mean = train_dataset[:,i].mean()
std = train_dataset[:,i].std()
train_dataset[:,i] = (train_dataset[:,i]-mean)/std
In [ ]: #train data
J,final_output = train(train_dataset,train_labels,iters=4001)
Reached an Accuracy of 97%
Plot the loss vs iteration graph
In [ ]: #plot loss graph
plt.plot(list(range(1,len(J))),J[1:])
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.title('Iterations VS Loss')
plt.show()
In [ ]:
http://localhost:8889/notebooks/notebooks/ml-learn/nn/a-simple-neural-network-from-scratch.ipynb Page 9 of 9
