diff --git a/README.md b/README.md index ae0c673..a61202e 100644 --- a/README.md +++ b/README.md @@ -1 +1,49 @@ -# simple_neural_network \ No newline at end of file +# Simple Neural Network Maker +a neural network maker uses preceptrons to create a neural network and implementation of forward and back propagation using Chain rule instead of matrix-based approach + +## Create Network: +### Create Perceptron: +a perceptron apply activation function on it's inputs sum. +use following command to create a perceptron `p` with initial weights `W` and activation function `f`: +``` +p = Perceptron(W,f) +``` +to calculate perceptron output and its derivatives with input `X` can use this command: +``` +p.cal_output(X) +``` +and for updating it's weights after back-propagation: +``` +p.update_weights() +``` +### Build Network: +to create a neural network with loss function `loss` +``` +n = Network(loss) +``` +a neural network must have at least one 'input' and one 'output' perceptron. +you can use several 'hidden' neurons in your network. +in designing the structure of network there is almost no limit. you can connect neurons from inputs to output in any way. +for adding perceptron `p` with `i` input and type `t`(output,input,hidden) to the network use: +``` +n.add_node(p,t,i) +``` +and for connecting neuron `p1` to `i`'th `p2`'s input use this +``` +n.connect_nodes(p1,p2,i) +``` +after designing the network, it's time to learn our network with training dataset. as you know, training procedure contains two part: forward-propagation and back-propagation. +in each epoch, we once do forward-propagation and then back-propagation. +for feeding data to network we do forward-propagation. with training data `X`, use following command: +``` +n.forward_prop(X) +``` +for updating weights we do back-propagation. with training data labels `Y` and learning rate `alpha`, use following command: +``` +n.back_prop(alpha,Y) +``` + +## To-Do +##### add `compile` function to `Network` class that checks connections between nodes and their inputs +##### make a higher level api + diff --git a/neural.py b/neural.py index 0c82b3a..c9239f7 100644 --- a/neural.py +++ b/neural.py @@ -2,7 +2,8 @@ import math -c = math.pow(10,-10) +# constant c for calculate derivatives +c = math.pow(10,-15) def d1_cal(x,f): #calculates derivative f bY x @@ -13,27 +14,30 @@ def d2_cal(x1,x2,f): #calculates derivative f bY x1 return float(f(x1,x2) - f(x1-c,x2))/float(c) +# loss functions def mse_loss(y_pred,y): return ((y_pred-y)**2)/2 -#activation functions +# activation functions def relu(input): - # print(input) return max(0,input) + def relu_n(input): if input>=0: return input else: return 0.01*input + def step(input): if input>=0: return 1 else: return -1 + class Perceptron(): def __init__(self,W,activation): self.output = [] @@ -55,7 +59,7 @@ def cal_output(self,X): def output_i(w,x): return w*x # print(self.output) - self.d_output.append(X) #is equal X? + self.d_output.append(X) # print(self.d_output) return self.output @@ -70,23 +74,22 @@ def cal_next_W(self,alpha,Y,i): def update_weights(self): self.W = self.W_next - # self.output = [] - # self.d_output = [] return self.W class Network: - def __init__(self): + def __init__(self,l): self.nodes = [] self.types = [] self.loss = 1 + self.loss_function = l def add_node(self,p,t,n): self.nodes.append(p) self.types.append(t) - + def connect_nodes(self,p1,p2,n_of_w_in): p1.next.append((p2,n_of_w_in)) # print(p1.next) @@ -94,11 +97,10 @@ def connect_nodes(self,p1,p2,n_of_w_in): # print(p2.pre) - def cal_loss(self,Y): for p in self.nodes: if self.types[self.nodes.index(p)] == 'output': - self.loss = sum([mse_loss(p.output[i],Y[i]) for i in range(len(Y))])/(len(Y)) + self.loss = sum([self.loss_function(p.output[i],Y[i]) for i in range(len(Y))])/(len(Y)) print('====================') return self.loss @@ -131,7 +133,7 @@ def backward_prop(self,alpha,Y): if self.types[self.nodes.index(p)] == 'output': for pre in p.pre: to_update.add(pre) - self.d_loss = [d2_cal(p.output[i],Y[i],mse_loss) for i in range(len(Y))] + self.d_loss = [d2_cal(p.output[i],Y[i],self.loss_function) for i in range(len(Y))] for i in range(len(Y)): p.c.append(self.d_loss[i]) p.cal_next_W(alpha,Y,i) @@ -166,13 +168,14 @@ def backward_prop(self,alpha,Y): p.c = [] p.update_weights() - -m = 1000 #num of data +# testing... +m = 100 #num of data n = 3 #num of input features -alpha = 1 #learning rate +alpha = 0.1 #learning rate epoch = 1000 #num of epochs +# data generation Y = [] #labels X = np.zeros((m,3)) #inputs for i in range(m): @@ -180,6 +183,7 @@ def backward_prop(self,alpha,Y): X[i][j] = np.random.normal() Y.append(sum([t**2 for t in X[i]])) +# initializing weights W = np.zeros(n) #initial weights for i in range(n): W[i]=np.random.normal() @@ -189,15 +193,28 @@ def backward_prop(self,alpha,Y): # print(Y[0]) # input() -N = Network() +# building network +N = Network(mse_loss) p1 = Perceptron(W,relu_n) p2 = Perceptron(W,relu_n) p3 = Perceptron(W[0:2],relu_n) -N.add_node(p3,'output',2) +p4 = Perceptron(W[0:2],relu_n) +p5 = Perceptron(W[0:2],relu_n) + +N.add_node(p5,'output',2) +N.add_node(p4,'hidden',2) +N.add_node(p3,'hidden',2) N.add_node(p2,'input',3) N.add_node(p1,'input',3) + N.connect_nodes(p1,p3,0) N.connect_nodes(p2,p3,1) +N.connect_nodes(p1,p4,0) +N.connect_nodes(p2,p4,1) +N.connect_nodes(p3,p5,0) +N.connect_nodes(p4,p5,1) + +# learning for i in range(epoch): print('--forward start--') N.forward_prop(X)