softmax Activation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "174c01d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Softmax is for output layer.\n",
    "the first thing to check the model is to see how wrong it it.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0a9a223b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[121.51041751873483, 3.353484652549023, 29.517992716151483]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "what we will do here is to use exponential function. This is because output can ne in neg too and we have to tackle that\n",
    "issue\n",
    "\"\"\"\n",
    "import math\n",
    "output_layer = [4.8,1.21,3.385]\n",
    "E =math.e\n",
    "expo_val = []\n",
    "for i in output_layer:\n",
    "    expo_val.append(E**i)\n",
    "print(expo_val)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e0d34359",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.7870768629140863, 0.02172200733119744, 0.1912011297547162]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Now we will normalize the value\n",
    "y = [u1/u1+u2\n",
    "      u2/u1+u2]\n",
    "this will give us probabilty distribution but we have to do exponential first. to convert neg to pos without giving\n",
    "up on the values.\n",
    "norm is done after exponetianl\n",
    "\"\"\"\n",
    "norm_sum = sum(expo_val)\n",
    "norm_val = []\n",
    "for i in expo_val:\n",
    "    norm_val.append(i/norm_sum)\n",
    "print(norm_val)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f53993cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.78707686 0.02172201 0.19120113]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Every thing using Numpy\n",
    "\"\"\"\n",
    "\n",
    "import numpy as np\n",
    "import math\n",
    "import nnfs\n",
    "nnfs.init()\n",
    "output_layer = [4.8,1.21,3.385]\n",
    "exponential_val = np.exp(output_layer)\n",
    "norm_value =exponential_val/np.sum(exponential_val)\n",
    "print(norm_value)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "46319f08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7.87076863e-01 2.17220073e-02 1.91201130e-01]\n",
      " [9.99811129e-01 2.23163963e-05 1.66554348e-04]\n",
      " [5.13097164e-01 3.58333899e-01 1.28568936e-01]]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Input > exponential > normalize > output\n",
    "exponential + normalization  = softmax activation\n",
    "lets make a batch size\n",
    "\"\"\"\n",
    "nnfs.init()\n",
    "Output = [[4.8,1.21,3.385],\n",
    "         [8.9,-1.81,0.2],\n",
    "         [1.41,1.051,0.026]]\n",
    "Exponential_val = np.exp(Output)\n",
    "#axis=1 means row wise sum\n",
    "Norm_val =Exponential_val/np.sum(Exponential_val, axis=1, keepdims=True)\n",
    "print(Norm_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "cd238f62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.33333334 0.33333334 0.33333334]\n",
      " [0.33331734 0.3333183  0.33336434]\n",
      " [0.3332888  0.33329153 0.33341965]\n",
      " [0.33325943 0.33326396 0.33347666]\n",
      " [0.33323312 0.33323926 0.33352762]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import nnfs\n",
    "from nnfs.datasets import spiral_data\n",
    "nnfs.init()\n",
    "##we will also use this for our dataset\n",
    "\n",
    "X = [[1,2,3,2.5],\n",
    "    [2.0,5.0,-1.0,2.0],\n",
    "    [-1.5,2.7,3.3,-0.8]]\n",
    "Input = [0,2,-1,3.3,-2.7,1.1,2.2,-100]\n",
    "X,y = spiral_data(100,3)\n",
    "class layer_Dense:\n",
    "    def __init__(self,n_input,n_neurons):\n",
    "        self.weights = 0.10*np.random.randn(n_input,n_neurons)\n",
    "        self.biases = np.zeros((1,n_neurons))\n",
    "   \n",
    "    def forward(self,inputs):\n",
    "        self.output = np.dot(inputs,self.weights)+self.biases\n",
    "\n",
    "class Activation_Relu:\n",
    "    #it will get input and produce an output\n",
    "    def forward(self,Input):\n",
    "        self.output = np.maximum(0,Input)\n",
    "        \n",
    "class Softmax_Activation:\n",
    "    def forward(self,Input):\n",
    "        ##this way w will not hit the overflow value\n",
    "        exp_val = np.exp(Input-np.max(Input,axis=1,keepdims=True))\n",
    "        probabilities  = exp_val/np.sum(exp_val,axis = 1 , keepdims=True)\n",
    "        self.output = probabilities\n",
    "\n",
    "layer1 = layer_Dense(2,3)\n",
    "activation1  = Activation_Relu()\n",
    "layer2 = layer_Dense(3,3)\n",
    "activation2  = Softmax_Activation()\n",
    "\n",
    "layer1.forward(X)\n",
    "activation1.forward(layer1.output)\n",
    "\n",
    "layer2.forward(activation1.output)\n",
    "activation2.forward(layer2.output)\n",
    "print(activation2.output[:5]) #start 5 val of batch\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcbd28e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}