Automatic-Speech-Recognition-using-Deep-Neural-Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d072dece",
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Conv2D, MaxPool2D, BatchNormalization, Bidirectional\n",
    "from keras.layers import Activation, Dropout, Flatten, Dense, Reshape, LSTM, Input, Lambda\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import regularizers\n",
    "from keras import backend as K\n",
    "import tensorflow as tf\n",
    "#import cv2\n",
    "#import time\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "import keras.utils\n",
    "from keras import utils as np_utils\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "05d10890",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_dir = \"E:/New folder/train\"\n",
    "#test_data_dir = 'E:/testing dataset'\n",
    "\n",
    "batch_size = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "738a374e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_datagen = ImageDataGenerator(rescale=1. /255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "27a6013d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 4592 images belonging to 39 classes.\n"
     ]
    }
   ],
   "source": [
    "train = train_datagen.flow_from_directory(train_data_dir,\n",
    "                                                    target_size=(128, 128),\n",
    "                                                    batch_size=32,\n",
    "                                                    class_mode ='categorical'\n",
    "                                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "898fdfe2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['aa', 'ae', 'ah', 'aw', 'axr', 'ay', 'b', 'bcl', 'ch', 'd', 'dh', 'dx', 'eh', 'em', 'ey', 'f', 'g', 'hh', 'ih', 'iy', 'jh', 'k', 'l', 'n', 'ng', 'ow', 'oy', 'p', 'r', 's', 'sh', 't', 'th', 'uh', 'uw', 'v', 'w', 'y', 'z']\n",
      "The vocabulary is: ['[UNK]', 'aa', 'ae', 'ah', 'aw', 'axr', 'ay', 'b', 'bcl', 'ch', 'd', 'dh', 'dx', 'eh', 'em', 'ey', 'f', 'g', 'hh', 'ih', 'iy', 'jh', 'k', 'l', 'n', 'ng', 'ow', 'oy', 'p', 'r', 's', 'sh', 't', 'th', 'uh', 'uw', 'v', 'w', 'y', 'z'] (size =40)\n",
      "tf.Tensor(\n",
      "[ 1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24\n",
      " 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39], shape=(39,), dtype=int64)\n"
     ]
    }
   ],
   "source": [
    "classes = train.class_indices\n",
    "label = list(classes.keys())\n",
    "#labels = ' '.join(map(str, labels))\n",
    "#labels = labels.encode()\n",
    "print(label)\n",
    "\n",
    "# The set of characters accepted in the transcription.\n",
    "characters = ['aa', 'ae', 'ah', 'aw', 'axr', 'ay', 'b', 'bcl', 'ch', 'd', 'dh', 'dx', 'eh', 'em', 'ey', \n",
    "              'f', 'g', 'hh', 'ih', 'iy', 'jh', 'k','l', 'n', 'ng', 'ow', 'oy', 'p', \n",
    "              'r', 's', 'sh', 't', 'th', 'uh', 'uw', 'v', 'w', 'y', 'z']\n",
    "# Mapping characters to integers\n",
    "char_to_num = keras.layers.StringLookup(vocabulary=characters)\n",
    "print(\n",
    "    f\"The vocabulary is: {char_to_num.get_vocabulary()} \"\n",
    "    f\"(size ={char_to_num.vocabulary_size()})\"\n",
    ")\n",
    "label = [x.encode('utf-8') for x in label]\n",
    "labels = char_to_num(label)\n",
    "print(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7efa4306",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 4592 images belonging to 39 classes.\n"
     ]
    }
   ],
   "source": [
    "validation = train_datagen.flow_from_directory(train_data_dir,\n",
    "                                                             target_size=(128, 128),\n",
    "                                                             batch_size=32,\n",
    "                                                             class_mode='categorical')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7b172111",
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4baaaad6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def CTCLoss(y_true, y_pred):\n",
    "    # Compute the training-time loss value\n",
    "    batch_len = tf.cast(tf.shape(y_true)[0], dtype=\"int64\")\n",
    "    input_length = tf.cast(tf.shape(y_pred)[1], dtype=\"int64\")\n",
    "    labels_length = tf.cast(tf.shape(y_true)[1], dtype=\"int64\")\n",
    "\n",
    "    input_length = input_length * tf.ones(shape=(batch_len, 1), dtype=\"int64\")\n",
    "    labels_length = labels_length * tf.ones(shape=(batch_len, 1), dtype=\"int64\")\n",
    "\n",
    "    loss = keras.backend.ctc_batch_cost(y_true, y_pred, input_length, labels_length)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8fc49df3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"Model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input (InputLayer)          [(None, 128, 128, 3)]     0         \n",
      "                                                                 \n",
      " conv_1 (Conv2D)             (None, 128, 128, 32)      384       \n",
      "                                                                 \n",
      " conv_1_bn (BatchNormalizati  (None, 128, 128, 32)     128       \n",
      " on)                                                             \n",
      "                                                                 \n",
      " conv_1_relu (ReLU)          (None, 128, 128, 32)      0         \n",
      "                                                                 \n",
      " conv_2 (Conv2D)             (None, 128, 128, 64)      8192      \n",
      "                                                                 \n",
      " conv_2_bn (BatchNormalizati  (None, 128, 128, 64)     256       \n",
      " on)                                                             \n",
      "                                                                 \n",
      " conv_2_relu (ReLU)          (None, 128, 128, 64)      0         \n",
      "                                                                 \n",
      " conv_3 (Conv2D)             (None, 128, 128, 128)     32768     \n",
      "                                                                 \n",
      " conv_3_bn (BatchNormalizati  (None, 128, 128, 128)    512       \n",
      " on)                                                             \n",
      "                                                                 \n",
      " conv_3_relu (ReLU)          (None, 128, 128, 128)     0         \n",
      "                                                                 \n",
      " reshape (Reshape)           (None, 128, 16384)        0         \n",
      "                                                                 \n",
      " bidirectional_1 (Bidirectio  (None, 128, 256)         12682752  \n",
      " nal)                                                            \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 128, 256)          65792     \n",
      "                                                                 \n",
      " dense_1_relu (ReLU)         (None, 128, 256)          0         \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 128, 256)          0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 128, 40)           10280     \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 12,801,064\n",
      "Trainable params: 12,800,616\n",
      "Non-trainable params: 448\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def build_model(output_dim, rnn_layers=1, rnn_units=128):\n",
    "\n",
    "    # Model's input\n",
    "    input_spectrogram = layers.Input((128, 128, 3), name=\"input\")\n",
    "    # Expand the dimension to use 2D CNN.\n",
    "    #x = layers.Reshape((-1, input_shape, 3), name=\"expand_dim\")(input_spectrogram)\n",
    "    # Convolution layer 1\n",
    "    x = layers.Conv2D(\n",
    "        filters=32,\n",
    "        kernel_size=[2, 2],\n",
    "        padding=\"same\",\n",
    "        use_bias=False,\n",
    "        name=\"conv_1\",\n",
    "    )(input_spectrogram)\n",
    "    x = layers.BatchNormalization(name=\"conv_1_bn\")(x)\n",
    "    x = layers.ReLU(name=\"conv_1_relu\")(x)\n",
    "    # Convolution layer 2\n",
    "    x = layers.Conv2D(\n",
    "        filters=64,\n",
    "        kernel_size=[2, 2],\n",
    "        padding=\"same\",\n",
    "        use_bias=False,\n",
    "        name=\"conv_2\",\n",
    "    )(x)\n",
    "    x = layers.BatchNormalization(name=\"conv_2_bn\")(x)\n",
    "    x = layers.ReLU(name=\"conv_2_relu\")(x)\n",
    "    # Convolution layer 3\n",
    "    x = layers.Conv2D(\n",
    "        filters=128,\n",
    "        kernel_size=[2, 2],\n",
    "        padding=\"same\",\n",
    "        use_bias=False,\n",
    "        name=\"conv_3\",\n",
    "    )(x)\n",
    "    x = layers.BatchNormalization(name=\"conv_3_bn\")(x)\n",
    "    x = layers.ReLU(name=\"conv_3_relu\")(x)\n",
    "    # Reshape the resulted volume to feed the RNNs layers\n",
    "    x = layers.Reshape((-1, x.shape[-2] * x.shape[-1]))(x)\n",
    "    # RNN layers\n",
    "    for i in range(1, rnn_layers + 1):\n",
    "        recurrent = layers.GRU(\n",
    "            units=rnn_units,\n",
    "            activation=\"tanh\",\n",
    "            recurrent_activation=\"sigmoid\",\n",
    "            use_bias=True,\n",
    "            return_sequences=True,\n",
    "            reset_after=True,\n",
    "            name=f\"gru_{i}\",\n",
    "        )\n",
    "        x = layers.Bidirectional(\n",
    "            recurrent, name=f\"bidirectional_{i}\", merge_mode=\"concat\"\n",
    "        )(x)\n",
    "        if i < rnn_layers:\n",
    "            x = layers.Dropout(rate=0.5)(x)\n",
    "    # Dense layer\n",
    "    x = layers.Dense(units=rnn_units * 2, name=\"dense_1\")(x)\n",
    "    x = layers.ReLU(name=\"dense_1_relu\")(x)\n",
    "    x = layers.Dropout(rate=0.5)(x)\n",
    "    # Classification layer\n",
    "    output = layers.Dense(units=output_dim + 1, activation=\"softmax\")(x)\n",
    "    # Model\n",
    "    model = keras.Model(input_spectrogram, output, name=\"Model\")\n",
    "    # Optimizer\n",
    "    #opt = keras.optimizers.Adam(learning_rate=1e-4)\n",
    "    # Compile the model and return\n",
    "    model.compile(optimizer=\"Adam\", loss=CTCLoss)\n",
    "    return model\n",
    "# Get the model\n",
    "model = build_model(\n",
    "    output_dim=39,\n",
    "    rnn_units=128,\n",
    ")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95451a9e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      " 68/144 [=============>................] - ETA: 55:45 - loss: 17.8760"
     ]
    }
   ],
   "source": [
    "# Define the number of epochs.\n",
    "epochs = 30\n",
    "# Callback function to check transcription on the val set.\n",
    "#validation_callback = CallbackEval(validation)\n",
    "# Train the model\n",
    "history = model.fit(\n",
    "    train,\n",
    "    validation_data=validation,\n",
    "    epochs=epochs\n",
    "    #callbacks=[validation_callback],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c25c1b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "speech_file = (\"E:/anveshan/archive/timit/timit/dr8-mbcg0/sa1.wav\")\n",
    "\n",
    "IPython.display.Audio(speech_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30fb51ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython.display as ipd\n",
    "import librosa\n",
    "import librosa.display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "914ec2e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(13,5))\n",
    "data,sample_rate=librosa.load(speech_file)\n",
    "waveform = librosa.display.waveshow(data,sample_rate)\n",
    "ipd.Audio(speech_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51f9a4d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85868262",
   "metadata": {},
   "outputs": [],
   "source": [
    "frame_size=2048\n",
    "hop_size=512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e1dd157",
   "metadata": {},
   "outputs": [],
   "source": [
    "def spectrogram(data,path):\n",
    "    signal=librosa.stft(y=data, hop_length=hop_size, \n",
    "                                   n_fft=frame_size)\n",
    "    spectrogram = np.abs(signal)\n",
    "    power_to_db = librosa.power_to_db(spectrogram, ref=np.max)\n",
    "    plt.figure(figsize=(8, 4))\n",
    "    librosa.display.specshow(power_to_db, sr=sample_rate, x_axis='time', y_axis='mel', \n",
    "     hop_length=hop_size)\n",
    "    plt.savefig(path)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7153c80e",
   "metadata": {},
   "outputs": [],
   "source": [
    "spectrogram(data, \"C:/Users/HP/Desktop/ddddd_new\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c5e5aac",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import cv2\n",
    "\n",
    "img = cv2.imread(\"C:/Users/HP/Desktop/ddddd_new.png\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fadf69f",
   "metadata": {},
   "outputs": [],
   "source": [
    "img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ff1afb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "img.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d924bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "img = tf.image.convert_image_dtype(img, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10cef536",
   "metadata": {},
   "outputs": [],
   "source": [
    "img = tf.image.resize(img, [128, 128])\n",
    "img = tf.transpose(img, perm=[1, 0, 2])\n",
    "img = tf.expand_dims(img, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7268b52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mapping integers back to original characters\n",
    "num_to_char = keras.layers.StringLookup(\n",
    "    vocabulary=char_to_num.get_vocabulary(), invert=True\n",
    ")\n",
    "def decode_predictions(y_pred):\n",
    "    input_len = np.ones(y_pred.shape[0]) * y_pred.shape[1]\n",
    "    # Use greedy search. For complex tasks, you can use beam search\n",
    "    results = keras.backend.ctc_decode(y_pred, input_length=input_len, greedy=False, beam_width=100, top_paths=3 )[0][0]\n",
    "    #print(results)\n",
    "    # Iterate over the results and get back the text\n",
    "    output_text = []\n",
    "    \n",
    "    for res in results:\n",
    "        res = tf.strings.reduce_join(num_to_char(res)).numpy().decode(\"utf-8\")\n",
    "        output_text.append(res)\n",
    "    \n",
    "    return output_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "116b9aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "targets = \"She had your dark suit in greasy wash water all year\"\n",
    "#targets = targets.split()\n",
    "\n",
    "preds = model.predict(img)\n",
    "pred_text = decode_predictions(preds)\n",
    "print(\"Target:\", targets)\n",
    "print(\"predictions:\", pred_text[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca231ec9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}