BTP_svm_char_ngram.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BTP_svm_char_ngram.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "13ejNUYXYYaUV9oZYJVPDlLP9cAMFV_Vk",
      "authorship_tag": "ABX9TyPcIoMgithpNN2ZJZyPqXsq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shreyajain4/WordLevelLangIdentificationJaech/blob/master/BTP_svm_char_ngram.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hNp_7H_L7Vv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import argparse\n",
        "import collections\n",
        "import json\n",
        "import logging\n",
        "import numpy as np\n",
        "import os\n",
        "import shutil\n",
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import itertools\n",
        "import re\n",
        "import pickle\n",
        "import random\n",
        "import matplotlib\n",
        "import csv\n",
        "from sklearn.datasets.samples_generator import make_blobs \n",
        "matplotlib.rc('font', family='DejaVu Sans')\n",
        "from matplotlib import pyplot\n",
        "from nltk import ngrams\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P4APsDUNc_Or",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_word_len = 13\n",
        "\n",
        "sentences = []\n",
        "words = []\n",
        "labels = []\n",
        "l = []\n",
        "i = 0\n",
        "n = 1\n",
        "w = []\n",
        "\n",
        "length = []\n",
        "# with open(\"drive/My Drive/dataset/POS Hindi English Code Mixed Tweets.tsv\") as tsvfile:\n",
        "#     tsvreader = csv.reader(tsvfile, delimiter=\"\\t\")\n",
        "#     for line in tsvreader:\n",
        "#       i = i+1\n",
        "#       if(len(line)==3 and line[1]!='rest'):\n",
        "#         length.append(len(Graphemes(line[0])))\n",
        "# length=np.array(length)\n",
        "# a=np.percentile(length,99.97)\n",
        "all_grams = set()\n",
        "word_unique = dict()\n",
        "with open(\"drive/My Drive/dataset/POS Hindi English Code Mixed Tweets.tsv\") as tsvfile:\n",
        "    tsvreader = csv.reader(tsvfile, delimiter=\"\\t\")\n",
        "    for line in tsvreader:\n",
        "      i = i+1\n",
        "      if (len(line) == 3):\n",
        "        if (len(line)>0 and (line[0]=='0' or line[0]=='')):\n",
        "          words.append(w) \n",
        "          labels.append(l)\n",
        "          \n",
        "          w = []\n",
        "          l = []\n",
        "        elif(line[1]!='rest'):\n",
        "          w.append(line[0])\n",
        "          l.append(line[1])\n",
        "          word_unique[line[0].lower()]=line[1]\n",
        "          word=line[0]\n",
        "          sixgrams = ngrams(list(word.lower()), n)\n",
        "          for grams in sixgrams:\n",
        "            g =  ''.join(grams)\n",
        "            # print (g)\n",
        "            all_grams.add(g)\n",
        "\n",
        "      elif (len(line)==0 or (len(line)>0 and (line[0]=='0' or line[0]==''))):\n",
        "          words.append(w) \n",
        "          labels.append(l)\n",
        "          w = []\n",
        "          l = []\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PgiSrcw4mi7C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5oFSOpG1kRxa",
        "colab_type": "code",
        "outputId": "f0898fd9-9e17-432a-ae98-112dff4507db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "ngram_voc=dict()\n",
        "def vocab():\n",
        "  i=0\n",
        "  for word in all_grams:\n",
        "    ngram_voc[word.lower()]=i\n",
        "    i=i+1\n",
        "vocab()\n",
        "ngram_voc\n",
        "\n",
        "out_vocab=dict()\n",
        "out_vocab['hi']=0\n",
        "out_vocab['en']=1\n",
        "len(ngram_voc)"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "41"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CnRleSCbHkaU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tC0Y3VzMqF92",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x=np.zeros((len(word_unique),len(ngram_voc)))\n",
        "y=np.zeros((len(word_unique)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Sxxr6j2nOTC",
        "colab_type": "code",
        "outputId": "12dc39ad-0f8c-4ec3-ef53-fc20b9157d2f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "i=0\n",
        "ww = []\n",
        "p=[]\n",
        "for w, l in word_unique.items():\n",
        "  word=w\n",
        "  sixgrams = ngrams(list(word.lower()), n)\n",
        "  # if (i<5):\n",
        "  #   print (list(word.lower()))\n",
        "  for grams in sixgrams:\n",
        "    g =  ''.join(grams)\n",
        "    x[i][ngram_voc[g]]=1\n",
        "    if(i==3):\n",
        "      p.append(ngram_voc[g])\n",
        "  if l=='en':\n",
        "    y[i]=1\n",
        "  i=i+1\n",
        "p"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[16, 6, 25, 6, 10]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0rYergniu-hl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# word_unique\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9fSIwiRXuvv4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# x[3][662]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lo-ONg2OsVPQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iHykcwK9seLY",
        "colab_type": "code",
        "outputId": "3dff7e14-c828-4afb-b028-818ccde6d07c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# p=x[0]==np.zeros((1,len(ngram_voc)))\n",
        "# p\n",
        "k=0\n",
        "indeces = []\n",
        "for z in x[0]:\n",
        "  if(z==1):\n",
        "    indeces.append(k)\n",
        "  k = k+1\n",
        "indeces"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[16, 20]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CiWmnG8_mIm1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# X, Y = make_blobs(n_samples=len(word_unique), centers=2, random_state=0, cluster_std=0.40) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iiTs7lOGsT70",
        "colab_type": "code",
        "outputId": "768daa9a-4bf7-4a0e-832a-a64b90ee4ee3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from sklearn.svm import SVC # \"Support Vector Classifier\" \n",
        "clf = SVC(kernel='rbf') \n",
        "len(word_unique)"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3113"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gSNDUCSknabq",
        "colab_type": "code",
        "outputId": "f247ba41-7d90-4094-bd15-63781820eef1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "clf.fit(x[0:2700], y[0:2700]) \n"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SVC(C=1.0, break_ties=False, cache_size=200, class_weight=None, coef0=0.0,\n",
              "    decision_function_shape='ovr', degree=3, gamma='scale', kernel='rbf',\n",
              "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
              "    tol=0.001, verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 71
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFk2f0ZDPzgQ",
        "colab_type": "code",
        "outputId": "5033a9ad-17d4-41fd-efc7-491b841c637f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#new_word = np.zeros(1,len(ngram_voc)))\n",
        "clf.score(x[2700:],y[2700:])\n"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8038740920096852"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ejw8iqDyZ_VS",
        "colab_type": "code",
        "outputId": "64fe3987-b616-4701-d47b-f068de6f1ea7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "len(ngram_voc)"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "41"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i3NY5DoomI9K",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    }
  ]
}