N분류_증분의_사본.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KorStats/classification-project/blob/main/N%EB%B6%84%EB%A5%98_%EC%A6%9D%EB%B6%84%EC%9D%98_%EC%82%AC%EB%B3%B8.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U3wrPXQ_ORLb"
      },
      "source": [
        "## 구글드라이브에서 파일 읽어오기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dqJbzoq_OPxK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d12a4478-dd95-4c1c-c50f-d5c985a82eab"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmtIZqMGOdVX",
        "outputId": "fb06752c-8b95-4000-ce7b-239088f4638a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n",
            " 산업분류자동화\t\t df_J_inc.csv\t   E_inc_model.pt    'My Drive'\n",
            " A_inc_model.pt\t\t df_K_inc.csv\t   F_ori_model.pt     N_spell_check.csv\n",
            " B_inc_model.pt\t\t df_M_inc.csv\t   H_ori_model.pt     Q_ori_model.pt\n",
            " B_spellcheck.csv\t df_N_inc.csv\t   H_spellcheck.csv   Q_spellcheck.csv\n",
            " classes.txt\t\t'# df_P_inc.csv'   K_inc_model.pt     R_inc_model.pt\n",
            "'Colab Notebooks'\t df_Q_inc.csv\t   labels.zip\t      S_ori_model.pt\n",
            "'답안 작성용 파일.csv'\t df_R_inc.csv\t   M_inc_model.pt     val.txt\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "!ln -s /content/gdrive/My\\ Drive/ /mydrive\n",
        "!ls /mydrive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xV_iSLl7QFgx",
        "outputId": "17fcace9-08bf-4b41-e7fc-20c5e3cd7c71"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/gdrive/My Drive/산업분류자동화/실습용 자료.zip\n",
            "1. 실습용자료.txt:  mismatching \"local\" filename (1. ьЛдьК╡ьЪйьЮРыгМ.txt),\n",
            "         continuing with \"central\" filename version\n",
            "  inflating: /content/dataset/1. 실습용자료.txt  \n"
          ]
        }
      ],
      "source": [
        "!unzip /content/gdrive/'My Drive'/산업분류자동화/'실습용 자료'.zip -d /content/dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "geFjTXHh5-dy"
      },
      "source": [
        "## 데이터 전처리"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df=pd.read_csv('/content/gdrive/MyDrive/N_spell_check.csv')"
      ],
      "metadata": {
        "id": "nr3DIWiLkC6b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['digit_3'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q73JFxUb4p9d",
        "outputId": "84366244-deae-43f7-a7f2-e8a6a1fc0070"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "751    3712\n",
              "759    3583\n",
              "752    2605\n",
              "762    1948\n",
              "742    1866\n",
              "763    1482\n",
              "761     938\n",
              "741     783\n",
              "753     524\n",
              "743     239\n",
              "764      21\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "bCsVXB0k21ei"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GV4cXBgUea3j"
      },
      "outputs": [],
      "source": [
        "# 데이터 증분 코드\n",
        "import random\n",
        "import pickle\n",
        "import re\n",
        "def get_only_hangul(line):\n",
        "\tparseText= re.compile('/ ^[ㄱ-ㅎㅏ-ㅣ가-힣]*$/').sub('',line)\n",
        "\treturn parseText\n",
        "\n",
        "def synonym_replacement(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\trandom_word_list = list(set([word for word in words]))\n",
        "\trandom.shuffle(random_word_list)\n",
        "\tnum_replaced = 0\n",
        "\tfor random_word in random_word_list:\n",
        "\t\tsynonyms = get_synonyms(random_word)\n",
        "\t\tif len(synonyms) >= 1:\n",
        "\t\t\tsynonym = random.choice(list(synonyms))\n",
        "\t\t\tnew_words = [synonym if word == random_word else word for word in new_words]\n",
        "\t\t\tnum_replaced += 1\n",
        "\t\tif num_replaced >= n:\n",
        "\t\t\tbreak\n",
        "\tif len(new_words) != 0:\n",
        "\t\tsentence = ' '.join(new_words)\n",
        "\t\tnew_words = sentence.split(\" \")\n",
        "\telse:\n",
        "\t\tnew_words = \"\"\n",
        "\treturn new_words\n",
        "\n",
        "def get_synonyms(word):\n",
        "\tsynomyms = []\n",
        "\ttry:\n",
        "\t\tfor syn in wordnet[word]:\n",
        "\t\t\tfor s in syn:\n",
        "\t\t\t\tsynomyms.append(s)\n",
        "\texcept:\n",
        "\t\tpass\n",
        "\treturn synomyms\n",
        "\n",
        "def random_deletion(words, p):\n",
        "\tif len(words) == 1:\n",
        "\t\treturn words\n",
        "\n",
        "\tnew_words = []\n",
        "\tfor word in words:\n",
        "\t\tr = random.uniform(0, 1)\n",
        "\t\tif r > p:\n",
        "\t\t\tnew_words.append(word)\n",
        "\tif len(new_words) == 0:\n",
        "\t\trand_int = random.randint(0, len(words)-1)\n",
        "\t\treturn [words[rand_int]]\n",
        "\treturn new_words\n",
        "\n",
        "########################################################################\n",
        "# Random swap\n",
        "# Randomly swap two words in the sentence n times\n",
        "########################################################################\n",
        "def random_swap(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\tfor _ in range(n):\n",
        "\t\tnew_words = swap_word(new_words)\n",
        "\treturn new_words\n",
        "\n",
        "def swap_word(new_words):\n",
        "\trandom_idx_1 = random.randint(0, len(new_words)-1)\n",
        "\trandom_idx_2 = random_idx_1\n",
        "\tcounter = 0\n",
        "\twhile random_idx_2 == random_idx_1:\n",
        "\t\trandom_idx_2 = random.randint(0, len(new_words)-1)\n",
        "\t\tcounter += 1\n",
        "\t\tif counter > 3:\n",
        "\t\t\treturn new_words\n",
        "\tnew_words[random_idx_1], new_words[random_idx_2] = new_words[random_idx_2], new_words[random_idx_1]\n",
        "\treturn new_words\n",
        "\n",
        "########################################################################\n",
        "# Random insertion\n",
        "# Randomly insert n words into the sentence\n",
        "########################################################################\n",
        "def random_insertion(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\tfor _ in range(n):\n",
        "\t\tadd_word(new_words)\t\n",
        "\treturn new_words\n",
        "\n",
        "def add_word(new_words):\n",
        "\tsynonyms = []\n",
        "\tcounter = 0\n",
        "\twhile len(synonyms) < 1:\n",
        "\t\tif len(new_words) >= 1:\n",
        "\t\t\trandom_word = new_words[random.randint(0, len(new_words)-1)]\n",
        "\t\t\tsynonyms = get_synonyms(random_word)\n",
        "\t\t\tcounter += 1\n",
        "\t\telse:\n",
        "\t\t\trandom_word = \"\"\n",
        "\t\tif counter >= 10:\n",
        "\t\t\treturn\n",
        "\t\t\n",
        "\trandom_synonym = synonyms[0]\n",
        "\trandom_idx = random.randint(0, len(new_words)-1)\n",
        "\tnew_words.insert(random_idx, random_synonym)\n",
        "\n",
        "def EDA(sentence, alpha_sr=0.1, alpha_ri=0.1, alpha_rs=0.1, p_rd=0.1, num_aug=9):\n",
        "\tsentence = get_only_hangul(sentence)\n",
        "\twords = sentence.split(' ')\n",
        "\twords = [word for word in words if word is not \"\"]\n",
        "\tnum_words = len(words)\n",
        "\taugmented_sentences = []\n",
        "\tnum_new_per_technique = int(num_aug/4) + 1\n",
        "\tn_sr = max(1, int(alpha_sr*num_words))\n",
        "\tn_ri = max(1, int(alpha_ri*num_words))\n",
        "\tn_rs = max(1, int(alpha_rs*num_words))\n",
        "\t# sr\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = synonym_replacement(words, n_sr)\n",
        "\t\taugmented_sentences.append(' '.join(a_words))\n",
        "\t# ri\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_insertion(words, n_ri)\n",
        "\t\taugmented_sentences.append(' '.join(a_words))\n",
        "\t# rs\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_swap(words, n_rs)\n",
        "\t\taugmented_sentences.append(\" \".join(a_words))\n",
        "\t# rd\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_deletion(words, p_rd)\n",
        "\t\taugmented_sentences.append(\" \".join(a_words))\n",
        "\taugmented_sentences = [get_only_hangul(sentence) for sentence in augmented_sentences]\n",
        "\trandom.shuffle(augmented_sentences)\n",
        "\tif num_aug >= 1:\n",
        "\t\taugmented_sentences = augmented_sentences[:num_aug]\n",
        "\telse:\n",
        "\t\tkeep_prob = num_aug / len(augmented_sentences)\n",
        "\t\taugmented_sentences = [s for s in augmented_sentences if random.uniform(0, 1) < keep_prob]\n",
        "\taugmented_sentences.append(sentence)\n",
        "\treturn augmented_sentences"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df1=df[df['digit_3']==751] \n",
        "df2=df[df['digit_3']==759] \n",
        "df3=df[df['digit_3']==752] \n",
        "df4=df[df['digit_3']==762] \n",
        "df5=df[df['digit_3']==742] \n",
        "df6=df[df['digit_3']==763]\n",
        "df7=df[df['digit_3']==761]\n",
        "df8=df[df['digit_3']==741]\n",
        "df9=df[df['digit_3']==753]\n",
        "df10=df[df['digit_3']==743]\n",
        "df11=df[df['digit_3']==764]\n",
        "\n",
        "\n",
        "df100=pd.concat([df1, df2, df3, df4, df5, df6])\n",
        "df200=pd.concat([df7, df8, df9])\n",
        "df300=pd.concat([df10, df11])\n",
        "\n",
        "df100=df100.reset_index(drop=True)\n",
        "df200=df200.reset_index(drop=True)\n",
        "df300=df300.reset_index(drop=True)"
      ],
      "metadata": {
        "id": "gF9Pg7zIGq-H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dzTWYLUwea6K"
      },
      "outputs": [],
      "source": [
        "temp_result=[]\n",
        "n=0\n",
        "id_list=[]\n",
        "digit_1_list=[]\n",
        "digit_2_list=[]\n",
        "digit_3_list=[]\n",
        "\n",
        "for sentence in df200['text']:\n",
        "  result_list=EDA(sentence, alpha_sr=0.0, alpha_ri=0.0, alpha_rs=0.5, p_rd=0.5, num_aug=2)\n",
        "  id_list=['p'+str(df200['AI_id'][n]), 'p'+str(df200['AI_id'][n]), 'p'+str(df200['AI_id'][n])]\n",
        "  digit_1_list=[df200['digit_1'][n], df200['digit_1'][n], df200['digit_1'][n]]\n",
        "  digit_2_list=[df200['digit_2'][n], df200['digit_2'][n], df200['digit_2'][n]]\n",
        "  digit_3_list=[df200['digit_3'][n], df200['digit_3'][n], df200['digit_3'][n]]\n",
        "  data_to_insert={'AI_id' : id_list, 'digit_1' : digit_1_list, 'digit_2' : digit_2_list, 'digit_3': digit_3_list, 'text' : result_list}\n",
        "  df200_to_insert = pd.DataFrame(data_to_insert)\n",
        "  df200= df200.append(df200_to_insert, ignore_index=True)\n",
        "  n=n+1"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df200['digit_3'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dBIhUWTLVf74",
        "outputId": "cc505e17-c797-4b2d-c90e-a9ff385e8228"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "761    3752\n",
              "741    3132\n",
              "753    2096\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df200=df200.reset_index(drop=True)"
      ],
      "metadata": {
        "id": "tzu_KQIVAaHI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "d-6OUKAU6T_2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "temp_result=[]\n",
        "n=0\n",
        "id_list=[]\n",
        "digit_1_list=[]\n",
        "digit_2_list=[]\n",
        "digit_3_list=[]\n",
        "\n",
        "for sentence in df300['text']:\n",
        "  result_list=EDA(sentence, alpha_sr=0.0, alpha_ri=0.0, alpha_rs=0.5, p_rd=0.5, num_aug=10)\n",
        "  id_list=['p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]),'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]),'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n]), 'p'+str(df300['AI_id'][n])]\n",
        "  digit_1_list=[df300['digit_1'][n], df300['digit_1'][n], df300['digit_1'][n], df300['digit_1'][n],df300['digit_1'][n], df300['digit_1'][n], df300['digit_1'][n], df300['digit_1'][n],df300['digit_1'][n], df300['digit_1'][n], df300['digit_1'][n]]\n",
        "  digit_2_list=[df300['digit_2'][n], df300['digit_2'][n], df300['digit_2'][n], df300['digit_2'][n],df300['digit_2'][n], df300['digit_2'][n], df300['digit_2'][n], df300['digit_2'][n],df300['digit_2'][n], df300['digit_2'][n], df300['digit_2'][n]]\n",
        "  digit_3_list=[df300['digit_3'][n], df300['digit_3'][n], df300['digit_3'][n], df300['digit_3'][n],df300['digit_3'][n], df300['digit_3'][n], df300['digit_3'][n], df300['digit_3'][n],df300['digit_3'][n], df300['digit_3'][n], df300['digit_3'][n]]\n",
        "  data_to_insert={'AI_id' : id_list, 'digit_1' : digit_1_list, 'digit_2' : digit_2_list, 'digit_3': digit_3_list, 'text' : result_list}\n",
        "  df300_to_insert = pd.DataFrame(data_to_insert)\n",
        "  df300= df300.append(df300_to_insert, ignore_index=True)\n",
        "  n=n+1"
      ],
      "metadata": {
        "id": "LRamnoi3nqhv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df300['digit_3'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wk7Ck6phnqmk",
        "outputId": "ce284b1e-7d4d-475b-913b-d35de05794b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "743    2868\n",
              "764     252\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "2AfWqZ_Snqs_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I90LypFcePR4"
      },
      "source": [
        "기존 데이터 합치기"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.concat([df100, df200, df300])"
      ],
      "metadata": {
        "id": "JvWl7U5S2EQE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['digit_3'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XgrUpfoPjsy2",
        "outputId": "7dda0d85-3685-4acf-ef8a-f8d9b4c9de79"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "761    3752\n",
              "751    3712\n",
              "759    3583\n",
              "741    3132\n",
              "743    2868\n",
              "752    2605\n",
              "753    2096\n",
              "762    1948\n",
              "742    1866\n",
              "763    1482\n",
              "764     252\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xdv6VPGk_dIX"
      },
      "outputs": [],
      "source": [
        "df.to_csv('df_N_inc.csv', index=False, encoding='utf-8-sig')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DvplvVddk7kT"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "N분류 증분의 사본",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}