3-1. L 데이터 증분.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/KorStats/ocr_receipt/blob/main/3-1.%20L%20%EB%8D%B0%EC%9D%B4%ED%84%B0%20%EC%A6%9D%EB%B6%84.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U3wrPXQ_ORLb"
      },
      "source": [
        "## 구글드라이브에서 파일 읽어오기"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dqJbzoq_OPxK",
        "outputId": "4e42e8c1-d28d-41d7-a764-82aeecea990f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmtIZqMGOdVX",
        "outputId": "d5d65120-ae36-4b0e-ba7a-d81df4761c29"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n",
            " 산업분류자동화  'Colab Notebooks'  'My Drive'\t'실습용 자료.zip의 사본'\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')\n",
        "!ln -s /content/gdrive/My\\ Drive/ /mydrive\n",
        "!ls /mydrive"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xV_iSLl7QFgx",
        "outputId": "d7e8b9f0-ddbe-458e-a07e-100818b338af"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  /content/gdrive/My Drive/산업분류자동화/실습용 자료.zip\n",
            "1. 실습용자료.txt:  mismatching \"local\" filename (1. ьЛдьК╡ьЪйьЮРыгМ.txt),\n",
            "         continuing with \"central\" filename version\n",
            "  inflating: /content/dataset/1. 실습용자료.txt  \n"
          ]
        }
      ],
      "source": [
        "!unzip /content/gdrive/'My Drive'/산업분류자동화/'실습용 자료'.zip -d /content/dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "-lGZy4fGpXlo"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "df=pd.read_csv('/content/dataset/1. 실습용자료.txt', sep = \"|\", encoding = \"euc-kr\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "geFjTXHh5-dy"
      },
      "source": [
        "## 데이터 전처리"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "5RbL6zqMdTtY"
      },
      "outputs": [],
      "source": [
        "df=df[(df['digit_1'] =='L')] #조건에 맞는 데이터 출력\n",
        "#ts=ts[(ts['digit_1'] =='D')] #조건에 맞는 데이터 출력"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.reset_index(drop=True)\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "3ISvtyKhpKd2",
        "outputId": "0ac0d30d-6ef8-47aa-b2de-eb2d532f35d6"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            AI_id digit_1  digit_2  digit_3  text_obj         text_mthd  \\\n",
              "0      id_0000099       L       68      682     사업장에서           아파트를 위해   \n",
              "1      id_0000106       L       68      682     사무실에서             계약의대리   \n",
              "2      id_0000142       L       68      681     사무실에서              비주거용   \n",
              "3      id_0000164       L       68      682   상가사무실에서          빌딩입주자 위해   \n",
              "4      id_0000193       L       68      682     사업장에서           아파트를 위해   \n",
              "...           ...     ...      ...      ...       ...               ...   \n",
              "40135  id_0999952       L       68      682       부동산                중개   \n",
              "40136  id_0999980       L       68      682     사업장에서          고객의요청에의해   \n",
              "40137  id_0999983       L       68      682     사업장에서            예채아파트를   \n",
              "40138  id_0999991       L       68      682  부동산중개소에서  계약및중개등을 통해 수수료받음   \n",
              "40139  id_0999995       L       68      681     사무실에서               NaN   \n",
              "\n",
              "           text_deal  \n",
              "0             아파트 관리  \n",
              "1              부동산중개  \n",
              "2                 임대  \n",
              "3            건물관리서비스  \n",
              "4      주거용 아파트 관리서비스  \n",
              "...              ...  \n",
              "40135          부동산중개  \n",
              "40136          부동산중개  \n",
              "40137             관리  \n",
              "40138       부동산 거래중개  \n",
              "40139        비주거용 임대  \n",
              "\n",
              "[40140 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2bfb1e54-87c3-4490-a962-233808e8dfb1\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AI_id</th>\n",
              "      <th>digit_1</th>\n",
              "      <th>digit_2</th>\n",
              "      <th>digit_3</th>\n",
              "      <th>text_obj</th>\n",
              "      <th>text_mthd</th>\n",
              "      <th>text_deal</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_0000099</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서</td>\n",
              "      <td>아파트를 위해</td>\n",
              "      <td>아파트 관리</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_0000106</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사무실에서</td>\n",
              "      <td>계약의대리</td>\n",
              "      <td>부동산중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_0000142</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>681</td>\n",
              "      <td>사무실에서</td>\n",
              "      <td>비주거용</td>\n",
              "      <td>임대</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_0000164</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>상가사무실에서</td>\n",
              "      <td>빌딩입주자 위해</td>\n",
              "      <td>건물관리서비스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_0000193</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서</td>\n",
              "      <td>아파트를 위해</td>\n",
              "      <td>주거용 아파트 관리서비스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40135</th>\n",
              "      <td>id_0999952</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>부동산</td>\n",
              "      <td>중개</td>\n",
              "      <td>부동산중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40136</th>\n",
              "      <td>id_0999980</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서</td>\n",
              "      <td>고객의요청에의해</td>\n",
              "      <td>부동산중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40137</th>\n",
              "      <td>id_0999983</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서</td>\n",
              "      <td>예채아파트를</td>\n",
              "      <td>관리</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40138</th>\n",
              "      <td>id_0999991</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>부동산중개소에서</td>\n",
              "      <td>계약및중개등을 통해 수수료받음</td>\n",
              "      <td>부동산 거래중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40139</th>\n",
              "      <td>id_0999995</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>681</td>\n",
              "      <td>사무실에서</td>\n",
              "      <td>NaN</td>\n",
              "      <td>비주거용 임대</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>40140 rows × 7 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2bfb1e54-87c3-4490-a962-233808e8dfb1')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2bfb1e54-87c3-4490-a962-233808e8dfb1 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2bfb1e54-87c3-4490-a962-233808e8dfb1');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "fjEcvrtH59q-",
        "outputId": "644eb76d-b610-4ed9-88b6-b0c0eb109ee2"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "            AI_id digit_1  digit_2  digit_3  \\\n",
              "0      id_0000099       L       68      682   \n",
              "1      id_0000106       L       68      682   \n",
              "2      id_0000142       L       68      681   \n",
              "3      id_0000164       L       68      682   \n",
              "4      id_0000193       L       68      682   \n",
              "...           ...     ...      ...      ...   \n",
              "40135  id_0999952       L       68      682   \n",
              "40136  id_0999980       L       68      682   \n",
              "40137  id_0999983       L       68      682   \n",
              "40138  id_0999991       L       68      682   \n",
              "40139  id_0999995       L       68      681   \n",
              "\n",
              "                                     text  \n",
              "0                    사업장에서 아파트를 위해 아파트 관리  \n",
              "1                       사무실에서 계약의대리 부동산중개  \n",
              "2                           사무실에서 비주거용 임대  \n",
              "3                상가사무실에서 빌딩입주자 위해 건물관리서비스  \n",
              "4             사업장에서 아파트를 위해 주거용 아파트 관리서비스  \n",
              "...                                   ...  \n",
              "40135                        부동산 중개 부동산중개  \n",
              "40136                사업장에서 고객의요청에의해 부동산중개  \n",
              "40137                     사업장에서 예채아파트를 관리  \n",
              "40138  부동산중개소에서 계약및중개등을 통해 수수료받음 부동산 거래중개  \n",
              "40139                      사무실에서  비주거용 임대  \n",
              "\n",
              "[40140 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4e34f65a-afe5-44e2-9e1a-ce79071c7f5a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AI_id</th>\n",
              "      <th>digit_1</th>\n",
              "      <th>digit_2</th>\n",
              "      <th>digit_3</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_0000099</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서 아파트를 위해 아파트 관리</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_0000106</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사무실에서 계약의대리 부동산중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_0000142</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>681</td>\n",
              "      <td>사무실에서 비주거용 임대</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_0000164</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>상가사무실에서 빌딩입주자 위해 건물관리서비스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_0000193</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서 아파트를 위해 주거용 아파트 관리서비스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40135</th>\n",
              "      <td>id_0999952</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>부동산 중개 부동산중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40136</th>\n",
              "      <td>id_0999980</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서 고객의요청에의해 부동산중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40137</th>\n",
              "      <td>id_0999983</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>사업장에서 예채아파트를 관리</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40138</th>\n",
              "      <td>id_0999991</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>682</td>\n",
              "      <td>부동산중개소에서 계약및중개등을 통해 수수료받음 부동산 거래중개</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40139</th>\n",
              "      <td>id_0999995</td>\n",
              "      <td>L</td>\n",
              "      <td>68</td>\n",
              "      <td>681</td>\n",
              "      <td>사무실에서  비주거용 임대</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>40140 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4e34f65a-afe5-44e2-9e1a-ce79071c7f5a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4e34f65a-afe5-44e2-9e1a-ce79071c7f5a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4e34f65a-afe5-44e2-9e1a-ce79071c7f5a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "df['text']=df['text_obj'].map(str)+\" \"+df['text_mthd'].map(str)+\" \"+df['text_deal'].map(str)\n",
        "i=[]\n",
        "for sentence in df['text']:\n",
        "    a=sentence.replace('nan', '')\n",
        "    i.append(a)\n",
        "df['text']=pd.DataFrame(i)\n",
        "df=df.drop(['text_obj','text_mthd', 'text_deal'], axis=1) # 기존에 있던 세개의 text변수 삭제\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FpA7i6nJ9wgV",
        "outputId": "9783d801-7f4f-4fe5-8089-ac5ade221e25"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "682    33212\n",
              "681     6928\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "df['digit_3'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yzAnY0FvePIR"
      },
      "source": [
        "## **# 데이터 증분 및 맞춤법 교정**"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 맞춤법 검사"
      ],
      "metadata": {
        "id": "av1898E8yEwM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/ssut/py-hanspell"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U0g6lgW8yFAt",
        "outputId": "3c438415-7c38-4df0-952a-698338173a44"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/ssut/py-hanspell\n",
            "  Cloning https://github.com/ssut/py-hanspell to /tmp/pip-req-build-xfqixn2s\n",
            "  Running command git clone -q https://github.com/ssut/py-hanspell /tmp/pip-req-build-xfqixn2s\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from py-hanspell==1.1) (2.23.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->py-hanspell==1.1) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->py-hanspell==1.1) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->py-hanspell==1.1) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->py-hanspell==1.1) (2.10)\n",
            "Building wheels for collected packages: py-hanspell\n",
            "  Building wheel for py-hanspell (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for py-hanspell: filename=py_hanspell-1.1-py3-none-any.whl size=4868 sha256=c3e879fd7bf6e015b4f651bd344df165b162099505e4e2a1cb8957da6851a025\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-u41ssfra/wheels/31/c2/c7/ddbbd9a84ddb0a67c2e52e3324d4b7b47dcf9581af9cd784de\n",
            "Successfully built py-hanspell\n",
            "Installing collected packages: py-hanspell\n",
            "Successfully installed py-hanspell-1.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from hanspell import spell_checker\n",
        "temp_result=[]\n",
        "for sentence in df['text']:\n",
        "  result=spell_checker.check(sentence)\n",
        "  temp_result.append(result.checked)\n",
        "\n",
        "df['text']=pd.DataFrame(temp_result)  "
      ],
      "metadata": {
        "id": "8IykblSnyRsV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GV4cXBgUea3j"
      },
      "outputs": [],
      "source": [
        "# 데이터 증분 코드\n",
        "import random\n",
        "import pickle\n",
        "import re\n",
        "def get_only_hangul(line):\n",
        "\tparseText= re.compile('/ ^[ㄱ-ㅎㅏ-ㅣ가-힣]*$/').sub('',line)\n",
        "\treturn parseText\n",
        "\n",
        "def synonym_replacement(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\trandom_word_list = list(set([word for word in words]))\n",
        "\trandom.shuffle(random_word_list)\n",
        "\tnum_replaced = 0\n",
        "\tfor random_word in random_word_list:\n",
        "\t\tsynonyms = get_synonyms(random_word)\n",
        "\t\tif len(synonyms) >= 1:\n",
        "\t\t\tsynonym = random.choice(list(synonyms))\n",
        "\t\t\tnew_words = [synonym if word == random_word else word for word in new_words]\n",
        "\t\t\tnum_replaced += 1\n",
        "\t\tif num_replaced >= n:\n",
        "\t\t\tbreak\n",
        "\tif len(new_words) != 0:\n",
        "\t\tsentence = ' '.join(new_words)\n",
        "\t\tnew_words = sentence.split(\" \")\n",
        "\telse:\n",
        "\t\tnew_words = \"\"\n",
        "\treturn new_words\n",
        "\n",
        "def get_synonyms(word):\n",
        "\tsynomyms = []\n",
        "\ttry:\n",
        "\t\tfor syn in wordnet[word]:\n",
        "\t\t\tfor s in syn:\n",
        "\t\t\t\tsynomyms.append(s)\n",
        "\texcept:\n",
        "\t\tpass\n",
        "\treturn synomyms\n",
        "\n",
        "def random_deletion(words, p):\n",
        "\tif len(words) == 1:\n",
        "\t\treturn words\n",
        "\n",
        "\tnew_words = []\n",
        "\tfor word in words:\n",
        "\t\tr = random.uniform(0, 1)\n",
        "\t\tif r > p:\n",
        "\t\t\tnew_words.append(word)\n",
        "\tif len(new_words) == 0:\n",
        "\t\trand_int = random.randint(0, len(words)-1)\n",
        "\t\treturn [words[rand_int]]\n",
        "\treturn new_words\n",
        "\n",
        "########################################################################\n",
        "# Random swap\n",
        "# Randomly swap two words in the sentence n times\n",
        "########################################################################\n",
        "def random_swap(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\tfor _ in range(n):\n",
        "\t\tnew_words = swap_word(new_words)\n",
        "\treturn new_words\n",
        "\n",
        "def swap_word(new_words):\n",
        "\trandom_idx_1 = random.randint(0, len(new_words)-1)\n",
        "\trandom_idx_2 = random_idx_1\n",
        "\tcounter = 0\n",
        "\twhile random_idx_2 == random_idx_1:\n",
        "\t\trandom_idx_2 = random.randint(0, len(new_words)-1)\n",
        "\t\tcounter += 1\n",
        "\t\tif counter > 3:\n",
        "\t\t\treturn new_words\n",
        "\tnew_words[random_idx_1], new_words[random_idx_2] = new_words[random_idx_2], new_words[random_idx_1]\n",
        "\treturn new_words\n",
        "\n",
        "########################################################################\n",
        "# Random insertion\n",
        "# Randomly insert n words into the sentence\n",
        "########################################################################\n",
        "def random_insertion(words, n):\n",
        "\tnew_words = words.copy()\n",
        "\tfor _ in range(n):\n",
        "\t\tadd_word(new_words)\t\n",
        "\treturn new_words\n",
        "\n",
        "def add_word(new_words):\n",
        "\tsynonyms = []\n",
        "\tcounter = 0\n",
        "\twhile len(synonyms) < 1:\n",
        "\t\tif len(new_words) >= 1:\n",
        "\t\t\trandom_word = new_words[random.randint(0, len(new_words)-1)]\n",
        "\t\t\tsynonyms = get_synonyms(random_word)\n",
        "\t\t\tcounter += 1\n",
        "\t\telse:\n",
        "\t\t\trandom_word = \"\"\n",
        "\t\tif counter >= 10:\n",
        "\t\t\treturn\n",
        "\t\t\n",
        "\trandom_synonym = synonyms[0]\n",
        "\trandom_idx = random.randint(0, len(new_words)-1)\n",
        "\tnew_words.insert(random_idx, random_synonym)\n",
        "\n",
        "def EDA(sentence, alpha_sr=0.1, alpha_ri=0.1, alpha_rs=0.1, p_rd=0.1, num_aug=9):\n",
        "\tsentence = get_only_hangul(sentence)\n",
        "\twords = sentence.split(' ')\n",
        "\twords = [word for word in words if word is not \"\"]\n",
        "\tnum_words = len(words)\n",
        "\taugmented_sentences = []\n",
        "\tnum_new_per_technique = int(num_aug/4) + 1\n",
        "\tn_sr = max(1, int(alpha_sr*num_words))\n",
        "\tn_ri = max(1, int(alpha_ri*num_words))\n",
        "\tn_rs = max(1, int(alpha_rs*num_words))\n",
        "\t# sr\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = synonym_replacement(words, n_sr)\n",
        "\t\taugmented_sentences.append(' '.join(a_words))\n",
        "\t# ri\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_insertion(words, n_ri)\n",
        "\t\taugmented_sentences.append(' '.join(a_words))\n",
        "\t# rs\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_swap(words, n_rs)\n",
        "\t\taugmented_sentences.append(\" \".join(a_words))\n",
        "\t# rd\n",
        "\tfor _ in range(num_new_per_technique):\n",
        "\t\ta_words = random_deletion(words, p_rd)\n",
        "\t\taugmented_sentences.append(\" \".join(a_words))\n",
        "\taugmented_sentences = [get_only_hangul(sentence) for sentence in augmented_sentences]\n",
        "\trandom.shuffle(augmented_sentences)\n",
        "\tif num_aug >= 1:\n",
        "\t\taugmented_sentences = augmented_sentences[:num_aug]\n",
        "\telse:\n",
        "\t\tkeep_prob = num_aug / len(augmented_sentences)\n",
        "\t\taugmented_sentences = [s for s in augmented_sentences if random.uniform(0, 1) < keep_prob]\n",
        "\taugmented_sentences.append(sentence)\n",
        "\treturn augmented_sentences"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df1=df[df['digit_3']==681] #4개 증분 필요\n",
        "df2=df[df['digit_3']==682] "
      ],
      "metadata": {
        "id": "gF9Pg7zIGq-H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df1=df1.reset_index(drop=True)"
      ],
      "metadata": {
        "id": "xac7pM-5IEDM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dzTWYLUwea6K"
      },
      "outputs": [],
      "source": [
        "temp_result=[]\n",
        "n=0\n",
        "id_list=[]\n",
        "digit_1_list=[]\n",
        "digit_2_list=[]\n",
        "digit_3_list=[]\n",
        "\n",
        "for sentence in df1['text']:\n",
        "  result_list=EDA(sentence, alpha_sr=0.0, alpha_ri=0.0, alpha_rs=0.5, p_rd=0.5, num_aug=4)\n",
        "  id_list=['p'+str(df1['AI_id'][n]), 'p'+str(df1['AI_id'][n]), 'p'+str(df1['AI_id'][n]), 'p'+str(df1['AI_id'][n]),'p'+str(df1['AI_id'][n])]\n",
        "  digit_1_list=[df1['digit_1'][n], df1['digit_1'][n], df1['digit_1'][n], df1['digit_1'][n], df1['digit_1'][n]]\n",
        "  digit_2_list=[df1['digit_2'][n], df1['digit_2'][n], df1['digit_2'][n], df1['digit_2'][n], df1['digit_2'][n]]\n",
        "  digit_3_list=[df1['digit_3'][n], df1['digit_3'][n], df1['digit_3'][n], df1['digit_3'][n], df1['digit_3'][n]]\n",
        "  data_to_insert={'AI_id' : id_list, 'digit_1' : digit_1_list, 'digit_2' : digit_2_list, 'digit_3': digit_3_list, 'text' : result_list}\n",
        "  df1_to_insert = pd.DataFrame(data_to_insert)\n",
        "  df1= df1.append(df1_to_insert, ignore_index=True)\n",
        "  n=n+1"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df1 #df1 완성함"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "0jTKsoRxJvlo",
        "outputId": "ebaf9b17-9cf6-42bf-f1e8-60911e2e8632"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           AI_id digit_1  digit_2  digit_3  \\\n",
              "0     id_0009566       D       35      352   \n",
              "1     id_0010075       D       35      352   \n",
              "2     id_0018122       D       35      352   \n",
              "3     id_0027650       D       35      352   \n",
              "4     id_0057362       D       35      352   \n",
              "..           ...     ...      ...      ...   \n",
              "665  pid_0997719       D       35      352   \n",
              "666  pid_0997719       D       35      352   \n",
              "667  pid_0997719       D       35      352   \n",
              "668  pid_0997719       D       35      352   \n",
              "669  pid_0997719       D       35      352   \n",
              "\n",
              "                                     text  \n",
              "0    사업장에서 음식물 쓰레기와 폐수 혐기성 소화를 이용하여 가스 생산  \n",
              "1          가스 배관 공급업 사업체 및 가정에 가스 공급 도시가스  \n",
              "2             가스 배관 라인을 통해 가스 제조 및 배관 공급업  \n",
              "3     LNG(가스) 가스압력 조적(안전하게) 소비자에게 도시가스 공급  \n",
              "4                  가스 분해, 분리, 화락 처리 가스 공급  \n",
              "..                                    ...  \n",
              "665                     사업장에서 도시가스배관 도시가스  \n",
              "666                              사업장에서 공급  \n",
              "667               공급 도시가스배관 도시가스 사업장에서 공급  \n",
              "668               사업장에서 공급 도시가스 공급 도시가스배관  \n",
              "669               사업장에서 도시가스배관 공급 도시가스 공급  \n",
              "\n",
              "[670 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-5e637618-c6c5-4ddb-ae39-bc7369d4a830\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AI_id</th>\n",
              "      <th>digit_1</th>\n",
              "      <th>digit_2</th>\n",
              "      <th>digit_3</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_0009566</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>사업장에서 음식물 쓰레기와 폐수 혐기성 소화를 이용하여 가스 생산</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_0010075</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>가스 배관 공급업 사업체 및 가정에 가스 공급 도시가스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_0018122</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>가스 배관 라인을 통해 가스 제조 및 배관 공급업</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_0027650</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>LNG(가스) 가스압력 조적(안전하게) 소비자에게 도시가스 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_0057362</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>가스 분해, 분리, 화락 처리 가스 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>665</th>\n",
              "      <td>pid_0997719</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>사업장에서 도시가스배관 도시가스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>666</th>\n",
              "      <td>pid_0997719</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>사업장에서 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>667</th>\n",
              "      <td>pid_0997719</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>공급 도시가스배관 도시가스 사업장에서 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>668</th>\n",
              "      <td>pid_0997719</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>사업장에서 공급 도시가스 공급 도시가스배관</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>669</th>\n",
              "      <td>pid_0997719</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>사업장에서 도시가스배관 공급 도시가스 공급</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>670 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5e637618-c6c5-4ddb-ae39-bc7369d4a830')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-5e637618-c6c5-4ddb-ae39-bc7369d4a830 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-5e637618-c6c5-4ddb-ae39-bc7369d4a830');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2=df[df['digit_3']==353] #21개 증분 필요"
      ],
      "metadata": {
        "id": "QZ8d6t1WKOXO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df2=df2.reset_index(drop=True)\n",
        "temp_result=[]\n",
        "n=0\n",
        "id_list=[]\n",
        "digit_1_list=[]\n",
        "digit_2_list=[]\n",
        "digit_3_list=[]\n",
        "\n",
        "for sentence in df2['text']:\n",
        "  result_list=EDA(sentence, alpha_sr=0.0, alpha_ri=0.0, alpha_rs=0.5, p_rd=0.5, num_aug=20)\n",
        "  id_list=['p'+str(df2['AI_id'][n]), 'p'+str(df2['AI_id'][n]), 'p'+str(df2['AI_id'][n]), 'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]), 'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n]),'p'+str(df2['AI_id'][n])]\n",
        "  digit_1_list=[df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n], df2['digit_1'][n]]\n",
        "  digit_2_list=[df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n], df2['digit_2'][n]]\n",
        "  digit_3_list=[df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n], df2['digit_3'][n]]\n",
        "  data_to_insert={'AI_id' : id_list, 'digit_1' : digit_1_list, 'digit_2' : digit_2_list, 'digit_3': digit_3_list, 'text' : result_list}\n",
        "  df2_to_insert = pd.DataFrame(data_to_insert)\n",
        "  df2= df2.append(df2_to_insert, ignore_index=True)\n",
        "  n=n+1"
      ],
      "metadata": {
        "id": "-87HSzBXi-sU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df2['digit_3'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-tKlkcQoKOZy",
        "outputId": "26dbe46c-5a40-48a4-e62d-a1ca0ab94c72"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "353    638\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I90LypFcePR4"
      },
      "source": [
        "기존 데이터 합치기"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=pd.concat([df1, df2, df3])"
      ],
      "metadata": {
        "id": "JvWl7U5S2EQE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['digit_3'].value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XgrUpfoPjsy2",
        "outputId": "c7e7d473-03e8-4105-c2d5-03aeacfee2ed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "352    670\n",
              "351    660\n",
              "353    638\n",
              "Name: digit_3, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "xGcRc2f3lyN-",
        "outputId": "aea5ceff-1869-4aaa-dcd6-9ef4828baa69"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          AI_id digit_1  digit_2  digit_3  \\\n",
              "0    id_0009566       D       35      352   \n",
              "1    id_0010075       D       35      352   \n",
              "2    id_0018122       D       35      352   \n",
              "3    id_0027650       D       35      352   \n",
              "4    id_0057362       D       35      352   \n",
              "..          ...     ...      ...      ...   \n",
              "749  id_0989325       D       35      351   \n",
              "750  id_0989511       D       35      351   \n",
              "751  id_0992466       D       35      351   \n",
              "752  id_0993939       D       35      351   \n",
              "753  id_0994334       D       35      351   \n",
              "\n",
              "                                     text  \n",
              "0    사업장에서 음식물 쓰레기와 폐수 혐기성 소화를 이용하여 가스 생산  \n",
              "1          가스 배관 공급업 사업체 및 가정에 가스 공급 도시가스  \n",
              "2             가스 배관 라인을 통해 가스 제조 및 배관 공급업  \n",
              "3     LNG(가스) 가스압력 조적(안전하게) 소비자에게 도시가스 공급  \n",
              "4                  가스 분해, 분리, 화락 처리 가스 공급  \n",
              "..                                    ...  \n",
              "749                   태양광을 이용하여 태양열 전기 생산  \n",
              "750                    사업장에서 의뢰에 의해 전기 공급  \n",
              "751                    태양광 이용 태양광발전 전기 생산  \n",
              "752                태양광 이용 태양광발전 태양광 전기 생산  \n",
              "753                    태양광 이용 태양광발전 전기 생산  \n",
              "\n",
              "[1968 rows x 5 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-1cb5d9fa-6077-4de2-b72c-846fad1347df\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AI_id</th>\n",
              "      <th>digit_1</th>\n",
              "      <th>digit_2</th>\n",
              "      <th>digit_3</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>id_0009566</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>사업장에서 음식물 쓰레기와 폐수 혐기성 소화를 이용하여 가스 생산</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>id_0010075</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>가스 배관 공급업 사업체 및 가정에 가스 공급 도시가스</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>id_0018122</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>가스 배관 라인을 통해 가스 제조 및 배관 공급업</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>id_0027650</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>LNG(가스) 가스압력 조적(안전하게) 소비자에게 도시가스 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>id_0057362</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>352</td>\n",
              "      <td>가스 분해, 분리, 화락 처리 가스 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>749</th>\n",
              "      <td>id_0989325</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>351</td>\n",
              "      <td>태양광을 이용하여 태양열 전기 생산</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>750</th>\n",
              "      <td>id_0989511</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>351</td>\n",
              "      <td>사업장에서 의뢰에 의해 전기 공급</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>751</th>\n",
              "      <td>id_0992466</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>351</td>\n",
              "      <td>태양광 이용 태양광발전 전기 생산</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>752</th>\n",
              "      <td>id_0993939</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>351</td>\n",
              "      <td>태양광 이용 태양광발전 태양광 전기 생산</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>753</th>\n",
              "      <td>id_0994334</td>\n",
              "      <td>D</td>\n",
              "      <td>35</td>\n",
              "      <td>351</td>\n",
              "      <td>태양광 이용 태양광발전 전기 생산</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1968 rows × 5 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1cb5d9fa-6077-4de2-b72c-846fad1347df')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-1cb5d9fa-6077-4de2-b72c-846fad1347df button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-1cb5d9fa-6077-4de2-b72c-846fad1347df');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xdv6VPGk_dIX"
      },
      "outputs": [],
      "source": [
        "df.to_csv('df_D_inc.csv', index=False, encoding='utf-8-sig')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DvplvVddk7kT"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "연습파일_220320의 사본",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}