Natural Language Processing 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ac520d9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init('/home/danielf/spark-3.3.1-bin-hadoop3')\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.ml.feature import Tokenizer, RegexTokenizer\n",
    "from pyspark.sql.functions import col, udf\n",
    "from pyspark.sql.types import IntegerType\n",
    "from pyspark.ml.feature import StopWordsRemover\n",
    "from pyspark.ml.feature import NGram\n",
    "from pyspark.ml.feature import HashingTF, IDF\n",
    "from pyspark.ml.feature import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "551936fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName('npl2').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cd42ae4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sen_df = spark.createDataFrame([\n",
    "    (0.0, 'Hi i heard about spark'),\n",
    "    (0.0, 'I wish java cloud use case classes'),\n",
    "    (1.0, 'Logistic regression models are neat')\n",
    "], ['label', 'sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "21f1d3b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(inputCol='sentence', outputCol='words')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5004235e",
   "metadata": {},
   "outputs": [],
   "source": [
    "words_data = tokenizer.transform(sen_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "63369725",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-----------------------------------+------------------------------------------+\n",
      "|label|sentence                           |words                                     |\n",
      "+-----+-----------------------------------+------------------------------------------+\n",
      "|0.0  |Hi i heard about spark             |[hi, i, heard, about, spark]              |\n",
      "|0.0  |I wish java cloud use case classes |[i, wish, java, cloud, use, case, classes]|\n",
      "|1.0  |Logistic regression models are neat|[logistic, regression, models, are, neat] |\n",
      "+-----+-----------------------------------+------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "words_data.show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d5f04fca",
   "metadata": {},
   "outputs": [],
   "source": [
    "hashing_ft = HashingTF(inputCol='words', outputCol='raw_features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7a32009b",
   "metadata": {},
   "outputs": [],
   "source": [
    "featurized_data = hashing_ft.transform(words_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b85d9f73",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+--------------------+--------------------+\n",
      "|label|            sentence|               words|        raw_features|\n",
      "+-----+--------------------+--------------------+--------------------+\n",
      "|  0.0|Hi i heard about ...|[hi, i, heard, ab...|(262144,[18700,19...|\n",
      "|  0.0|I wish java cloud...|[i, wish, java, c...|(262144,[19036,20...|\n",
      "|  1.0|Logistic regressi...|[logistic, regres...|(262144,[46243,58...|\n",
      "+-----+--------------------+--------------------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "featurized_data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7bc2e81b",
   "metadata": {},
   "outputs": [],
   "source": [
    "idf = IDF(inputCol='raw_features', outputCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5f5c510b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "idf_model = idf.fit(featurized_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "529d274f",
   "metadata": {},
   "outputs": [],
   "source": [
    "rescaled_data = idf_model.transform(featurized_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "aae43d10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/03/06 09:55:44 WARN DAGScheduler: Broadcasting large task binary with size 4.0 MiB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "[Stage 9:>                                                          (0 + 1) / 1]\r",
      "\r",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23/03/06 09:55:45 WARN DAGScheduler: Broadcasting large task binary with size 4.0 MiB\n",
      "+-----+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|label|features                                                                                                                                                                                       |\n",
      "+-----+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|0.0  |(262144,[18700,19036,33808,66273,173558],[0.6931471805599453,0.28768207245178085,0.6931471805599453,0.6931471805599453,0.6931471805599453])                                                    |\n",
      "|0.0  |(262144,[19036,20719,55551,98717,109547,137955,192310],[0.28768207245178085,0.6931471805599453,0.6931471805599453,0.6931471805599453,0.6931471805599453,0.6931471805599453,0.6931471805599453])|\n",
      "|1.0  |(262144,[46243,58267,91006,160975,190884],[0.6931471805599453,0.6931471805599453,0.6931471805599453,0.6931471805599453,0.6931471805599453])                                                    |\n",
      "+-----+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "rescaled_data.select(['label', 'features']).show(truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0aa442f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------+\n",
      "| id|          words|\n",
      "+---+---------------+\n",
      "|  0|      [a, b, c]|\n",
      "|  1|[a, b, b, c, a]|\n",
      "+---+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = spark.createDataFrame([\n",
    "    (0, \"a b c\".split(\" \")),\n",
    "    (1, \"a b b c a\".split(\" \"))\n",
    "], ['id', 'words'])\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "19ea9717",
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(inputCol='words', outputCol='features', vocabSize=3, minDF=2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "824dedb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = cv.fit(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "bbbc80d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = model.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a45a34d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+---------------+-------------------------+\n",
      "|id |words          |features                 |\n",
      "+---+---------------+-------------------------+\n",
      "|0  |[a, b, c]      |(3,[0,1,2],[1.0,1.0,1.0])|\n",
      "|1  |[a, b, b, c, a]|(3,[0,1,2],[2.0,2.0,1.0])|\n",
      "+---+---------------+-------------------------+\n",
      "\n",
      "23/03/06 10:11:15 WARN HeartbeatReceiver: Removing executor driver with no recent heartbeats: 124971 ms exceeds timeout 120000 ms\n",
      "23/03/06 10:11:15 WARN SparkContext: Killing executors is not supported by current scheduler.\n"
     ]
    }
   ],
   "source": [
    "results.show(truncate=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}