read.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import bigrams, FreqDist\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18565270\n"
     ]
    }
   ],
   "source": [
    "fileRead=open('train.txt', encoding='utf-8')\n",
    "lines=fileRead.readlines()\n",
    "longString=' '.join(lines)\n",
    "longStringSplited=re.sub(r\"\\n\", ' ', longString)\n",
    "longStringSplited=longStringSplited.split('.')\n",
    "print(len(longStringSplited))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "arabic_letters = [\n",
    "    'ا', 'ب', 'ت', 'ث', 'ج', 'ح', 'خ', 'د', 'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط', 'ظ', 'ع', 'غ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه', 'و', 'ي','ى','ة'\n",
    "]\n",
    "special_characters = [\n",
    "    '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+',\n",
    "    ',', '-', '.', '/', ':', ';', '<', '=', '>', '?', '@',\n",
    "    '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~'\n",
    "]\n",
    "digits = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
    "diacritics_happy_face = [\n",
    "    \"\\u064B\",  # Fatha\n",
    "    \"\\u064C\",  # Damma\n",
    "    \"\\u064D\",  # Kasra\n",
    "    \"\\u064E\",  # Fathatan\n",
    "    \"\\u064F\",  # Dammah\n",
    "    \"\\u0650\",  # Kasrah\n",
    "    \"\\u0651\",  # Shadda\n",
    "    \"\\u0652\",  # Sukūn\n",
    "    \"\\u0653\",  # Maddah\n",
    "    \"\\u0654\",  # Hamza Above\n",
    "    \"\\u0655\",  # Hamza Below\n",
    "    \"\\u0656\",  # Alif Khanjarah\n",
    "    \"\\u0657\",  # Alif Maddah\n",
    "    \"\\u0658\",  # Hamza on Hamza Above\n",
    "    \"\\u0659\",  # Hamza on Hamza Below\n",
    "    \"\\u065A\",  # Waw with Hamza Above\n",
    "    \"\\u065B\",  # Waw with Hamza Below\n",
    "    \"\\u065C\",  # Ya with Hamza Above\n",
    "    \"\\u065D\",  # Ya with Hamza Below\n",
    "    \"\\u065E\",  # Waw with Two Dots Above\n",
    "    \"\\u065F\",  # Waw with Hamzah Above and Two Dots Above\n",
    "    \"\\u0670\",  # Alef Wasla\n",
    "    \"\\u06D6\",  # Dotless Kasra Above\n",
    "    \"\\u06D7\",  # Dotless Kasra Below\n",
    "    \"\\u06D8\",  # Dotless Kasra Above and Three Dots Above\n",
    "    \"\\u06D9\",  # Ring Above\n",
    "    \"\\u06DA\",  # Double Ring Above\n",
    "    \"\\u06DB\",  # Ring Below\n",
    "    \"\\u06DC\",  # Double Ring Below\n",
    "    \"\\u06DD\",  # Tah with Three Dots Above\n",
    "    \"\\u06DE\",  # Fa with Three Dots Above\n",
    "    \"\\u06DF\",  # Ain with Three Dots Above\n",
    "    \"\\u06E0\",  # Ghain with Three Dots Above\n",
    "    \"\\u06E1\",  # Ha with Two Dots Above\n",
    "    \"\\u06E2\",  # Kha with Two Dots Above\n",
    "    \"\\u06E3\",  # Sa with Two Dots Above\n",
    "    \"\\u06E4\",  # Dad with Two Dots Above\n",
    "    \"\\u06E5\",  # Ta with Two Dots Above\n",
    "    \"\\u06E6\",  # Za with Two Dots Above\n",
    "    \"\\u06E7\",  # Ain with Two Dots Above\n",
    "    \"\\u06E8\",  # Ghain with Two Dots Above\n",
    "    \"\\u06E9\",  # Fa with Three Dots Below\n",
    "    \"\\u06EA\",  # Ain with Three Dots Below\n",
    "    \"\\u06EB\",  # Ghain with Three Dots Below\n",
    "    \"\\u06EC\",  # Ya with Three Dots Above\n",
    "]\n",
    "unicodeRem=['\\u200f']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'َ', 'ٌ', 'ٍ', 'ِ', 'ّ', 'ُ', 'ْ', 'ً'}\n"
     ]
    }
   ],
   "source": [
    "# arabic_diacritics = [\n",
    "#     '\\u064E',  # Fatha\n",
    "#     '\\u064F',  # Damma\n",
    "#     '\\u0650',  # Kasra\n",
    "#     '\\u0651',  # Sukun\n",
    "#     '\\u0652',  # Shadda\n",
    "#     '\\u0653',  # Maddah\n",
    "#     '\\u0654',  # Hamza Above\n",
    "#     '\\u0655',  # Hamza Below\n",
    "#     '\\u065B',  # Small High Seen\n",
    "#     '\\u065C',  # Small Low Seen\n",
    "#     '\\u065D',  # Small High Meem\n",
    "#     '\\u065E',  # Small Low Meem\n",
    "#     '\\u065F',  # Small Low Waw\n",
    "# ]\n",
    "# print(arabic_diacritics)\n",
    "\n",
    "s = set()\n",
    "s.update(longStringSplited)\n",
    "# rem = set()\n",
    "# rem.update(arabic_letters)\n",
    "# rem.update(special_characters)\n",
    "# rem.update(digits)\n",
    "# rem.update(unicodeRem)\n",
    "newSet=s.intersection(diacritics_happy_face)\n",
    "\n",
    "\n",
    "print(newSet)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}