Data Preparation(Step 1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f2363ad1-ec97-43ca-9750-ea0f63e0cfd1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.context import SQLContext \n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ef9cbb4-97d6-47d6-8847-90b05a472ed9",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Creating a Spark Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d9269bf-8b52-4ac1-a45d-7638acb99ea3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sc=SparkContext()\n",
    "sqlContext=SQLContext(sc)\n",
    "spark=SparkSession(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d94f30ea-77af-44a9-b439-06b8b6a1116c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "import glob"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbea5378-f8dc-475e-9926-29a67a0a0788",
   "metadata": {},
   "source": [
    "# Inserting every log file having a similar pattern with extension .gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "14722ebb-5ecd-4601-a575-a893ae6bb05d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "raw_files=glob.glob('/home/mav_27/Project/Data/nginx-feb2023/nginx/*.gz')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e03db245-81f3-4aed-93e9-72a1905de903",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230217.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230220.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230214.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230215.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230222.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230217.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230213.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230217.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230219.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230218.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230213.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230216.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230220.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230220.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230218.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230221.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230222.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230216.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230214.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230216.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230221.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230218.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230222.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230219.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230215.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/proxy.log-20230221.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230214.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230219.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/access.log-20230215.gz',\n",
       " '/home/mav_27/Project/Data/nginx-feb2023/nginx/error.log-20230213.gz']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de3e1411-7796-4deb-8a12-04f8aab66939",
   "metadata": {},
   "source": [
    "# Creating a Base Dataframe which contains all raw files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bdbd0898-8c8e-493a-9322-83d7eb05db78",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "base_df=spark.read.text(raw_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cb87a217-f8fd-466a-8649-c6de2e9780c7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- value: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "base_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1d6e9378-43c2-444f-86af-922893dd3f16",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6218"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "83544085-900a-4683-bf08-bc46047aa443",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(base_df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "1ebf495c-eb38-466e-a3f0-da546ae2b6da",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|value                                                                                                                                                                                                                                                                                                                        |\n",
      "+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "|143.244.50.172 - - [16/Feb/2023:03:28:45 +0530] \"GET /config/getuser?index=0 HTTP/1.1\" 400 248 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0\"                                                                                                                                            |\n",
      "|164.90.235.116 - - [16/Feb/2023:04:11:34 +0530] \"GET / HTTP/1.1\" 200 5952 \"http://14.139.152.12/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\"                                                                                                          |\n",
      "|66.249.69.126 - - [16/Feb/2023:04:39:52 +0530] \"GET /robots.txt HTTP/1.1\" 404 146 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"                                                                                                                                                             |\n",
      "|66.249.69.126 - - [16/Feb/2023:04:39:52 +0530] \"GET /assets/img/favicon.png HTTP/1.1\" 200 491 \"-\" \"Googlebot-Image/1.0\"                                                                                                                                                                                                      |\n",
      "|185.221.219.172 - - [16/Feb/2023:05:06:47 +0530] \"GET /.git/config HTTP/1.1\" 404 548 \"-\" \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; Trident/5.0)\"                                                                                                                                                                   |\n",
      "|66.249.69.101 - - [16/Feb/2023:05:24:52 +0530] \"GET /apim/devportal/site/public/images/logo.svg HTTP/1.1\" 304 0 \"-\" \"Googlebot-Image/1.0\"                                                                                                                                                                                    |\n",
      "|52.167.144.90 - - [16/Feb/2023:06:26:16 +0530] \"GET / HTTP/1.1\" 200 5952 \"-\" \"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36\"                                                                                             |\n",
      "|152.89.196.211 - - [16/Feb/2023:07:00:15 +0530] \"GET /vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php HTTP/1.1\" 404 548 \"http://14.139.152.12:80/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\"|\n",
      "|114.119.133.53 - - [16/Feb/2023:07:01:52 +0530] \"GET /robots.txt HTTP/1.1\" 404 146 \"-\" \"Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)\"                                                                                                                                                  |\n",
      "|198.20.69.98 - - [16/Feb/2023:07:26:00 +0530] \"GET / HTTP/1.1\" 200 5952 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36\"                                                                                                                                      |\n",
      "+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "base_df.show(10,truncate=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d66a596a-f520-48bd-b29b-bf4fbc006a69",
   "metadata": {},
   "source": [
    "# Taking a Sample data from Base Dataframe for data parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "77ab9bff-c238-4a5f-82e6-21a98ee7add8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['143.244.50.172 - - [16/Feb/2023:03:28:45 +0530] \"GET /config/getuser?index=0 HTTP/1.1\" 400 248 \"-\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:76.0) Gecko/20100101 Firefox/76.0\"',\n",
       " '164.90.235.116 - - [16/Feb/2023:04:11:34 +0530] \"GET / HTTP/1.1\" 200 5952 \"http://14.139.152.12/\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\"',\n",
       " '66.249.69.126 - - [16/Feb/2023:04:39:52 +0530] \"GET /robots.txt HTTP/1.1\" 404 146 \"-\" \"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\"',\n",
       " '66.249.69.126 - - [16/Feb/2023:04:39:52 +0530] \"GET /assets/img/favicon.png HTTP/1.1\" 200 491 \"-\" \"Googlebot-Image/1.0\"',\n",
       " '185.221.219.172 - - [16/Feb/2023:05:06:47 +0530] \"GET /.git/config HTTP/1.1\" 404 548 \"-\" \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.2; Trident/5.0)\"',\n",
       " '66.249.69.101 - - [16/Feb/2023:05:24:52 +0530] \"GET /apim/devportal/site/public/images/logo.svg HTTP/1.1\" 304 0 \"-\" \"Googlebot-Image/1.0\"',\n",
       " '52.167.144.90 - - [16/Feb/2023:06:26:16 +0530] \"GET / HTTP/1.1\" 200 5952 \"-\" \"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/103.0.5060.134 Safari/537.36\"',\n",
       " '152.89.196.211 - - [16/Feb/2023:07:00:15 +0530] \"GET /vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php HTTP/1.1\" 404 548 \"http://14.139.152.12:80/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36\"',\n",
       " '114.119.133.53 - - [16/Feb/2023:07:01:52 +0530] \"GET /robots.txt HTTP/1.1\" 404 146 \"-\" \"Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)\"',\n",
       " '198.20.69.98 - - [16/Feb/2023:07:26:00 +0530] \"GET / HTTP/1.1\" 200 5952 \"-\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36\"',\n",
       " '198.20.69.98 - - [16/Feb/2023:07:26:16 +0530] \"\" 400 0 \"-\" \"-\"',\n",
       " '198.20.69.98 - - [16/Feb/2023:07:26:20 +0530] \"\" 400 0 \"-\" \"-\"',\n",
       " '198.20.69.98 - - [16/Feb/2023:07:26:21 +0530] \"\" 400 0 \"-\" \"-\"',\n",
       " '198.20.69.98 - - [16/Feb/2023:07:26:24 +0530] \"\" 400 0 \"-\" \"-\"',\n",
       " '198.20.69.98 - - [16/Feb/2023:07:26:30 +0530] \"GET /robots.txt HTTP/1.1\" 404 146 \"-\" \"-\"']"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_logs=[items['value'] for items in base_df.take(15)]\n",
    "sample_logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "277b5380-cc16-42b2-86e4-86b7e61aedfc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "host_pattern=r'(^\\S+\\.[\\S+\\.]+\\S+)\\s'\n",
    "hosts=[re.search(host_pattern,items).group(1) \n",
    "       if re.search(host_pattern,items) \n",
    "       else 'no match' \n",
    "       for items in sample_logs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "42cbbc61-941a-451d-845e-4ed0e6c0b211",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['143.244.50.172',\n",
       " '164.90.235.116',\n",
       " '66.249.69.126',\n",
       " '66.249.69.126',\n",
       " '185.221.219.172',\n",
       " '66.249.69.101',\n",
       " '52.167.144.90',\n",
       " '152.89.196.211',\n",
       " '114.119.133.53',\n",
       " '198.20.69.98',\n",
       " '198.20.69.98',\n",
       " '198.20.69.98',\n",
       " '198.20.69.98',\n",
       " '198.20.69.98',\n",
       " '198.20.69.98']"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hosts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "id": "ad367702-375d-41a7-ad17-a63b3041dbfa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023',\n",
       " '16/Feb/2023']"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_stamp_pattern=r'\\[(\\d{2}/\\w{3}/\\d{4})'\n",
    "date=[re.search(time_stamp_pattern,items).group(1)\n",
    "            for items in sample_logs]\n",
    "date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "d9aed422-e59a-4251-972a-2f7f89e775cf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[':03:28:45',\n",
       " ':04:11:34',\n",
       " ':04:39:52',\n",
       " ':04:39:52',\n",
       " ':05:06:47',\n",
       " ':05:24:52',\n",
       " ':06:26:16',\n",
       " ':07:00:15',\n",
       " ':07:01:52',\n",
       " ':07:26:00',\n",
       " ':07:26:16',\n",
       " ':07:26:20',\n",
       " ':07:26:21',\n",
       " ':07:26:24',\n",
       " ':07:26:30']"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time_pattern=r':\\d{2}:\\d{2}:\\d{2}'\n",
    "time_raw=[re.search(time_pattern,items.strip(':')).group()\n",
    "            for items in sample_logs]\n",
    "time_raw\n",
    "# time=[]\n",
    "# for i in time_raw:\n",
    "#     time.append(i.lstrip(':'))\n",
    "# time    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "id": "ccf43ae8-b438-421c-97e4-041370920ae5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "method_pattern=r'\\\"(\\S+)\\s(\\S+)\\s*(\\S*)\\\"'\n",
    "method_uri=[re.search(method_pattern,items).group()\n",
    "           for items in sample_logs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "2f054268-9eb2-40d1-ba6d-f3b2506f1b47",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\"GET /config/getuser?index=0 HTTP/1.1\"',\n",
       " '\"GET / HTTP/1.1\"',\n",
       " '\"GET /robots.txt HTTP/1.1\"',\n",
       " '\"GET /assets/img/favicon.png HTTP/1.1\"',\n",
       " '\"GET /.git/config HTTP/1.1\"',\n",
       " '\"GET /apim/devportal/site/public/images/logo.svg HTTP/1.1\"',\n",
       " '\"GET / HTTP/1.1\"',\n",
       " '\"GET /vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php HTTP/1.1\"',\n",
       " '\"GET /robots.txt HTTP/1.1\"',\n",
       " '\"GET / HTTP/1.1\"',\n",
       " '\"-\" \"-\"',\n",
       " '\"-\" \"-\"',\n",
       " '\"-\" \"-\"',\n",
       " '\"-\" \"-\"',\n",
       " '\"GET /robots.txt HTTP/1.1\"']"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "method_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "c6d1937c-1811-4962-b952-ad8d087dcad6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "status_pattern=r'\\s(\\d{3})\\s'\n",
    "status_code=[re.search(status_pattern,items).group(1)\n",
    "            for items in sample_logs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "6b9590fb-0915-43e5-a5ed-976fbddc500a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['400',\n",
       " '200',\n",
       " '404',\n",
       " '200',\n",
       " '404',\n",
       " '304',\n",
       " '200',\n",
       " '404',\n",
       " '404',\n",
       " '200',\n",
       " '400',\n",
       " '400',\n",
       " '400',\n",
       " '400',\n",
       " '404']"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "7be5959a-c837-4e24-bfa0-cc2871b01333",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "content_size_pattarn=r'\\s(\\d+) \"'\n",
    "content_size=[re.search(content_size_pattarn,items).group(1)\n",
    "              for items in sample_logs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "9e3134c8-9960-4443-9d08-ff8705774231",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['248',\n",
       " '5952',\n",
       " '146',\n",
       " '491',\n",
       " '548',\n",
       " '0',\n",
       " '5952',\n",
       " '548',\n",
       " '146',\n",
       " '5952',\n",
       " '0',\n",
       " '0',\n",
       " '0',\n",
       " '0',\n",
       " '146']"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a37da3e8-ab6c-4677-a9c3-a998fc41f0a8",
   "metadata": {},
   "source": [
    "# Creating a clean dataframe with parsed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "962ef7d7-bdcc-4e56-bdc0-2fa865814d1f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import regexp_extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "47fb56b1-f44a-41dd-99c4-92c00204acbf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "logs_df=base_df.select(regexp_extract('value',host_pattern,1).alias('Host'),\n",
    "                       regexp_extract('value',time_stamp_pattern,1).alias('Date'),\n",
    "                       regexp_extract('value',time_pattern,0).alias('Time'),\n",
    "                       regexp_extract('value',method_pattern,1).alias('Method'),\n",
    "                       regexp_extract('value',method_pattern,2).alias('Endpoint'),\n",
    "                       regexp_extract('value',method_pattern,3).alias('Protocol'),\n",
    "                       regexp_extract('value',status_pattern,1).alias(\"Status Code\"),\n",
    "                       regexp_extract('value',content_size_pattarn,1).cast('integer').alias('Content Size'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "45a43cba-5b16-481b-be32-e4992136995a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+-----------+---------+------+--------------------+--------+-----------+------------+\n",
      "|           Host|       Date|     Time|Method|            Endpoint|Protocol|Status Code|Content Size|\n",
      "+---------------+-----------+---------+------+--------------------+--------+-----------+------------+\n",
      "| 143.244.50.172|16/Feb/2023|:03:28:45|   GET|/config/getuser?i...|HTTP/1.1|        400|         248|\n",
      "| 164.90.235.116|16/Feb/2023|:04:11:34|   GET|                   /|HTTP/1.1|        200|        5952|\n",
      "|  66.249.69.126|16/Feb/2023|:04:39:52|   GET|         /robots.txt|HTTP/1.1|        404|         146|\n",
      "|  66.249.69.126|16/Feb/2023|:04:39:52|   GET|/assets/img/favic...|HTTP/1.1|        200|         491|\n",
      "|185.221.219.172|16/Feb/2023|:05:06:47|   GET|        /.git/config|HTTP/1.1|        404|         548|\n",
      "|  66.249.69.101|16/Feb/2023|:05:24:52|   GET|/apim/devportal/s...|HTTP/1.1|        304|           0|\n",
      "|  52.167.144.90|16/Feb/2023|:06:26:16|   GET|                   /|HTTP/1.1|        200|        5952|\n",
      "| 152.89.196.211|16/Feb/2023|:07:00:15|   GET|/vendor/phpunit/p...|HTTP/1.1|        404|         548|\n",
      "| 114.119.133.53|16/Feb/2023|:07:01:52|   GET|         /robots.txt|HTTP/1.1|        404|         146|\n",
      "|   198.20.69.98|16/Feb/2023|:07:26:00|   GET|                   /|HTTP/1.1|        200|        5952|\n",
      "+---------------+-----------+---------+------+--------------------+--------+-----------+------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "logs_df.show(10,truncate=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "f3ea64eb-0e3f-4a9b-87e2-e580e55a6d2b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6218"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "b4c6ba9f-96a9-40bf-a18a-4c9fe8a16b39",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import ipaddress"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a7ce18c-fedb-4ca6-b4a6-9a4eab4f8bdb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "17fd9b85-806f-4b29-9da9-d936788a2555",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(base_df.filter(base_df['value'].isNull()).count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "id": "adb16eed-3df3-4ac0-a34a-0294c44740ff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bad_row_df=logs_df.filter(logs_df['Host'].isNull() |\n",
    "                          logs_df['Date'].isNull() |\n",
    "                          logs_df['Time'].isNull() |\n",
    "                          logs_df['Method'].isNull() |\n",
    "                          logs_df['Endpoint'].isNull() |\n",
    "                          logs_df['Protocol'].isNull() |\n",
    "                          logs_df['Status Code'].isNull() |\n",
    "                          logs_df['Content Size'].isNull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "af9e8bfa-aead-4656-b0b3-fef76d739b48",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "482"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_row_df.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ea700b8-6186-4d17-9d05-5e5d0d2d6ecb",
   "metadata": {},
   "source": [
    "# Converting the Spark DataFrame to Pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "id": "bb5e7845-02ef-4199-9332-d3119a569675",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "logs_df=logs_df.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "329f781e-1b12-4097-ad2d-e6c7df844f8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Host             object\n",
       "Date             object\n",
       "Time             object\n",
       "Method           object\n",
       "Endpoint         object\n",
       "Protocol         object\n",
       "Status Code      object\n",
       "Content Size    float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "c30c8cbe-3152-46f2-b771-22a3344e0841",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>143.244.50.172</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>:03:28:45</td>\n",
       "      <td>GET</td>\n",
       "      <td>/config/getuser?index=0</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>400</td>\n",
       "      <td>248.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>164.90.235.116</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>:04:11:34</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.249.69.126</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>:04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/robots.txt</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>66.249.69.126</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>:04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/assets/img/favicon.png</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>491.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>185.221.219.172</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>:05:06:47</td>\n",
       "      <td>GET</td>\n",
       "      <td>/.git/config</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>548.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Host         Date       Time Method                 Endpoint  \\\n",
       "0   143.244.50.172  16/Feb/2023  :03:28:45    GET  /config/getuser?index=0   \n",
       "1   164.90.235.116  16/Feb/2023  :04:11:34    GET                        /   \n",
       "2    66.249.69.126  16/Feb/2023  :04:39:52    GET              /robots.txt   \n",
       "3    66.249.69.126  16/Feb/2023  :04:39:52    GET  /assets/img/favicon.png   \n",
       "4  185.221.219.172  16/Feb/2023  :05:06:47    GET             /.git/config   \n",
       "\n",
       "   Protocol Status Code  Content Size  \n",
       "0  HTTP/1.1         400         248.0  \n",
       "1  HTTP/1.1         200        5952.0  \n",
       "2  HTTP/1.1         404         146.0  \n",
       "3  HTTP/1.1         200         491.0  \n",
       "4  HTTP/1.1         404         548.0  "
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "e217ae33-84b3-4ac4-8178-423f9e9a84ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "time=[]\n",
    "for i in logs_df.Time:\n",
    "       time.append(i.strip(':'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "f16ba1df-f24d-433b-8e14-2eb95505fbb4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "logs_df['Time']=time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "e9c78aa3-1055-4aca-a2bd-0f4f866d97ca",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>143.244.50.172</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>03:28:45</td>\n",
       "      <td>GET</td>\n",
       "      <td>/config/getuser?index=0</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>400</td>\n",
       "      <td>248.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>164.90.235.116</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:11:34</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.249.69.126</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/robots.txt</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>66.249.69.126</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/assets/img/favicon.png</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>491.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>185.221.219.172</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>05:06:47</td>\n",
       "      <td>GET</td>\n",
       "      <td>/.git/config</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>548.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6213</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>GET</td>\n",
       "      <td>/sitemaps.xml</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6214</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>GET</td>\n",
       "      <td>/robots.txt</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6215</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>GET</td>\n",
       "      <td>/favicon.ico</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6216</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>GET</td>\n",
       "      <td>/owa/auth/logon.aspx?url=https%3a%2f%2f1%2fecp%2f</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6217</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>GET</td>\n",
       "      <td>/wp-login.php</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6218 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Host         Date      Time Method  \\\n",
       "0      143.244.50.172  16/Feb/2023  03:28:45    GET   \n",
       "1      164.90.235.116  16/Feb/2023  04:11:34    GET   \n",
       "2       66.249.69.126  16/Feb/2023  04:39:52    GET   \n",
       "3       66.249.69.126  16/Feb/2023  04:39:52    GET   \n",
       "4     185.221.219.172  16/Feb/2023  05:06:47    GET   \n",
       "...               ...          ...       ...    ...   \n",
       "6213                                            GET   \n",
       "6214                                            GET   \n",
       "6215                                            GET   \n",
       "6216                                            GET   \n",
       "6217                                            GET   \n",
       "\n",
       "                                               Endpoint  Protocol Status Code  \\\n",
       "0                               /config/getuser?index=0  HTTP/1.1         400   \n",
       "1                                                     /  HTTP/1.1         200   \n",
       "2                                           /robots.txt  HTTP/1.1         404   \n",
       "3                               /assets/img/favicon.png  HTTP/1.1         200   \n",
       "4                                          /.git/config  HTTP/1.1         404   \n",
       "...                                                 ...       ...         ...   \n",
       "6213                                      /sitemaps.xml  HTTP/1.1               \n",
       "6214                                        /robots.txt  HTTP/1.1               \n",
       "6215                                       /favicon.ico  HTTP/1.1               \n",
       "6216  /owa/auth/logon.aspx?url=https%3a%2f%2f1%2fecp%2f  HTTP/1.1               \n",
       "6217                                      /wp-login.php  HTTP/1.1               \n",
       "\n",
       "      Content Size  \n",
       "0            248.0  \n",
       "1           5952.0  \n",
       "2            146.0  \n",
       "3            491.0  \n",
       "4            548.0  \n",
       "...            ...  \n",
       "6213           NaN  \n",
       "6214           NaN  \n",
       "6215           NaN  \n",
       "6216           NaN  \n",
       "6217           NaN  \n",
       "\n",
       "[6218 rows x 8 columns]"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "ebe333dd-8551-4b4b-8f5b-b06b0b737ab2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "logs_df=logs_df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "8dd73922-983e-4dcd-ae40-ac9eb301a877",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>143.244.50.172</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>03:28:45</td>\n",
       "      <td>GET</td>\n",
       "      <td>/config/getuser?index=0</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>400</td>\n",
       "      <td>248.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>164.90.235.116</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:11:34</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>66.249.69.126</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/robots.txt</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>66.249.69.126</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/assets/img/favicon.png</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>491.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>185.221.219.172</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>05:06:47</td>\n",
       "      <td>GET</td>\n",
       "      <td>/.git/config</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>548.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5731</th>\n",
       "      <td>154.12.227.174</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:08</td>\n",
       "      <td>POST</td>\n",
       "      <td>/core/.env</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5732</th>\n",
       "      <td>154.12.227.174</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:10</td>\n",
       "      <td>GET</td>\n",
       "      <td>/core/.env</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5733</th>\n",
       "      <td>154.12.227.174</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:12</td>\n",
       "      <td>POST</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>405</td>\n",
       "      <td>150.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5734</th>\n",
       "      <td>154.12.227.174</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:13</td>\n",
       "      <td>POST</td>\n",
       "      <td>/core/.env</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5735</th>\n",
       "      <td>188.165.87.111</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>03:06:29</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5736 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Host         Date      Time Method                 Endpoint  \\\n",
       "0      143.244.50.172  16/Feb/2023  03:28:45    GET  /config/getuser?index=0   \n",
       "1      164.90.235.116  16/Feb/2023  04:11:34    GET                        /   \n",
       "2       66.249.69.126  16/Feb/2023  04:39:52    GET              /robots.txt   \n",
       "3       66.249.69.126  16/Feb/2023  04:39:52    GET  /assets/img/favicon.png   \n",
       "4     185.221.219.172  16/Feb/2023  05:06:47    GET             /.git/config   \n",
       "...               ...          ...       ...    ...                      ...   \n",
       "5731   154.12.227.174  19/Feb/2023  02:48:08   POST               /core/.env   \n",
       "5732   154.12.227.174  19/Feb/2023  02:48:10    GET               /core/.env   \n",
       "5733   154.12.227.174  19/Feb/2023  02:48:12   POST                        /   \n",
       "5734   154.12.227.174  19/Feb/2023  02:48:13   POST               /core/.env   \n",
       "5735   188.165.87.111  19/Feb/2023  03:06:29    GET                        /   \n",
       "\n",
       "      Protocol Status Code  Content Size  \n",
       "0     HTTP/1.1         400         248.0  \n",
       "1     HTTP/1.1         200        5952.0  \n",
       "2     HTTP/1.1         404         146.0  \n",
       "3     HTTP/1.1         200         491.0  \n",
       "4     HTTP/1.1         404         548.0  \n",
       "...        ...         ...           ...  \n",
       "5731  HTTP/1.1         404         146.0  \n",
       "5732  HTTP/1.1         404         146.0  \n",
       "5733  HTTP/1.1         405         150.0  \n",
       "5734  HTTP/1.1         404         146.0  \n",
       "5735  HTTP/1.1         200        5952.0  \n",
       "\n",
       "[5736 rows x 8 columns]"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "id": "68101903-a7ee-4fcb-a396-d963f48db90b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ipadd=[]\n",
    "for i in logs_df.Host:\n",
    "    ipadd.append(int(ipaddress.IPv4Address(i)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "id": "8a628105-dd79-46b2-9ba8-4099a0111462",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_185/2148316297.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df.Host=ipadd\n"
     ]
    }
   ],
   "source": [
    "logs_df.Host=ipadd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4236a8b0-cbcc-45bb-a6e8-34374330a000",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "f5874956-3864-4a8e-a6e7-22776dc6ca6a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GET                                                                                                                                      3467\n",
       "-\"                                                                                                                                        789\n",
       "POST                                                                                                                                      707\n",
       "CONNECT                                                                                                                                   596\n",
       "PRI                                                                                                                                        40\n",
       "                                                                                                                                         ... \n",
       "\\x16\\x03\\x01\\x01H\\x01\\x00\\x01D\\x03\\x03\\x96\\xD0uu\\x19\\x14\\xFC-                                                                               1\n",
       "A\\x00\\x00\\x00\\x03fH\\xBBd~\\x8E\\xFC\\x94g\\xD2\\xDB\\xFC\\xEE\\x8D\\xFF\\x98                                                                          1\n",
       "\\x16\\x03\\x01\\x00\\xEE\\x01\\x00\\x00\\xEA\\x03\\x03\\x1F\\xFCQ3\\xA1m{!\\xE40\\x9B\\xBD\\xCB\\xFA\\xACLo\\x81\\xA5u\\xC1\\xD1\\xC30\\x1D\\xD6\\xDBEL\\xC4`\\x07       1\n",
       "\\x16\\x03\\x01\\x00{\\x01\\x00\\x00w\\x03\\x03t*\\x1C\\x1E\\x83\\xB5\\xCE\\xC4\\xEF?\\xAF\\x8D\\xDD                                                           1\n",
       "\\x16\\x03\\x01\\x00M\\x01\\x00\\x00I\\x03\\x035&\\xB3\\xEE\\xAB=\\x95h:V\\xAD4\\x94W\\xCB\\xAA\\xBD\\x15\\x05w\\xE5\\xE7\\xC5\\x04                                 1\n",
       "Name: Method, Length: 79, dtype: int64"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.Method.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "d2aeb452-318d-419d-834a-b785b60953a7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "301    1948\n",
       "400    1658\n",
       "200    1605\n",
       "404     486\n",
       "302      12\n",
       "405      11\n",
       "206      10\n",
       "204       4\n",
       "304       1\n",
       "500       1\n",
       "Name: Status Code, dtype: int64"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df['Status Code'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "id": "bf6ab3d0-84b0-40dd-98bf-54fe49f8d81d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "a=logs_df.Endpoint.groupby(logs_df.Host).count().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "d90900bb-52a5-4214-b709-10595991fb07",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Host\n",
       "1597610172    549\n",
       "2050150420    282\n",
       "3079201056    198\n",
       "3240763672    197\n",
       "3260520401    196\n",
       "             ... \n",
       "1806363924      1\n",
       "1806363153      1\n",
       "1806363146      1\n",
       "1806362897      1\n",
       "2059537226      1\n",
       "Name: Endpoint, Length: 765, dtype: int64"
      ]
     },
     "execution_count": 234,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "id": "431673e6-0609-411b-a77f-49cab44c4eb4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "301    1948\n",
       "400    1658\n",
       "200    1605\n",
       "404     486\n",
       "302      12\n",
       "405      11\n",
       "206      10\n",
       "204       4\n",
       "304       1\n",
       "500       1\n",
       "Name: Status Code, dtype: int64"
      ]
     },
     "execution_count": 235,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df['Status Code'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cfcc15c-7ee0-40e7-8d3d-82f4b9c32575",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "1c522ae2-013a-4029-86eb-83302e2f964c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_185/1435016481.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df['No of Requests']=logs_df.groupby([\"Host\"])['Endpoint'].transform('count')\n"
     ]
    }
   ],
   "source": [
    "logs_df['No of Requests']=logs_df.groupby([\"Host\"])['Endpoint'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "53ea31e9-4216-4dd1-ac1f-3c0c54897ca3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "      <th>No of Requests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2415145644</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>03:28:45</td>\n",
       "      <td>GET</td>\n",
       "      <td>/config/getuser?index=0</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>400</td>\n",
       "      <td>248.0</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2757421940</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:11:34</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/robots.txt</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/assets/img/favicon.png</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>491.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3118324652</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>05:06:47</td>\n",
       "      <td>GET</td>\n",
       "      <td>/.git/config</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>548.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1123632485</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>05:24:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/apim/devportal/site/public/images/logo.svg</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>304</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>883396698</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>06:26:16</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2556019923</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>07:00:15</td>\n",
       "      <td>GET</td>\n",
       "      <td>/vendor/phpunit/phpunit/src/Util/PHP/eval-stdi...</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>548.0</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1920435509</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>07:01:52</td>\n",
       "      <td>GET</td>\n",
       "      <td>/robots.txt</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>404</td>\n",
       "      <td>146.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3323217250</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>07:26:00</td>\n",
       "      <td>GET</td>\n",
       "      <td>/</td>\n",
       "      <td>HTTP/1.1</td>\n",
       "      <td>200</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Host         Date      Time Method  \\\n",
       "0  2415145644  16/Feb/2023  03:28:45    GET   \n",
       "1  2757421940  16/Feb/2023  04:11:34    GET   \n",
       "2  1123632510  16/Feb/2023  04:39:52    GET   \n",
       "3  1123632510  16/Feb/2023  04:39:52    GET   \n",
       "4  3118324652  16/Feb/2023  05:06:47    GET   \n",
       "5  1123632485  16/Feb/2023  05:24:52    GET   \n",
       "6   883396698  16/Feb/2023  06:26:16    GET   \n",
       "7  2556019923  16/Feb/2023  07:00:15    GET   \n",
       "8  1920435509  16/Feb/2023  07:01:52    GET   \n",
       "9  3323217250  16/Feb/2023  07:26:00    GET   \n",
       "\n",
       "                                            Endpoint  Protocol Status Code  \\\n",
       "0                            /config/getuser?index=0  HTTP/1.1         400   \n",
       "1                                                  /  HTTP/1.1         200   \n",
       "2                                        /robots.txt  HTTP/1.1         404   \n",
       "3                            /assets/img/favicon.png  HTTP/1.1         200   \n",
       "4                                       /.git/config  HTTP/1.1         404   \n",
       "5        /apim/devportal/site/public/images/logo.svg  HTTP/1.1         304   \n",
       "6                                                  /  HTTP/1.1         200   \n",
       "7  /vendor/phpunit/phpunit/src/Util/PHP/eval-stdi...  HTTP/1.1         404   \n",
       "8                                        /robots.txt  HTTP/1.1         404   \n",
       "9                                                  /  HTTP/1.1         200   \n",
       "\n",
       "   Content Size  No of Requests  \n",
       "0         248.0              62  \n",
       "1        5952.0               5  \n",
       "2         146.0              20  \n",
       "3         491.0              20  \n",
       "4         548.0               1  \n",
       "5           0.0               3  \n",
       "6        5952.0              19  \n",
       "7         548.0              84  \n",
       "8         146.0               1  \n",
       "9        5952.0              10  "
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "id": "29358135-37bb-481b-b195-5e99c7d14136",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "185421df-0d20-4da4-babb-90899607796a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.DataFrame.to_csv(logs_df,'logs_df.csv',index=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "826f1938-633b-43a3-9357-1d2c60fb8d24",
   "metadata": {},
   "source": [
    "# Label Encoding the features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "id": "83e63752-1e83-4fe2-9f66-f11f78cce8ff",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "le=LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "4481beeb-b759-4641-8fe9-51aafbb45774",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cols='Method','Status Code','Protocol','Endpoint'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "4cf39114-ffac-4f79-8ae4-1626cb602a7e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_185/1174627788.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df[i]=le.fit_transform(logs_df[i])\n",
      "/tmp/ipykernel_185/1174627788.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df[i]=le.fit_transform(logs_df[i])\n",
      "/tmp/ipykernel_185/1174627788.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df[i]=le.fit_transform(logs_df[i])\n",
      "/tmp/ipykernel_185/1174627788.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df[i]=le.fit_transform(logs_df[i])\n"
     ]
    }
   ],
   "source": [
    "for i in cols:\n",
    "    logs_df[i]=le.fit_transform(logs_df[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "3856e3ec-b0f4-4941-8280-5a9af96a13ea",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "      <th>No of Requests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2415145644</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>03:28:45</td>\n",
       "      <td>4</td>\n",
       "      <td>236</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>248.0</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2757421940</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:11:34</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>4</td>\n",
       "      <td>400</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>4</td>\n",
       "      <td>177</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3118324652</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>05:06:47</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>548.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5731</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:08</td>\n",
       "      <td>7</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5732</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:10</td>\n",
       "      <td>4</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5733</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:12</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>150.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5734</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:13</td>\n",
       "      <td>7</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5735</th>\n",
       "      <td>3164952431</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>03:06:29</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5736 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Host         Date      Time  Method  Endpoint  Protocol  \\\n",
       "0     2415145644  16/Feb/2023  03:28:45       4       236         6   \n",
       "1     2757421940  16/Feb/2023  04:11:34       4        10         6   \n",
       "2     1123632510  16/Feb/2023  04:39:52       4       400         6   \n",
       "3     1123632510  16/Feb/2023  04:39:52       4       177         6   \n",
       "4     3118324652  16/Feb/2023  05:06:47       4        40         6   \n",
       "...          ...          ...       ...     ...       ...       ...   \n",
       "5731  2584535982  19/Feb/2023  02:48:08       7       240         6   \n",
       "5732  2584535982  19/Feb/2023  02:48:10       4       240         6   \n",
       "5733  2584535982  19/Feb/2023  02:48:12       7        10         6   \n",
       "5734  2584535982  19/Feb/2023  02:48:13       7       240         6   \n",
       "5735  3164952431  19/Feb/2023  03:06:29       4        10         6   \n",
       "\n",
       "      Status Code  Content Size  No of Requests  \n",
       "0               6         248.0              62  \n",
       "1               0        5952.0               5  \n",
       "2               7         146.0              20  \n",
       "3               0         491.0              20  \n",
       "4               7         548.0               1  \n",
       "...           ...           ...             ...  \n",
       "5731            7         146.0               6  \n",
       "5732            7         146.0               6  \n",
       "5733            8         150.0               6  \n",
       "5734            7         146.0               6  \n",
       "5735            0        5952.0               1  \n",
       "\n",
       "[5736 rows x 9 columns]"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c574fba-4baa-4063-9abe-ca148ab6c228",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2f669ce-15c7-4d6f-8655-ba34bff4f6f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "d62bfdf5-1233-432f-9b98-69ae9199c732",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3    1948\n",
       "6    1658\n",
       "0    1605\n",
       "7     486\n",
       "4      12\n",
       "8      11\n",
       "2      10\n",
       "1       4\n",
       "5       1\n",
       "9       1\n",
       "Name: Status Code, dtype: int64"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df['Status Code'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "b7a55992-f31f-40ed-88cd-a312978828ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "id": "7e457d8e-99c4-4380-8cc4-cbd82246fa6f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "      <th>No of Requests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2415145644</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>03:28:45</td>\n",
       "      <td>4</td>\n",
       "      <td>236</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>248.0</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2757421940</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:11:34</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>4</td>\n",
       "      <td>400</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>4</td>\n",
       "      <td>177</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3118324652</td>\n",
       "      <td>16/Feb/2023</td>\n",
       "      <td>05:06:47</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>548.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5731</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:08</td>\n",
       "      <td>7</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5732</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:10</td>\n",
       "      <td>4</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5733</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:12</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>150.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5734</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>02:48:13</td>\n",
       "      <td>7</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5735</th>\n",
       "      <td>3164952431</td>\n",
       "      <td>19/Feb/2023</td>\n",
       "      <td>03:06:29</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5736 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Host         Date      Time  Method  Endpoint  Protocol  \\\n",
       "0     2415145644  16/Feb/2023  03:28:45       4       236         6   \n",
       "1     2757421940  16/Feb/2023  04:11:34       4        10         6   \n",
       "2     1123632510  16/Feb/2023  04:39:52       4       400         6   \n",
       "3     1123632510  16/Feb/2023  04:39:52       4       177         6   \n",
       "4     3118324652  16/Feb/2023  05:06:47       4        40         6   \n",
       "...          ...          ...       ...     ...       ...       ...   \n",
       "5731  2584535982  19/Feb/2023  02:48:08       7       240         6   \n",
       "5732  2584535982  19/Feb/2023  02:48:10       4       240         6   \n",
       "5733  2584535982  19/Feb/2023  02:48:12       7        10         6   \n",
       "5734  2584535982  19/Feb/2023  02:48:13       7       240         6   \n",
       "5735  3164952431  19/Feb/2023  03:06:29       4        10         6   \n",
       "\n",
       "      Status Code  Content Size  No of Requests  \n",
       "0               6         248.0              62  \n",
       "1               0        5952.0               5  \n",
       "2               7         146.0              20  \n",
       "3               0         491.0              20  \n",
       "4               7         548.0               1  \n",
       "...           ...           ...             ...  \n",
       "5731            7         146.0               6  \n",
       "5732            7         146.0               6  \n",
       "5733            8         150.0               6  \n",
       "5734            7         146.0               6  \n",
       "5735            0        5952.0               1  \n",
       "\n",
       "[5736 rows x 9 columns]"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "2ca8e31d-8099-492a-b172-6d02294f1375",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "date=pd.to_datetime(logs_df.Date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "a42a656f-7394-4fd8-aa8a-bc9dd54e126a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      2023-02-16\n",
       "1      2023-02-16\n",
       "2      2023-02-16\n",
       "3      2023-02-16\n",
       "4      2023-02-16\n",
       "          ...    \n",
       "5731   2023-02-19\n",
       "5732   2023-02-19\n",
       "5733   2023-02-19\n",
       "5734   2023-02-19\n",
       "5735   2023-02-19\n",
       "Name: Date, Length: 5736, dtype: datetime64[ns]"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "6a8f42ca-1624-45b4-aebb-c5d8e5088cda",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_185/3120451882.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  logs_df.Date=date\n"
     ]
    }
   ],
   "source": [
    "logs_df.Date=date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "d8b9d3f0-affb-4549-a0a6-356187d47763",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Host</th>\n",
       "      <th>Date</th>\n",
       "      <th>Time</th>\n",
       "      <th>Method</th>\n",
       "      <th>Endpoint</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Status Code</th>\n",
       "      <th>Content Size</th>\n",
       "      <th>No of Requests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2415145644</td>\n",
       "      <td>2023-02-16</td>\n",
       "      <td>03:28:45</td>\n",
       "      <td>4</td>\n",
       "      <td>236</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>248.0</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2757421940</td>\n",
       "      <td>2023-02-16</td>\n",
       "      <td>04:11:34</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>2023-02-16</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>4</td>\n",
       "      <td>400</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1123632510</td>\n",
       "      <td>2023-02-16</td>\n",
       "      <td>04:39:52</td>\n",
       "      <td>4</td>\n",
       "      <td>177</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3118324652</td>\n",
       "      <td>2023-02-16</td>\n",
       "      <td>05:06:47</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>548.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5731</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>2023-02-19</td>\n",
       "      <td>02:48:08</td>\n",
       "      <td>7</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5732</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>2023-02-19</td>\n",
       "      <td>02:48:10</td>\n",
       "      <td>4</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5733</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>2023-02-19</td>\n",
       "      <td>02:48:12</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>150.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5734</th>\n",
       "      <td>2584535982</td>\n",
       "      <td>2023-02-19</td>\n",
       "      <td>02:48:13</td>\n",
       "      <td>7</td>\n",
       "      <td>240</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>146.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5735</th>\n",
       "      <td>3164952431</td>\n",
       "      <td>2023-02-19</td>\n",
       "      <td>03:06:29</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>5952.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5736 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Host       Date      Time  Method  Endpoint  Protocol  \\\n",
       "0     2415145644 2023-02-16  03:28:45       4       236         6   \n",
       "1     2757421940 2023-02-16  04:11:34       4        10         6   \n",
       "2     1123632510 2023-02-16  04:39:52       4       400         6   \n",
       "3     1123632510 2023-02-16  04:39:52       4       177         6   \n",
       "4     3118324652 2023-02-16  05:06:47       4        40         6   \n",
       "...          ...        ...       ...     ...       ...       ...   \n",
       "5731  2584535982 2023-02-19  02:48:08       7       240         6   \n",
       "5732  2584535982 2023-02-19  02:48:10       4       240         6   \n",
       "5733  2584535982 2023-02-19  02:48:12       7        10         6   \n",
       "5734  2584535982 2023-02-19  02:48:13       7       240         6   \n",
       "5735  3164952431 2023-02-19  03:06:29       4        10         6   \n",
       "\n",
       "      Status Code  Content Size  No of Requests  \n",
       "0               6         248.0              62  \n",
       "1               0        5952.0               5  \n",
       "2               7         146.0              20  \n",
       "3               0         491.0              20  \n",
       "4               7         548.0               1  \n",
       "...           ...           ...             ...  \n",
       "5731            7         146.0               6  \n",
       "5732            7         146.0               6  \n",
       "5733            8         150.0               6  \n",
       "5734            7         146.0               6  \n",
       "5735            0        5952.0               1  \n",
       "\n",
       "[5736 rows x 9 columns]"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logs_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}