cidr2mmdb.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-07T17:14:19.250663Z",
     "start_time": "2023-06-07T17:14:18.747476Z"
    }
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:48:20.116331Z",
     "start_time": "2024-01-14T21:48:20.112401Z"
    }
   },
   "outputs": [],
   "source": [
    "# use this command for installing the MMDBWriter package\n",
    "# !pip install -U git+https://github.com/VimT/MaxMind-DB-Writer-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:48:20.645543Z",
     "start_time": "2024-01-14T21:48:20.122586Z"
    }
   },
   "outputs": [],
   "source": [
    "import netaddr\n",
    "import ipaddress\n",
    "import maxminddb\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from io import StringIO\n",
    "from netaddr import IPSet\n",
    "from mmdb_writer import MMDBWriter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:48:20.695611Z",
     "start_time": "2024-01-14T21:48:20.691805Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "with open(\"./data/cidrs.txt\", \"r\") as file:\n",
    "    text = file.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess\n",
    "## Remove redundancy, select only IP v4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:48:21.874391Z",
     "start_time": "2024-01-14T21:48:21.801596Z"
    }
   },
   "outputs": [],
   "source": [
    "cidrs = text.split(\"\\n\")\n",
    "cidrs = [item for item in cidrs if len(item) > 0]\n",
    "cidrs = pd.Series(cidrs).unique()\n",
    "# only use version 4 cidrs\n",
    "cidrs = [cidr for cidr in cidrs if ipaddress.ip_network(cidr).version == 4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:48:33.382864Z",
     "start_time": "2024-01-14T21:48:25.910800Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1945/1945 [00:07<00:00, 261.77it/s]\n"
     ]
    }
   ],
   "source": [
    "def remove_not_superne(cidrs_list):    \n",
    "    remove_list = []\n",
    "    # calc network for each cirdr, catch for speed up\n",
    "    cidr2network = {}\n",
    "    for cidr in cidrs_list:\n",
    "        cidr2network[cidr] = ipaddress.ip_network(cidr)\n",
    "    \n",
    "    for i in tqdm(range(0, len(cidrs_list))):\n",
    "        cird_i = cidrs_list[i]    \n",
    "        if(cird_i in remove_list):\n",
    "            continue\n",
    "        network1 = cidr2network[cird_i]\n",
    "        for j in range(i+1, len(cidrs_list)):        \n",
    "            cird_j = cidrs_list[j]        \n",
    "            network2 = cidr2network[cird_j]\n",
    "            if network1.supernet_of(network2):                \n",
    "                remove_list.append(cird_j)            \n",
    "            elif network2.supernet_of(network1):                \n",
    "                remove_list.append(cird_i)  \n",
    "                break        \n",
    "    return remove_list \n",
    "\n",
    "rm = remove_not_superne(cidrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:48:42.149995Z",
     "start_time": "2024-01-14T21:48:42.142289Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1945\n",
      "0\n",
      "1945\n"
     ]
    }
   ],
   "source": [
    "print(len(cidrs))\n",
    "print(len(rm))\n",
    "cidrs = set(cidrs) - set(rm)\n",
    "print(len(cidrs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MMDB Writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:49:12.260675Z",
     "start_time": "2024-01-14T21:49:12.070254Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1945/1945 [00:00<00:00, 17506.91it/s]\n"
     ]
    }
   ],
   "source": [
    "writer = MMDBWriter()\n",
    "for cidr in tqdm(cidrs):    \n",
    "    writer.insert_network(IPSet([cidr]), {'country': 'IR', 'isp': \"NA\"})    \n",
    "writer.to_db_file('./result/geoipIR.mmdb')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Test dataste"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:49:29.887396Z",
     "start_time": "2024-01-14T21:49:29.879972Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'country': 'IR', 'isp': 'NA'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db = maxminddb.open_database('./result/geoipIR.mmdb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:49:59.395686Z",
     "start_time": "2024-01-14T21:49:59.389631Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'country': 'IR', 'isp': 'NA'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db.get(\"79.127.127.16\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-14T21:59:17.950835Z",
     "start_time": "2024-01-14T21:59:17.945667Z"
    }
   },
   "outputs": [],
   "source": [
    "db.get(\"172.67.73.163\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}