IoTID20_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0k5B9cN6TqmE"
   },
   "source": [
    "## Imports and settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3ZxRq5lcTqmF",
    "outputId": "c6208193-349b-4e89-a311-31430240017e"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "outdir = './processed/IoTID20/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import raw IoTID20 datafile\n",
    "The raw .csv datafileof the IoTID20 dataset can be downloaded from here:\n",
    "https://sites.google.com/view/iot-network-intrusion-dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df = pd.read_csv(\"./IoTID20_dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625783, 86)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 625783 entries, 0 to 625782\n",
      "Data columns (total 86 columns):\n",
      " #   Column             Non-Null Count   Dtype  \n",
      "---  ------             --------------   -----  \n",
      " 0   Flow_ID            625783 non-null  object \n",
      " 1   Src_IP             625783 non-null  object \n",
      " 2   Src_Port           625783 non-null  int64  \n",
      " 3   Dst_IP             625783 non-null  object \n",
      " 4   Dst_Port           625783 non-null  int64  \n",
      " 5   Protocol           625783 non-null  int64  \n",
      " 6   Timestamp          625783 non-null  object \n",
      " 7   Flow_Duration      625783 non-null  int64  \n",
      " 8   Tot_Fwd_Pkts       625783 non-null  int64  \n",
      " 9   Tot_Bwd_Pkts       625783 non-null  int64  \n",
      " 10  TotLen_Fwd_Pkts    625783 non-null  float64\n",
      " 11  TotLen_Bwd_Pkts    625783 non-null  float64\n",
      " 12  Fwd_Pkt_Len_Max    625783 non-null  float64\n",
      " 13  Fwd_Pkt_Len_Min    625783 non-null  float64\n",
      " 14  Fwd_Pkt_Len_Mean   625783 non-null  float64\n",
      " 15  Fwd_Pkt_Len_Std    625783 non-null  float64\n",
      " 16  Bwd_Pkt_Len_Max    625783 non-null  float64\n",
      " 17  Bwd_Pkt_Len_Min    625783 non-null  float64\n",
      " 18  Bwd_Pkt_Len_Mean   625783 non-null  float64\n",
      " 19  Bwd_Pkt_Len_Std    625783 non-null  float64\n",
      " 20  Flow_Byts/s        625783 non-null  float64\n",
      " 21  Flow_Pkts/s        625783 non-null  float64\n",
      " 22  Flow_IAT_Mean      625783 non-null  float64\n",
      " 23  Flow_IAT_Std       625783 non-null  float64\n",
      " 24  Flow_IAT_Max       625783 non-null  float64\n",
      " 25  Flow_IAT_Min       625783 non-null  float64\n",
      " 26  Fwd_IAT_Tot        625783 non-null  float64\n",
      " 27  Fwd_IAT_Mean       625783 non-null  float64\n",
      " 28  Fwd_IAT_Std        625783 non-null  float64\n",
      " 29  Fwd_IAT_Max        625783 non-null  float64\n",
      " 30  Fwd_IAT_Min        625783 non-null  float64\n",
      " 31  Bwd_IAT_Tot        625783 non-null  float64\n",
      " 32  Bwd_IAT_Mean       625783 non-null  float64\n",
      " 33  Bwd_IAT_Std        625783 non-null  float64\n",
      " 34  Bwd_IAT_Max        625783 non-null  float64\n",
      " 35  Bwd_IAT_Min        625783 non-null  float64\n",
      " 36  Fwd_PSH_Flags      625783 non-null  int64  \n",
      " 37  Bwd_PSH_Flags      625783 non-null  int64  \n",
      " 38  Fwd_URG_Flags      625783 non-null  int64  \n",
      " 39  Bwd_URG_Flags      625783 non-null  int64  \n",
      " 40  Fwd_Header_Len     625783 non-null  int64  \n",
      " 41  Bwd_Header_Len     625783 non-null  int64  \n",
      " 42  Fwd_Pkts/s         625783 non-null  float64\n",
      " 43  Bwd_Pkts/s         625783 non-null  float64\n",
      " 44  Pkt_Len_Min        625783 non-null  float64\n",
      " 45  Pkt_Len_Max        625783 non-null  float64\n",
      " 46  Pkt_Len_Mean       625783 non-null  float64\n",
      " 47  Pkt_Len_Std        625783 non-null  float64\n",
      " 48  Pkt_Len_Var        625783 non-null  float64\n",
      " 49  FIN_Flag_Cnt       625783 non-null  int64  \n",
      " 50  SYN_Flag_Cnt       625783 non-null  int64  \n",
      " 51  RST_Flag_Cnt       625783 non-null  int64  \n",
      " 52  PSH_Flag_Cnt       625783 non-null  int64  \n",
      " 53  ACK_Flag_Cnt       625783 non-null  int64  \n",
      " 54  URG_Flag_Cnt       625783 non-null  int64  \n",
      " 55  CWE_Flag_Count     625783 non-null  int64  \n",
      " 56  ECE_Flag_Cnt       625783 non-null  int64  \n",
      " 57  Down/Up_Ratio      625783 non-null  float64\n",
      " 58  Pkt_Size_Avg       625783 non-null  float64\n",
      " 59  Fwd_Seg_Size_Avg   625783 non-null  float64\n",
      " 60  Bwd_Seg_Size_Avg   625783 non-null  float64\n",
      " 61  Fwd_Byts/b_Avg     625783 non-null  int64  \n",
      " 62  Fwd_Pkts/b_Avg     625783 non-null  int64  \n",
      " 63  Fwd_Blk_Rate_Avg   625783 non-null  int64  \n",
      " 64  Bwd_Byts/b_Avg     625783 non-null  int64  \n",
      " 65  Bwd_Pkts/b_Avg     625783 non-null  int64  \n",
      " 66  Bwd_Blk_Rate_Avg   625783 non-null  int64  \n",
      " 67  Subflow_Fwd_Pkts   625783 non-null  int64  \n",
      " 68  Subflow_Fwd_Byts   625783 non-null  int64  \n",
      " 69  Subflow_Bwd_Pkts   625783 non-null  int64  \n",
      " 70  Subflow_Bwd_Byts   625783 non-null  int64  \n",
      " 71  Init_Fwd_Win_Byts  625783 non-null  int64  \n",
      " 72  Init_Bwd_Win_Byts  625783 non-null  int64  \n",
      " 73  Fwd_Act_Data_Pkts  625783 non-null  int64  \n",
      " 74  Fwd_Seg_Size_Min   625783 non-null  int64  \n",
      " 75  Active_Mean        625783 non-null  float64\n",
      " 76  Active_Std         625783 non-null  float64\n",
      " 77  Active_Max         625783 non-null  float64\n",
      " 78  Active_Min         625783 non-null  float64\n",
      " 79  Idle_Mean          625783 non-null  float64\n",
      " 80  Idle_Std           625783 non-null  float64\n",
      " 81  Idle_Max           625783 non-null  float64\n",
      " 82  Idle_Min           625783 non-null  float64\n",
      " 83  Label              625783 non-null  object \n",
      " 84  Cat                625783 non-null  object \n",
      " 85  Sub_Cat            625783 non-null  object \n",
      "dtypes: float64(45), int64(34), object(7)\n",
      "memory usage: 410.6+ MB\n"
     ]
    }
   ],
   "source": [
    "raw_df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data pre-processing\n",
    "Pre-processing is done according to the paper by Song et. al. (2021):  \n",
    "*Song, Youngrok, Sangwon Hyun, and Yun-Gyung Cheong. \"Analysis of autoencoders for network intrusion detection.\" Sensors 21.13 (2021): 4294.*\n",
    "\n",
    "- Drop all rows containing -infinite, +infinite or NaN values \n",
    "- The columns Timestamp, Flow_ID, Scr_IP, Dst_IP can be dropped from the dataframe, as this information is specific to the capture\n",
    "- Also drop the specific attack type (columns Cat and Sub_Cat), as only attack or normal traffic will be classified\n",
    "- Covert the Protocol column (3 categories) into a one-hot-encoded vector\n",
    "- Transform columns Src_Port and Dst_Port into 3 categories and encode as a one-hot-encoded vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df = raw_df.replace([np.inf, -np.inf], np.nan)\n",
    "raw_df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.isnull().values.any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df.drop(['Flow_ID','Src_IP','Dst_IP','Timestamp'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df.drop(['Cat','Sub_Cat'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df = pd.get_dummies(raw_df, columns=['Protocol'], dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625415, 82)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = [-np.inf, 0, 1023, 49151, 65536, np.inf]\n",
    "names = ['unkown_port_negative', 'well_known_port', 'registered_port', 'dynamic_port', 'unkown_port_positive']\n",
    "raw_df['Src_Port_Type'] = pd.cut(raw_df['Src_Port'], bins, labels=names)\n",
    "raw_df['Dst_Port_Type'] = pd.cut(raw_df['Dst_Port'], bins, labels=names)\n",
    "raw_df.drop(['Src_Port','Dst_Port'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#raw_df[['Src_Port_Type']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#raw_df[['Dst_Port_Type']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df = pd.get_dummies(raw_df, columns=['Src_Port_Type'], dtype=int)\n",
    "raw_df = pd.get_dummies(raw_df, columns=['Dst_Port_Type'], dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop non-existent port numbers (< 0; > 65536)\n",
    "raw_df.drop(['Src_Port_Type_unkown_port_negative','Src_Port_Type_unkown_port_positive'],axis=1,inplace=True)\n",
    "raw_df.drop(['Dst_Port_Type_unkown_port_negative','Dst_Port_Type_unkown_port_positive'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(625415, 86)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data into Normal and Attack  traffic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_df = pd.get_dummies(raw_df, columns=['Label'], dtype=int)\n",
    "raw_df.drop(['Label_Normal'],axis=1,inplace=True)\n",
    "raw_df.rename(columns={\"Label_Anomaly\": \"class\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class\n",
       "1        585342\n",
       "0         40073\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_df[['class']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "normal_df = raw_df[raw_df['class'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40073, 86)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "attack_df = raw_df[raw_df['class'] != 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(585342, 86)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attack_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Training and Testing files\n",
    "The dataset is split into training and testing sets. After the raw data is split into normal and attack traffic, \n",
    "the training dataset is formed by using 75% of the normal data, and the test dataset is formed by combining the \n",
    "remaining 25% of the normal traffic with the attack traffic again. This is the same procedure and ratio as that \n",
    "which was used by Meidan et. al. (2018)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_split_perc = 0.75"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle all rows to avoid implicit ordering\n",
    "normal_df.sample(frac=1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "split = math.floor(normal_df.shape[0] * train_split_perc)\n",
    "training_df = normal_df.iloc[:split,:]\n",
    "test_normal_df = normal_df.iloc[split:,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30054, 86)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10019, 86)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_normal_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_df = pd.concat([attack_df, test_normal_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(595361, 86)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "class\n",
       "1        585342\n",
       "0         10019\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_df[['class']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Training and Testing files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_df.to_csv('{}/{}'.format(outdir,'training.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "testing_df.to_csv('{}/{}'.format(outdir,'testing.csv'))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}