notebooks/imperceptible_attack_on_tabular_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imperceptible attack on tabular data using LowProFool algorithm\n",
    "In this notebook, we will learn how to execute imperceptible attack on tabular data with the LowProFool algorithm (https://arxiv.org/abs/1911.03274). We will use **iris flowers** and **breast cancer** datasets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paul/Desktop/art-aml-fork/adversarial-robustness-toolbox/art-aml/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from art.estimators.classification.scikitlearn import ScikitlearnLogisticRegression\n",
    "from art.estimators.classification.pytorch import PyTorchClassifier\n",
    "from art.attacks.evasion import LowProFool\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "# Data preparation\n",
    "\n",
    "Firstly, we load the datasets, standardize them, and split into training and validation sets. We also choose the clipping values for both datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def standardize(data):\n",
    "    \"\"\"\n",
    "    Get both the standardized data and the used scaler.\n",
    "    \"\"\"\n",
    "    columns = data.columns\n",
    "    scaler = StandardScaler()\n",
    "    x_scaled = scaler.fit_transform(data)\n",
    "    \n",
    "    return pd.DataFrame(data=x_scaled, columns=columns), scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=0)\n",
    "\n",
    "def get_train_and_valid(design_matrix, labels):\n",
    "    \"\"\"\n",
    "    Split dataset into training and validation sets.\n",
    "    \"\"\"\n",
    "    for train_idx, valid_idx in split.split(design_matrix, labels):\n",
    "        X_train = design_matrix.iloc[train_idx].copy()\n",
    "        X_valid = design_matrix.iloc[valid_idx].copy()\n",
    "        y_train = labels.iloc[train_idx].copy()\n",
    "        y_valid = labels.iloc[valid_idx].copy()\n",
    "\n",
    "    return X_train, y_train, X_valid, y_valid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading and preparation of the iris flowers dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>6.7</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>6.3</td>\n",
       "      <td>2.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>6.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.2</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>6.2</td>\n",
       "      <td>3.4</td>\n",
       "      <td>5.4</td>\n",
       "      <td>2.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>5.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.1</td>\n",
       "      <td>1.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)\n",
       "0                  5.1               3.5                1.4               0.2\n",
       "1                  4.9               3.0                1.4               0.2\n",
       "2                  4.7               3.2                1.3               0.2\n",
       "3                  4.6               3.1                1.5               0.2\n",
       "4                  5.0               3.6                1.4               0.2\n",
       "..                 ...               ...                ...               ...\n",
       "145                6.7               3.0                5.2               2.3\n",
       "146                6.3               2.5                5.0               1.9\n",
       "147                6.5               3.0                5.2               2.0\n",
       "148                6.2               3.4                5.4               2.3\n",
       "149                5.9               3.0                5.1               1.8\n",
       "\n",
       "[150 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "iris = datasets.load_iris()\n",
    "design_matrix_iris = pd.DataFrame(data=iris['data'], columns=iris['feature_names'])\n",
    "labels_iris = pd.Series(data=iris['target'])\n",
    "display(design_matrix_iris)\n",
    "\n",
    "design_matrix_iris_scaled, iris_scaler = standardize(design_matrix_iris)\n",
    "\n",
    "X_train_iris, y_train_iris, X_valid_iris, y_valid_iris =\\\n",
    "    get_train_and_valid(design_matrix_iris_scaled, labels_iris)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading and preparation of the breast cancer dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst radius</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.30010</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>25.380</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.16220</td>\n",
       "      <td>0.66560</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.08690</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>24.990</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.12380</td>\n",
       "      <td>0.18660</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.19740</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>23.570</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.14440</td>\n",
       "      <td>0.42450</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.24140</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>14.910</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.20980</td>\n",
       "      <td>0.86630</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.19800</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>22.540</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.13740</td>\n",
       "      <td>0.20500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>21.56</td>\n",
       "      <td>22.39</td>\n",
       "      <td>142.00</td>\n",
       "      <td>1479.0</td>\n",
       "      <td>0.11100</td>\n",
       "      <td>0.11590</td>\n",
       "      <td>0.24390</td>\n",
       "      <td>0.13890</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.05623</td>\n",
       "      <td>...</td>\n",
       "      <td>25.450</td>\n",
       "      <td>26.40</td>\n",
       "      <td>166.10</td>\n",
       "      <td>2027.0</td>\n",
       "      <td>0.14100</td>\n",
       "      <td>0.21130</td>\n",
       "      <td>0.4107</td>\n",
       "      <td>0.2216</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.07115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>20.13</td>\n",
       "      <td>28.25</td>\n",
       "      <td>131.20</td>\n",
       "      <td>1261.0</td>\n",
       "      <td>0.09780</td>\n",
       "      <td>0.10340</td>\n",
       "      <td>0.14400</td>\n",
       "      <td>0.09791</td>\n",
       "      <td>0.1752</td>\n",
       "      <td>0.05533</td>\n",
       "      <td>...</td>\n",
       "      <td>23.690</td>\n",
       "      <td>38.25</td>\n",
       "      <td>155.00</td>\n",
       "      <td>1731.0</td>\n",
       "      <td>0.11660</td>\n",
       "      <td>0.19220</td>\n",
       "      <td>0.3215</td>\n",
       "      <td>0.1628</td>\n",
       "      <td>0.2572</td>\n",
       "      <td>0.06637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>16.60</td>\n",
       "      <td>28.08</td>\n",
       "      <td>108.30</td>\n",
       "      <td>858.1</td>\n",
       "      <td>0.08455</td>\n",
       "      <td>0.10230</td>\n",
       "      <td>0.09251</td>\n",
       "      <td>0.05302</td>\n",
       "      <td>0.1590</td>\n",
       "      <td>0.05648</td>\n",
       "      <td>...</td>\n",
       "      <td>18.980</td>\n",
       "      <td>34.12</td>\n",
       "      <td>126.70</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>0.11390</td>\n",
       "      <td>0.30940</td>\n",
       "      <td>0.3403</td>\n",
       "      <td>0.1418</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>0.07820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>20.60</td>\n",
       "      <td>29.33</td>\n",
       "      <td>140.10</td>\n",
       "      <td>1265.0</td>\n",
       "      <td>0.11780</td>\n",
       "      <td>0.27700</td>\n",
       "      <td>0.35140</td>\n",
       "      <td>0.15200</td>\n",
       "      <td>0.2397</td>\n",
       "      <td>0.07016</td>\n",
       "      <td>...</td>\n",
       "      <td>25.740</td>\n",
       "      <td>39.42</td>\n",
       "      <td>184.60</td>\n",
       "      <td>1821.0</td>\n",
       "      <td>0.16500</td>\n",
       "      <td>0.86810</td>\n",
       "      <td>0.9387</td>\n",
       "      <td>0.2650</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.12400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>7.76</td>\n",
       "      <td>24.54</td>\n",
       "      <td>47.92</td>\n",
       "      <td>181.0</td>\n",
       "      <td>0.05263</td>\n",
       "      <td>0.04362</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.1587</td>\n",
       "      <td>0.05884</td>\n",
       "      <td>...</td>\n",
       "      <td>9.456</td>\n",
       "      <td>30.37</td>\n",
       "      <td>59.16</td>\n",
       "      <td>268.6</td>\n",
       "      <td>0.08996</td>\n",
       "      <td>0.06444</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.07039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>569 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "0          17.99         10.38          122.80     1001.0          0.11840   \n",
       "1          20.57         17.77          132.90     1326.0          0.08474   \n",
       "2          19.69         21.25          130.00     1203.0          0.10960   \n",
       "3          11.42         20.38           77.58      386.1          0.14250   \n",
       "4          20.29         14.34          135.10     1297.0          0.10030   \n",
       "..           ...           ...             ...        ...              ...   \n",
       "564        21.56         22.39          142.00     1479.0          0.11100   \n",
       "565        20.13         28.25          131.20     1261.0          0.09780   \n",
       "566        16.60         28.08          108.30      858.1          0.08455   \n",
       "567        20.60         29.33          140.10     1265.0          0.11780   \n",
       "568         7.76         24.54           47.92      181.0          0.05263   \n",
       "\n",
       "     mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "0             0.27760         0.30010              0.14710         0.2419   \n",
       "1             0.07864         0.08690              0.07017         0.1812   \n",
       "2             0.15990         0.19740              0.12790         0.2069   \n",
       "3             0.28390         0.24140              0.10520         0.2597   \n",
       "4             0.13280         0.19800              0.10430         0.1809   \n",
       "..                ...             ...                  ...            ...   \n",
       "564           0.11590         0.24390              0.13890         0.1726   \n",
       "565           0.10340         0.14400              0.09791         0.1752   \n",
       "566           0.10230         0.09251              0.05302         0.1590   \n",
       "567           0.27700         0.35140              0.15200         0.2397   \n",
       "568           0.04362         0.00000              0.00000         0.1587   \n",
       "\n",
       "     mean fractal dimension  ...  worst radius  worst texture  \\\n",
       "0                   0.07871  ...        25.380          17.33   \n",
       "1                   0.05667  ...        24.990          23.41   \n",
       "2                   0.05999  ...        23.570          25.53   \n",
       "3                   0.09744  ...        14.910          26.50   \n",
       "4                   0.05883  ...        22.540          16.67   \n",
       "..                      ...  ...           ...            ...   \n",
       "564                 0.05623  ...        25.450          26.40   \n",
       "565                 0.05533  ...        23.690          38.25   \n",
       "566                 0.05648  ...        18.980          34.12   \n",
       "567                 0.07016  ...        25.740          39.42   \n",
       "568                 0.05884  ...         9.456          30.37   \n",
       "\n",
       "     worst perimeter  worst area  worst smoothness  worst compactness  \\\n",
       "0             184.60      2019.0           0.16220            0.66560   \n",
       "1             158.80      1956.0           0.12380            0.18660   \n",
       "2             152.50      1709.0           0.14440            0.42450   \n",
       "3              98.87       567.7           0.20980            0.86630   \n",
       "4             152.20      1575.0           0.13740            0.20500   \n",
       "..               ...         ...               ...                ...   \n",
       "564           166.10      2027.0           0.14100            0.21130   \n",
       "565           155.00      1731.0           0.11660            0.19220   \n",
       "566           126.70      1124.0           0.11390            0.30940   \n",
       "567           184.60      1821.0           0.16500            0.86810   \n",
       "568            59.16       268.6           0.08996            0.06444   \n",
       "\n",
       "     worst concavity  worst concave points  worst symmetry  \\\n",
       "0             0.7119                0.2654          0.4601   \n",
       "1             0.2416                0.1860          0.2750   \n",
       "2             0.4504                0.2430          0.3613   \n",
       "3             0.6869                0.2575          0.6638   \n",
       "4             0.4000                0.1625          0.2364   \n",
       "..               ...                   ...             ...   \n",
       "564           0.4107                0.2216          0.2060   \n",
       "565           0.3215                0.1628          0.2572   \n",
       "566           0.3403                0.1418          0.2218   \n",
       "567           0.9387                0.2650          0.4087   \n",
       "568           0.0000                0.0000          0.2871   \n",
       "\n",
       "     worst fractal dimension  \n",
       "0                    0.11890  \n",
       "1                    0.08902  \n",
       "2                    0.08758  \n",
       "3                    0.17300  \n",
       "4                    0.07678  \n",
       "..                       ...  \n",
       "564                  0.07115  \n",
       "565                  0.06637  \n",
       "566                  0.07820  \n",
       "567                  0.12400  \n",
       "568                  0.07039  \n",
       "\n",
       "[569 rows x 30 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cancer = datasets.load_breast_cancer()\n",
    "design_matrix_cancer = pd.DataFrame(data=cancer['data'], columns=cancer['feature_names'])\n",
    "labels_cancer = pd.Series(data=cancer['target'])\n",
    "display(design_matrix_cancer)\n",
    "\n",
    "design_matrix_cancer_scaled, cancer_scaler = standardize(design_matrix_cancer)\n",
    "\n",
    "X_train_cancer, y_train_cancer, X_valid_cancer, y_valid_cancer =\\\n",
    "    get_train_and_valid(design_matrix_cancer_scaled, labels_cancer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Clip-values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Iris flowers dataset** - minimum and maximum values in training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clip-values:\n",
      "  Lower bound: [-1.87002413 -2.43394714 -1.56757623 -1.44707648]\n",
      "  Upper bound: [2.4920192  3.09077525 1.78583195 1.71209594]\n",
      "\n",
      "Clip-values in original scale:\n",
      "  Lower bound: [4.3 2.  1.  0.1]\n",
      "  Upper bound: [7.9 4.4 6.9 2.5]\n"
     ]
    }
   ],
   "source": [
    "scaled_clip_values_iris = (\n",
    "    np.array(X_train_iris.min()),\n",
    "    np.array(X_train_iris.max())\n",
    ")\n",
    "print(\"Clip-values:\")\n",
    "print(\"  Lower bound:\", scaled_clip_values_iris[0])\n",
    "print(\"  Upper bound:\", scaled_clip_values_iris[1])\n",
    "\n",
    "print(\"\\nClip-values in original scale:\")\n",
    "clip_values_iris = iris_scaler.inverse_transform(scaled_clip_values_iris)\n",
    "print(\"  Lower bound:\", clip_values_iris[0])\n",
    "print(\"  Upper bound:\", clip_values_iris[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Breast cancer dataset** - 1 standard deviation boundary.\n",
    "\n",
    "Note: Here, we create clip values such that all values should fall within the one standard deviation interval. Thanks to the dataset being priorly standardized, it is a trivial problem. Moreover clip values can be concisely expressed as just a single tuple `(-1., 1.)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_clip_values_cancer = (-1., 1.)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "# Quick LowProFool example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A kickoff example using LowProFool to generate adversaries for SVC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import SVC\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "# import ART wrapper for scikit-learn SVC\n",
    "from art.estimators.classification.scikitlearn import ScikitlearnSVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC()"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize and fit SVC\n",
    "svc_clf = SVC()\n",
    "svc_clf.fit(X_train_cancer.values, y_train_cancer) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wrap SVC using the wrapper\n",
    "classif_svc = ScikitlearnSVC(\n",
    "    model = svc_clf,\n",
    "    clip_values = scaled_clip_values_cancer\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<art.attacks.evasion.lowprofool.LowProFool at 0x7f38dfcee5c0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize LowProFool instance and fit feature importance\n",
    "lpf_svc = LowProFool(\n",
    "    classifier = classif_svc,\n",
    "    n_steps    = 15,\n",
    "    eta        = 15,\n",
    "    lambd      = 1.75,\n",
    "    eta_decay  = 0.985,\n",
    "    verbose    = True\n",
    ")\n",
    "lpf_svc.fit_importances(X_train_cancer, y_train_cancer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LowProFool: 100%|██████████| 15/15 [00:02<00:00,  5.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set size: 114\n",
      "Success rate:  100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Create random array of samples to be used as adversaries\n",
    "n_classes = lpf_svc.n_classes\n",
    "targets = np.eye(n_classes)[np.array(\n",
    "    y_valid_cancer.apply(lambda x: np.random.choice([i for i in range(n_classes) if i != x]))\n",
    ")]\n",
    "\n",
    "# Generate adversaries\n",
    "adversaries = lpf_svc.generate(x=X_valid_cancer, y=targets)\n",
    "\n",
    "# Check the success rate\n",
    "expected = np.argmax(targets, axis=1)\n",
    "predicted = np.argmax(classif_svc.predict(adversaries), axis=1)\n",
    "\n",
    "correct = (expected == predicted)\n",
    "success_rate = np.sum(correct) / correct.shape[0]\n",
    "print(\"Test set size: {}\".format(targets.shape[0]))\n",
    "print(\"Success rate:  {:.2f}%\".format(100*success_rate))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, one can easily generate good quality adversary examples in just a few lines of code."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "# Extended LowProFool example\n",
    "\n",
    "In this section we present you few more examples of `LowProFool` adversarial attacks carried out in a similar fashion, but employing different underlying models and on different datasets."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation of classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training on iris flowers dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_regression_clf_iris = LogisticRegression()\n",
    "log_regression_clf_iris.fit(X_train_iris.values, y_train_iris)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training on breast cancer dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_regression_clf_cancer = LogisticRegression()\n",
    "log_regression_clf_cancer.fit(X_train_cancer.values, y_train_cancer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_nn_model(input_dimensions, hidden_neurons, output_dimensions):\n",
    "    \"\"\"\n",
    "    Prepare PyTorch (torch) neural network.\n",
    "    \"\"\"\n",
    "    return torch.nn.Sequential(\n",
    "        nn.Linear(input_dimensions, hidden_neurons),\n",
    "        nn.ReLU(),\n",
    "        nn.Linear(hidden_neurons, output_dimensions),\n",
    "        nn.Softmax(dim=1)\n",
    "    )\n",
    "\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "def train_nn(nn_model, X, y, learning_rate, epochs):\n",
    "    \"\"\"\n",
    "    Train provided neural network.\n",
    "    \"\"\"\n",
    "    optimizer = optim.SGD(nn_model.parameters(), lr=learning_rate)\n",
    "    \n",
    "    for _ in range(epochs):\n",
    "        y_pred = nn_model.forward(X)\n",
    "        loss = loss_fn(y_pred, y)\n",
    "        nn_model.zero_grad()\n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training on iris flowers dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = Variable(torch.FloatTensor(np.array(X_train_iris)))\n",
    "y = Variable(torch.FloatTensor(np.eye(3)[y_train_iris]))\n",
    "nn_model_iris = get_nn_model(4, 10, 3)\n",
    "train_nn(nn_model_iris, X, y, 1e-4, 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training on breast cancer dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = Variable(torch.FloatTensor(np.array(X_train_cancer.values)))\n",
    "y = Variable(torch.FloatTensor(np.eye(2)[y_train_cancer]))\n",
    "nn_model_cancer = get_nn_model(30, 50, 2)\n",
    "train_nn(nn_model_cancer, X, y, 1e-4, 1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actual usage of LowProFool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lowprofool_generate_adversaries_test_lr(lowprofool, classifier, x_valid, y_valid):\n",
    "    \"\"\"\n",
    "    Testing utility.\n",
    "    \"\"\"\n",
    "    n_classes = lowprofool.n_classes\n",
    "    \n",
    "    # Generate targets\n",
    "    target = np.eye(n_classes)[np.array(\n",
    "        y_valid.apply(\n",
    "            lambda x: np.random.choice([i for i in range(n_classes) if i != x]))\n",
    "    )]\n",
    "    \n",
    "    # Generate adversaries\n",
    "    adversaries = lowprofool.generate(x=x_valid, y=target)\n",
    "\n",
    "    # Test - check the success rate\n",
    "    expected = np.argmax(target, axis=1)\n",
    "    predicted = np.argmax(classifier.predict_proba(adversaries), axis=1)\n",
    "    correct = (expected == predicted)\n",
    "    \n",
    "    success_rate = np.sum(correct) / correct.shape[0]\n",
    "    print(\"Success rate: {:.2f}%\".format(100*success_rate))\n",
    "    \n",
    "    return adversaries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Iris flowers dataset test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success rate: 100.00%\n"
     ]
    }
   ],
   "source": [
    "# Wrapping classifier into appropriate ART-friendly wrapper\n",
    "logistic_regression_iris_wrapper = ScikitlearnLogisticRegression(\n",
    "    model       = log_regression_clf_iris, \n",
    "    clip_values = scaled_clip_values_iris\n",
    ")\n",
    "\n",
    "# Creating LowProFool instance\n",
    "lpf_logistic_regression_iris = LowProFool(\n",
    "    classifier = logistic_regression_iris_wrapper, \n",
    "    eta        = 5,\n",
    "    lambd      = 0.2, \n",
    "    eta_decay  = 0.9\n",
    ")\n",
    "\n",
    "# Fitting feature importance\n",
    "lpf_logistic_regression_iris.fit_importances(X_train_iris, y_train_iris)\n",
    "\n",
    "# Testing\n",
    "results_lr_ir = lowprofool_generate_adversaries_test_lr(\n",
    "    lowprofool = lpf_logistic_regression_iris,\n",
    "    classifier = log_regression_clf_iris, \n",
    "    x_valid    = X_valid_iris, \n",
    "    y_valid    = y_valid_iris\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Successful adversarial attack. Below we can see the original features and their classes, as well as the adversaries generated by `LowProFool` and predicted class-wise probabilities of them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_predictions(values, preds, max_features=4):\n",
    "    \"\"\"\n",
    "    Utility function for printing predictions.\n",
    "    \"\"\"\n",
    "    predictions = zip(list(map(lambda e: e[:max_features], values.tolist())), preds.tolist())\n",
    "    \n",
    "    for features, pred in predictions:\n",
    "        print(\"Features[:{}]:\".format(max_features))\n",
    "        for i, val in enumerate(features):\n",
    "            if i % 6 != 5: print(\"{:>10.4f}\".format(val), end='')\n",
    "            else:          print(\"{:>10.4f}\\n\".format(val), end='')\n",
    "        if len(features) % 6 != 0: print()\n",
    "        \n",
    "        print(\"Prediction (probability -> class):\")\n",
    "        for val in pred:\n",
    "            print(\"{:>8.3f}\".format(val), end='')\n",
    "        print(\"  ->  {}\\n\".format(np.argmax(pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Original values ===\n",
      "\n",
      "Features[:4]:\n",
      "    6.3000    3.3000    6.0000    2.5000\n",
      "Prediction (probability -> class):\n",
      "   0.000   0.010   0.989  ->  2\n",
      "\n",
      "Features[:4]:\n",
      "    5.1000    3.5000    1.4000    0.2000\n",
      "Prediction (probability -> class):\n",
      "   0.983   0.017   0.000  ->  0\n",
      "\n",
      "Features[:4]:\n",
      "    4.9000    3.1000    1.5000    0.1000\n",
      "Prediction (probability -> class):\n",
      "   0.957   0.043   0.000  ->  0\n",
      "\n",
      "\n",
      "=== Adversaries (LowProFool results) ===\n",
      "\n",
      "Features[:4]:\n",
      "    4.4880    4.3761    1.6556    0.4242\n",
      "Prediction (probability -> class):\n",
      "   0.999   0.001   0.000  ->  0\n",
      "\n",
      "Features[:4]:\n",
      "    6.9912    2.2775    6.3588    2.3217\n",
      "Prediction (probability -> class):\n",
      "   0.000   0.007   0.993  ->  2\n",
      "\n",
      "Features[:4]:\n",
      "    6.7039    2.1788    3.4307    0.1476\n",
      "Prediction (probability -> class):\n",
      "   0.007   0.993   0.000  ->  1\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paul/Desktop/art-aml-fork/adversarial-robustness-toolbox/art-aml/lib/python3.10/site-packages/sklearn/base.py:443: UserWarning: X has feature names, but LogisticRegression was fitted without feature names\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "print(\"=== Original values ===\\n\")\n",
    "\n",
    "print_predictions(iris_scaler.inverse_transform(X_valid_iris[-3:].values), \n",
    "    log_regression_clf_iris.predict_proba(X_valid_iris[-3:]))\n",
    "    \n",
    "print(\"\\n=== Adversaries (LowProFool results) ===\\n\")\n",
    "    \n",
    "print_predictions(iris_scaler.inverse_transform(results_lr_ir[-3:]), \n",
    "    log_regression_clf_iris.predict_proba(results_lr_ir[-3:]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Breast cancer dataset test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success rate: 100.00%\n"
     ]
    }
   ],
   "source": [
    "# Wrapping classifier into appropriate ART-friendly wrapper\n",
    "logistic_regression_cancer_wrapper = ScikitlearnLogisticRegression(\n",
    "    model       = log_regression_clf_cancer, \n",
    "    clip_values = scaled_clip_values_cancer\n",
    ")\n",
    "\n",
    "# Creating LowProFool instance\n",
    "lpf_logistic_regression_cancer = LowProFool(\n",
    "    classifier = logistic_regression_cancer_wrapper, \n",
    "    eta        = 5,\n",
    "    lambd      = 0.2, \n",
    "    eta_decay  = 0.9\n",
    ")\n",
    "\n",
    "# Fitting feature importance\n",
    "lpf_logistic_regression_cancer.fit_importances(X_train_cancer, y_train_cancer)\n",
    "\n",
    "# Testing\n",
    "results_lr_bc = lowprofool_generate_adversaries_test_lr(\n",
    "    lowprofool = lpf_logistic_regression_cancer,\n",
    "    classifier = log_regression_clf_cancer, \n",
    "    x_valid    = X_valid_cancer, \n",
    "    y_valid    = y_valid_cancer\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Original values ===\n",
      "\n",
      "Features[:30]:\n",
      "   12.8700   19.5400   82.6700  509.2000    0.0914    0.0788\n",
      "    0.0180    0.0209    0.1861    0.0635    0.3665    0.7693\n",
      "    2.5970   26.5000    0.0059    0.0136    0.0071    0.0065\n",
      "    0.0222    0.0024   14.4500   24.3800   95.1400  626.9000\n",
      "    0.1214    0.1652    0.0713    0.0638    0.3313    0.0774\n",
      "Prediction (probability -> class):\n",
      "   0.006   0.994  ->  1\n",
      "\n",
      "Features[:30]:\n",
      "   11.8100   17.3900   75.2700  428.9000    0.1007    0.0556\n",
      "    0.0235    0.0155    0.1718    0.0578    0.1859    1.9260\n",
      "    1.0110   14.4700    0.0078    0.0088    0.0156    0.0062\n",
      "    0.0314    0.0020   12.5700   26.4800   79.5700  489.5000\n",
      "    0.1356    0.1000    0.0880    0.0431    0.3200    0.0658\n",
      "Prediction (probability -> class):\n",
      "   0.000   1.000  ->  1\n",
      "\n",
      "\n",
      "=== Adversaries (LowProFool results) ===\n",
      "\n",
      "Features[:30]:\n",
      "   17.6168   23.5787  116.0187 1003.4447    0.0997    0.0517\n",
      "    0.1675    0.0872    0.1735    0.0557    0.6810    0.7894\n",
      "    4.8775   85.5782    0.0085    0.0076    0.0017    0.0179\n",
      "    0.0123    0.0012   21.0495   31.7998  140.4935 1444.1356\n",
      "    0.1551    0.3039    0.4784    0.1794    0.3519    0.1020\n",
      "Prediction (probability -> class):\n",
      "   1.000   0.000  ->  0\n",
      "\n",
      "Features[:30]:\n",
      "   17.6098   23.5744  115.9685 1002.9523    0.1091    0.0516\n",
      "    0.1675    0.0871    0.1591    0.0557    0.6803    1.7680\n",
      "    4.8715   85.5356    0.0100    0.0076    0.0039    0.0179\n",
      "    0.0137    0.0012   21.0357   31.8050  140.3774 1443.2493\n",
      "    0.1551    0.2395    0.4785    0.1792    0.3518    0.1019\n",
      "Prediction (probability -> class):\n",
      "   1.000   0.000  ->  0\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paul/Desktop/art-aml-fork/adversarial-robustness-toolbox/art-aml/lib/python3.10/site-packages/sklearn/base.py:443: UserWarning: X has feature names, but LogisticRegression was fitted without feature names\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "print(\"=== Original values ===\\n\")\n",
    "\n",
    "print_predictions(cancer_scaler.inverse_transform(X_valid_cancer[-2:]), \n",
    "                  log_regression_clf_cancer.predict_proba(X_valid_cancer[-2:]), max_features=30)\n",
    "\n",
    "print(\"\\n=== Adversaries (LowProFool results) ===\\n\")\n",
    "\n",
    "print_predictions(cancer_scaler.inverse_transform(results_lr_bc[-2:]), \n",
    "                  log_regression_clf_cancer.predict_proba(results_lr_bc[-2:]), max_features=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lowprofool_generate_adversaries_test_nn(lowprofool, classifier, x_valid, y_valid):\n",
    "    \"\"\"\n",
    "    Testing utility.\n",
    "    \"\"\"\n",
    "    n_classes = lowprofool.n_classes\n",
    "    \n",
    "    # Generate targets\n",
    "    target = np.eye(n_classes)[np.array(\n",
    "        y_valid.apply(\n",
    "            lambda x: np.random.choice([i for i in range(n_classes) if i != x]))\n",
    "    )]\n",
    "    \n",
    "    # Generate adversaries\n",
    "    adversaries = lowprofool.generate(x=x_valid, y=target)\n",
    "\n",
    "    # Test - check the success rate\n",
    "    expected = np.argmax(target, axis=1)\n",
    "    x = Variable(torch.from_numpy(adversaries.astype(np.float32)))\n",
    "    predicted = np.argmax(classifier.forward(x).detach().numpy(), axis=1)\n",
    "    correct = (expected == predicted)\n",
    "    \n",
    "    success_rate = np.sum(correct) / correct.shape[0]\n",
    "    print(\"Success rate: {:.2f}%\".format(100*success_rate))\n",
    "    \n",
    "    return adversaries"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Iris flowers dataset test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success rate: 100.00%\n"
     ]
    }
   ],
   "source": [
    "# Wrapping classifier into appropriate ART-friendly wrapper\n",
    "# (in this case it is PyTorch NN classifier wrapper from ART)\n",
    "neural_network_iris_wrapper = PyTorchClassifier(\n",
    "    model       = nn_model_iris, \n",
    "    loss        = loss_fn,\n",
    "    input_shape = (4,),\n",
    "    nb_classes  = 3,\n",
    "    clip_values = scaled_clip_values_iris\n",
    ")\n",
    "\n",
    "# Creating LowProFool instance\n",
    "lpf_neural_network_iris = LowProFool(\n",
    "    classifier = neural_network_iris_wrapper,\n",
    "    n_steps    = 100,\n",
    "    eta        = 7,\n",
    "    lambd      = 1.75, \n",
    "    eta_decay  = 0.95\n",
    ")\n",
    "\n",
    "# Fitting feature importance\n",
    "lpf_neural_network_iris.fit_importances(X_train_iris, y_train_iris)\n",
    "\n",
    "# Testing\n",
    "results_nn_ir = lowprofool_generate_adversaries_test_nn(\n",
    "    lowprofool = lpf_neural_network_iris,\n",
    "    classifier = nn_model_iris, \n",
    "    x_valid    = X_valid_iris, \n",
    "    y_valid    = y_valid_iris\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Original values ===\n",
      "\n",
      "Features[:4]:\n",
      "    5.5000    3.5000    1.3000    0.2000\n",
      "Prediction (probability -> class):\n",
      "   0.905   0.079   0.016  ->  0\n",
      "\n",
      "Features[:4]:\n",
      "    5.7000    2.8000    4.5000    1.3000\n",
      "Prediction (probability -> class):\n",
      "   0.111   0.609   0.280  ->  1\n",
      "\n",
      "Features[:4]:\n",
      "    5.1000    3.8000    1.9000    0.4000\n",
      "Prediction (probability -> class):\n",
      "   0.926   0.060   0.015  ->  0\n",
      "\n",
      "\n",
      "=== Adversaries (LowProFool results) ===\n",
      "\n",
      "Features[:4]:\n",
      "    6.0665    2.0000    2.7633    0.5005\n",
      "Prediction (probability -> class):\n",
      "   0.143   0.752   0.105  ->  1\n",
      "\n",
      "Features[:4]:\n",
      "    4.7821    3.9589    3.2572    0.7321\n",
      "Prediction (probability -> class):\n",
      "   0.890   0.088   0.022  ->  0\n",
      "\n",
      "Features[:4]:\n",
      "    5.6058    2.0000    3.1666    0.6729\n",
      "Prediction (probability -> class):\n",
      "   0.123   0.758   0.118  ->  1\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"=== Original values ===\\n\")\n",
    "\n",
    "print_predictions(iris_scaler.inverse_transform(X_valid_iris[:3].values),\n",
    "      neural_network_iris_wrapper.predict(X_valid_iris[:3].values.astype(np.float32)))\n",
    "\n",
    "print(\"\\n=== Adversaries (LowProFool results) ===\\n\")\n",
    "\n",
    "print_predictions(iris_scaler.inverse_transform(results_nn_ir[:3]), \n",
    "      neural_network_iris_wrapper.predict(results_nn_ir.astype(np.float32)[:3]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Breast cancer dataset test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success rate: 98.25%\n"
     ]
    }
   ],
   "source": [
    "# Wrapping classifier into appropriate ART-friendly wrapper\n",
    "# (in this case it is PyTorch NN classifier wrapper from ART)\n",
    "neural_network_cancer_wrapper = PyTorchClassifier(\n",
    "    model       = nn_model_cancer, \n",
    "    loss        = loss_fn, \n",
    "    input_shape = (30,),\n",
    "    nb_classes  = 2,\n",
    "    clip_values = scaled_clip_values_cancer\n",
    ")\n",
    "\n",
    "# Creating LowProFool instance\n",
    "lpf_neural_network_cancer = LowProFool(\n",
    "    classifier = neural_network_cancer_wrapper,\n",
    "    n_steps    = 200,\n",
    "    eta        = 10,\n",
    "    lambd      = 2, \n",
    "    eta_decay  = 0.99\n",
    ")\n",
    "\n",
    "# Fitting feature importance\n",
    "lpf_neural_network_cancer.fit_importances(X_train_cancer, y_train_cancer)\n",
    "\n",
    "# Testing\n",
    "results_nn_bc = lowprofool_generate_adversaries_test_nn(\n",
    "    lowprofool = lpf_neural_network_cancer,\n",
    "    classifier = nn_model_cancer, \n",
    "    x_valid    = X_valid_cancer, \n",
    "    y_valid    = y_valid_cancer\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Original values ===\n",
      "\n",
      "Features[:30]:\n",
      "   12.8700   19.5400   82.6700  509.2000    0.0914    0.0788\n",
      "    0.0180    0.0209    0.1861    0.0635    0.3665    0.7693\n",
      "    2.5970   26.5000    0.0059    0.0136    0.0071    0.0065\n",
      "    0.0222    0.0024   14.4500   24.3800   95.1400  626.9000\n",
      "    0.1214    0.1652    0.0713    0.0638    0.3313    0.0774\n",
      "Prediction (probability -> class):\n",
      "   0.011   0.989  ->  1\n",
      "\n",
      "Features[:30]:\n",
      "   11.8100   17.3900   75.2700  428.9000    0.1007    0.0556\n",
      "    0.0235    0.0155    0.1718    0.0578    0.1859    1.9260\n",
      "    1.0110   14.4700    0.0078    0.0088    0.0156    0.0062\n",
      "    0.0314    0.0020   12.5700   26.4800   79.5700  489.5000\n",
      "    0.1356    0.1000    0.0880    0.0431    0.3200    0.0658\n",
      "Prediction (probability -> class):\n",
      "   0.003   0.997  ->  1\n",
      "\n",
      "\n",
      "=== Adversaries (LowProFool results) ===\n",
      "\n",
      "Features[:30]:\n",
      "   13.4231   23.0837   93.0995  683.2430    0.0904    0.0758\n",
      "    0.0521    0.0349    0.2020    0.0557    0.5099    0.6657\n",
      "    3.6778   51.7879    0.0040    0.0076    0.0017    0.0068\n",
      "    0.0123    0.0012   16.9088   30.4553  108.6628  934.3693\n",
      "    0.1479    0.2470    0.1955    0.0923    0.3519    0.0993\n",
      "Prediction (probability -> class):\n",
      "   0.976   0.024  ->  0\n",
      "\n",
      "Features[:30]:\n",
      "   13.7101   18.5734   87.2021  649.0093    0.1026    0.0516\n",
      "    0.0625    0.0402    0.1758    0.0557    0.2958    1.7680\n",
      "    2.4153   46.3015    0.0077    0.0076    0.0102    0.0059\n",
      "    0.0277    0.0012   14.7154   29.4682  100.6762 1091.2357\n",
      "    0.1552    0.1281    0.1646    0.0957    0.3482    0.0672\n",
      "Prediction (probability -> class):\n",
      "   0.505   0.495  ->  0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"=== Original values ===\\n\")\n",
    "\n",
    "print_predictions(\n",
    "    cancer_scaler.inverse_transform(X_valid_cancer[-2:]),\n",
    "    neural_network_cancer_wrapper.predict(X_valid_cancer[-2:].values.astype(np.float32)),\n",
    "    max_features=30\n",
    ")\n",
    "\n",
    "print(\"\\n=== Adversaries (LowProFool results) ===\\n\")\n",
    "\n",
    "print_predictions(\n",
    "    cancer_scaler.inverse_transform(results_nn_bc[-2:]), \n",
    "    neural_network_cancer_wrapper.predict(results_nn_bc.astype(np.float32)[-2:]),\n",
    "    max_features=30\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('art-aml': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "vscode": {
   "interpreter": {
    "hash": "c317be1f25458f4932790ed219f1158e57c12adaecb215f67c10dfd49657577c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}