Neural_networks_hyperparameter_tuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/omarmoh26/BE-Alert/blob/main/Neural_networks_hyperparameter_tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ByvMxbT2M0H_",
        "outputId": "0d3d9c20-213e-4b32-a2ef-29898946726a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive/\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "y8V2VBkU06Aj"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "import seaborn as sns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "uClP9R9aNbQp"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv('/content/gdrive/My Drive/R6_EEG_Final_Columns.csv')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "wiSiroA7Nm_T",
        "outputId": "8789993d-9ccc-4302-9c43-4f6b1cca1be0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Theta_mean  Theta_median   Theta_mode  Theta_range    Theta_std  \\\n",
              "0   124965.5903       81382.5  124965.5903      1029881  163202.8949   \n",
              "1   161064.2286      109448.0  161064.2286       826196  165275.3640   \n",
              "2   149816.8732       96455.0  149816.8732       957235  166569.0373   \n",
              "3   162653.3607       64971.0  162653.3607      1696851  260393.0996   \n",
              "4   126893.9483       45817.5  126893.9483      1009030  181363.0166   \n",
              "..          ...           ...          ...          ...          ...   \n",
              "85  282487.5122      176070.0  282487.5122      1324433  284497.0738   \n",
              "86  193226.8707       93990.0  193226.8707      1591630  255014.6299   \n",
              "87  137478.2679       74496.0  137478.2679      1811067  224541.4304   \n",
              "88  198926.0484      134007.5  198926.0484      1808890  229844.4600   \n",
              "89  162498.6803       68510.0  162498.6803      1356267  238339.2248   \n",
              "\n",
              "       Theta_var  Theta_IQR  Alpha1_mean  Alpha1_median  Alpha1_mode  ...  \\\n",
              "0   2.663518e+10  113547.50  36693.70139        13622.0  36693.70139  ...   \n",
              "1   2.731595e+10  159941.75  34918.02857        23867.5  34918.02857  ...   \n",
              "2   2.774524e+10  157763.25  30493.87324        18521.0  30493.87324  ...   \n",
              "3   6.780457e+10  142617.25  33367.27869        15899.5  33367.27869  ...   \n",
              "4   3.289254e+10  107785.75  23113.84483        14845.5  23113.84483  ...   \n",
              "..           ...        ...          ...            ...          ...  ...   \n",
              "85  8.093859e+10  332013.00  70498.08943        35332.0  70498.08943  ...   \n",
              "86  6.503246e+10  199486.50  74629.12931        29970.5  74629.12931  ...   \n",
              "87  5.041885e+10   94241.50  37676.38393        16918.5  37676.38393  ...   \n",
              "88  5.282848e+10  191991.75  42279.81452        21442.0  42279.81452  ...   \n",
              "89  5.680559e+10  132674.75  42521.19672        21412.0  42521.19672  ...   \n",
              "\n",
              "      Alpha1_std    Alpha1_var  Alpha1_IQR  Alpha2_mean  Alpha2_median  \\\n",
              "0    63481.77426  4.029936e+09    38614.75  25875.29861        14065.5   \n",
              "1    39782.07114  1.582613e+09    42486.00  25078.93571        16974.5   \n",
              "2    48986.98127  2.399724e+09    27804.50  21667.59155        13002.0   \n",
              "3    46917.19719  2.201223e+09    23909.75  26281.50000        12748.5   \n",
              "4    26936.61478  7.255812e+08    21237.00  17017.05172        10775.5   \n",
              "..           ...           ...         ...          ...            ...   \n",
              "85  117544.85050  1.381679e+10    60303.00  40760.39837        23387.0   \n",
              "86  135330.53550  1.831435e+10    65976.50  44065.68103        23556.0   \n",
              "87   59632.28771  3.556010e+09    27377.75  28570.58036        18242.5   \n",
              "88   56829.44188  3.229585e+09    40276.50  25929.18548        14637.5   \n",
              "89   56449.57407  3.186554e+09    38849.25  37859.87705        19257.0   \n",
              "\n",
              "    Alpha2_mode  Alpha2_range   Alpha2_std    Alpha2_var  Alpha2_IQR  \n",
              "0   25875.29861        233815  36455.75323  1.329022e+09    24251.75  \n",
              "1   25078.93571        221456  29358.05140  8.618952e+08    26703.00  \n",
              "2   21667.59155        122532  22420.34038  5.026717e+08    23148.25  \n",
              "3   26281.50000        235114  39171.09175  1.534374e+09    24834.50  \n",
              "4   17017.05172         90226  18215.48147  3.318038e+08    17462.25  \n",
              "..          ...           ...          ...           ...         ...  \n",
              "85  40760.39837        396568  59298.73607  3.516340e+09    35767.00  \n",
              "86  44065.68103        278036  54258.10917  2.943942e+09    43450.50  \n",
              "87  28570.58036        180749  33487.30808  1.121400e+09    24441.00  \n",
              "88  25929.18548        218985  30583.85946  9.353725e+08    31127.50  \n",
              "89  37859.87705        336646  50897.94364  2.590601e+09    29939.75  \n",
              "\n",
              "[90 rows x 21 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-945ad7e7-0677-46e4-ac3b-40d071c9cba5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Theta_mean</th>\n",
              "      <th>Theta_median</th>\n",
              "      <th>Theta_mode</th>\n",
              "      <th>Theta_range</th>\n",
              "      <th>Theta_std</th>\n",
              "      <th>Theta_var</th>\n",
              "      <th>Theta_IQR</th>\n",
              "      <th>Alpha1_mean</th>\n",
              "      <th>Alpha1_median</th>\n",
              "      <th>Alpha1_mode</th>\n",
              "      <th>...</th>\n",
              "      <th>Alpha1_std</th>\n",
              "      <th>Alpha1_var</th>\n",
              "      <th>Alpha1_IQR</th>\n",
              "      <th>Alpha2_mean</th>\n",
              "      <th>Alpha2_median</th>\n",
              "      <th>Alpha2_mode</th>\n",
              "      <th>Alpha2_range</th>\n",
              "      <th>Alpha2_std</th>\n",
              "      <th>Alpha2_var</th>\n",
              "      <th>Alpha2_IQR</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>124965.5903</td>\n",
              "      <td>81382.5</td>\n",
              "      <td>124965.5903</td>\n",
              "      <td>1029881</td>\n",
              "      <td>163202.8949</td>\n",
              "      <td>2.663518e+10</td>\n",
              "      <td>113547.50</td>\n",
              "      <td>36693.70139</td>\n",
              "      <td>13622.0</td>\n",
              "      <td>36693.70139</td>\n",
              "      <td>...</td>\n",
              "      <td>63481.77426</td>\n",
              "      <td>4.029936e+09</td>\n",
              "      <td>38614.75</td>\n",
              "      <td>25875.29861</td>\n",
              "      <td>14065.5</td>\n",
              "      <td>25875.29861</td>\n",
              "      <td>233815</td>\n",
              "      <td>36455.75323</td>\n",
              "      <td>1.329022e+09</td>\n",
              "      <td>24251.75</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>161064.2286</td>\n",
              "      <td>109448.0</td>\n",
              "      <td>161064.2286</td>\n",
              "      <td>826196</td>\n",
              "      <td>165275.3640</td>\n",
              "      <td>2.731595e+10</td>\n",
              "      <td>159941.75</td>\n",
              "      <td>34918.02857</td>\n",
              "      <td>23867.5</td>\n",
              "      <td>34918.02857</td>\n",
              "      <td>...</td>\n",
              "      <td>39782.07114</td>\n",
              "      <td>1.582613e+09</td>\n",
              "      <td>42486.00</td>\n",
              "      <td>25078.93571</td>\n",
              "      <td>16974.5</td>\n",
              "      <td>25078.93571</td>\n",
              "      <td>221456</td>\n",
              "      <td>29358.05140</td>\n",
              "      <td>8.618952e+08</td>\n",
              "      <td>26703.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>149816.8732</td>\n",
              "      <td>96455.0</td>\n",
              "      <td>149816.8732</td>\n",
              "      <td>957235</td>\n",
              "      <td>166569.0373</td>\n",
              "      <td>2.774524e+10</td>\n",
              "      <td>157763.25</td>\n",
              "      <td>30493.87324</td>\n",
              "      <td>18521.0</td>\n",
              "      <td>30493.87324</td>\n",
              "      <td>...</td>\n",
              "      <td>48986.98127</td>\n",
              "      <td>2.399724e+09</td>\n",
              "      <td>27804.50</td>\n",
              "      <td>21667.59155</td>\n",
              "      <td>13002.0</td>\n",
              "      <td>21667.59155</td>\n",
              "      <td>122532</td>\n",
              "      <td>22420.34038</td>\n",
              "      <td>5.026717e+08</td>\n",
              "      <td>23148.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>162653.3607</td>\n",
              "      <td>64971.0</td>\n",
              "      <td>162653.3607</td>\n",
              "      <td>1696851</td>\n",
              "      <td>260393.0996</td>\n",
              "      <td>6.780457e+10</td>\n",
              "      <td>142617.25</td>\n",
              "      <td>33367.27869</td>\n",
              "      <td>15899.5</td>\n",
              "      <td>33367.27869</td>\n",
              "      <td>...</td>\n",
              "      <td>46917.19719</td>\n",
              "      <td>2.201223e+09</td>\n",
              "      <td>23909.75</td>\n",
              "      <td>26281.50000</td>\n",
              "      <td>12748.5</td>\n",
              "      <td>26281.50000</td>\n",
              "      <td>235114</td>\n",
              "      <td>39171.09175</td>\n",
              "      <td>1.534374e+09</td>\n",
              "      <td>24834.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>126893.9483</td>\n",
              "      <td>45817.5</td>\n",
              "      <td>126893.9483</td>\n",
              "      <td>1009030</td>\n",
              "      <td>181363.0166</td>\n",
              "      <td>3.289254e+10</td>\n",
              "      <td>107785.75</td>\n",
              "      <td>23113.84483</td>\n",
              "      <td>14845.5</td>\n",
              "      <td>23113.84483</td>\n",
              "      <td>...</td>\n",
              "      <td>26936.61478</td>\n",
              "      <td>7.255812e+08</td>\n",
              "      <td>21237.00</td>\n",
              "      <td>17017.05172</td>\n",
              "      <td>10775.5</td>\n",
              "      <td>17017.05172</td>\n",
              "      <td>90226</td>\n",
              "      <td>18215.48147</td>\n",
              "      <td>3.318038e+08</td>\n",
              "      <td>17462.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>282487.5122</td>\n",
              "      <td>176070.0</td>\n",
              "      <td>282487.5122</td>\n",
              "      <td>1324433</td>\n",
              "      <td>284497.0738</td>\n",
              "      <td>8.093859e+10</td>\n",
              "      <td>332013.00</td>\n",
              "      <td>70498.08943</td>\n",
              "      <td>35332.0</td>\n",
              "      <td>70498.08943</td>\n",
              "      <td>...</td>\n",
              "      <td>117544.85050</td>\n",
              "      <td>1.381679e+10</td>\n",
              "      <td>60303.00</td>\n",
              "      <td>40760.39837</td>\n",
              "      <td>23387.0</td>\n",
              "      <td>40760.39837</td>\n",
              "      <td>396568</td>\n",
              "      <td>59298.73607</td>\n",
              "      <td>3.516340e+09</td>\n",
              "      <td>35767.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>193226.8707</td>\n",
              "      <td>93990.0</td>\n",
              "      <td>193226.8707</td>\n",
              "      <td>1591630</td>\n",
              "      <td>255014.6299</td>\n",
              "      <td>6.503246e+10</td>\n",
              "      <td>199486.50</td>\n",
              "      <td>74629.12931</td>\n",
              "      <td>29970.5</td>\n",
              "      <td>74629.12931</td>\n",
              "      <td>...</td>\n",
              "      <td>135330.53550</td>\n",
              "      <td>1.831435e+10</td>\n",
              "      <td>65976.50</td>\n",
              "      <td>44065.68103</td>\n",
              "      <td>23556.0</td>\n",
              "      <td>44065.68103</td>\n",
              "      <td>278036</td>\n",
              "      <td>54258.10917</td>\n",
              "      <td>2.943942e+09</td>\n",
              "      <td>43450.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>137478.2679</td>\n",
              "      <td>74496.0</td>\n",
              "      <td>137478.2679</td>\n",
              "      <td>1811067</td>\n",
              "      <td>224541.4304</td>\n",
              "      <td>5.041885e+10</td>\n",
              "      <td>94241.50</td>\n",
              "      <td>37676.38393</td>\n",
              "      <td>16918.5</td>\n",
              "      <td>37676.38393</td>\n",
              "      <td>...</td>\n",
              "      <td>59632.28771</td>\n",
              "      <td>3.556010e+09</td>\n",
              "      <td>27377.75</td>\n",
              "      <td>28570.58036</td>\n",
              "      <td>18242.5</td>\n",
              "      <td>28570.58036</td>\n",
              "      <td>180749</td>\n",
              "      <td>33487.30808</td>\n",
              "      <td>1.121400e+09</td>\n",
              "      <td>24441.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>198926.0484</td>\n",
              "      <td>134007.5</td>\n",
              "      <td>198926.0484</td>\n",
              "      <td>1808890</td>\n",
              "      <td>229844.4600</td>\n",
              "      <td>5.282848e+10</td>\n",
              "      <td>191991.75</td>\n",
              "      <td>42279.81452</td>\n",
              "      <td>21442.0</td>\n",
              "      <td>42279.81452</td>\n",
              "      <td>...</td>\n",
              "      <td>56829.44188</td>\n",
              "      <td>3.229585e+09</td>\n",
              "      <td>40276.50</td>\n",
              "      <td>25929.18548</td>\n",
              "      <td>14637.5</td>\n",
              "      <td>25929.18548</td>\n",
              "      <td>218985</td>\n",
              "      <td>30583.85946</td>\n",
              "      <td>9.353725e+08</td>\n",
              "      <td>31127.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>89</th>\n",
              "      <td>162498.6803</td>\n",
              "      <td>68510.0</td>\n",
              "      <td>162498.6803</td>\n",
              "      <td>1356267</td>\n",
              "      <td>238339.2248</td>\n",
              "      <td>5.680559e+10</td>\n",
              "      <td>132674.75</td>\n",
              "      <td>42521.19672</td>\n",
              "      <td>21412.0</td>\n",
              "      <td>42521.19672</td>\n",
              "      <td>...</td>\n",
              "      <td>56449.57407</td>\n",
              "      <td>3.186554e+09</td>\n",
              "      <td>38849.25</td>\n",
              "      <td>37859.87705</td>\n",
              "      <td>19257.0</td>\n",
              "      <td>37859.87705</td>\n",
              "      <td>336646</td>\n",
              "      <td>50897.94364</td>\n",
              "      <td>2.590601e+09</td>\n",
              "      <td>29939.75</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>90 rows × 21 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-945ad7e7-0677-46e4-ac3b-40d071c9cba5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-945ad7e7-0677-46e4-ac3b-40d071c9cba5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-945ad7e7-0677-46e4-ac3b-40d071c9cba5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "y=data.pop('Result')\n",
        "data=data.drop(columns=['Attention_median'])\n",
        "data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "oYlLoAbPS9XL"
      },
      "outputs": [],
      "source": [
        "myColumns=data[['Theta_median','Alpha1_IQR','Alpha2_median']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "IsSPXDj1R85o"
      },
      "outputs": [],
      "source": [
        "scaler = StandardScaler()\n",
        "X=scaler.fit_transform(myColumns)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "l1Nh87BBewW6"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn import preprocessing\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense\n",
        "from scikeras.wrappers import KerasClassifier\n",
        "from tensorflow.keras.constraints import MaxNorm\n",
        "from tensorflow.keras.layers import Dropout\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "z5lVU6ZQK0EE"
      },
      "outputs": [],
      "source": [
        "x_train, x_test, y_train, y_test=train_test_split(X,y,test_size=0.3,stratify=y,random_state=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7AptCOow7HZC",
        "outputId": "76df6afb-3c2d-46b9-8f3b-dd2afa104776"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting scikeras\n",
            "  Downloading scikeras-0.10.0-py3-none-any.whl (27 kB)\n",
            "Requirement already satisfied: scikit-learn>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from scikeras) (1.0.2)\n",
            "Requirement already satisfied: packaging>=0.21 in /usr/local/lib/python3.8/dist-packages (from scikeras) (21.3)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=0.21->scikeras) (3.0.9)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=1.0.0->scikeras) (1.2.0)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=1.0.0->scikeras) (1.7.3)\n",
            "Requirement already satisfied: numpy>=1.14.6 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=1.0.0->scikeras) (1.21.6)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=1.0.0->scikeras) (3.1.0)\n",
            "Installing collected packages: scikeras\n",
            "Successfully installed scikeras-0.10.0\n"
          ]
        }
      ],
      "source": [
        "try:\n",
        "    import scikeras\n",
        "except ImportError:\n",
        "    !python -m pip install scikeras"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "81pX3UHm9Ays",
        "outputId": "0a296178-7290-4a30-9910-7333bb317077"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.746032 using {'model__activation': 'linear'}\n",
            "0.603175 (0.059391) with: {'model__activation': 'softmax'}\n",
            "0.698413 (0.022448) with: {'model__activation': 'softplus'}\n",
            "0.714286 (0.102869) with: {'model__activation': 'softsign'}\n",
            "0.714286 (0.102869) with: {'model__activation': 'relu'}\n",
            "0.714286 (0.102869) with: {'model__activation': 'tanh'}\n",
            "0.634921 (0.059391) with: {'model__activation': 'sigmoid'}\n",
            "0.603175 (0.059391) with: {'model__activation': 'hard_sigmoid'}\n",
            "0.746032 (0.080937) with: {'model__activation': 'linear'}\n"
          ]
        }
      ],
      "source": [
        "def create_model(activation='relu'):\n",
        "\t# create model\n",
        "\tmodel = Sequential()\n",
        "\tmodel.add(Dense(12, input_shape=(3,), kernel_initializer='uniform', activation=activation))\n",
        "\tmodel.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))\n",
        "\treturn model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, epochs=100, batch_size=10, verbose=0, metrics=['accuracy'],loss='binary_crossentropy',random_state=0)\n",
        "# define the grid search parameters\n",
        "activation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']\n",
        "param_grid = dict(model__activation=activation)\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train,y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s7o_PbxB8Mho",
        "outputId": "66531f22-ec34-4f2f-f741-bd6f269fa087"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.682540 using {'optimizer__learning_rate': 0.01}\n",
            "0.428571 (0.116642) with: {'optimizer__learning_rate': 0.001}\n",
            "0.682540 (0.080937) with: {'optimizer__learning_rate': 0.01}\n",
            "0.682540 (0.059391) with: {'optimizer__learning_rate': 0.1}\n",
            "0.666667 (0.038881) with: {'optimizer__learning_rate': 0.2}\n",
            "0.666667 (0.038881) with: {'optimizer__learning_rate': 0.3}\n"
          ]
        }
      ],
      "source": [
        "def create_model():\n",
        " # create model\n",
        " model = Sequential()\n",
        " model.add(Dense(12, input_shape=(3,), activation='linear'))\n",
        " model.add(Dense(1, activation='sigmoid'))\n",
        " return model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, loss=\"binary_crossentropy\", optimizer=\"SGD\", epochs=100, verbose=0, metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "learn_rate = [0.001, 0.01, 0.1, 0.2, 0.3]\n",
        "param_grid = dict(optimizer__learning_rate=learn_rate)\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train,y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8WdUWhCO73Fm",
        "outputId": "c9e878eb-2da8-4907-f453-805241ffa038"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.746032 using {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01}\n",
            "0.746032 (0.022448) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01}\n",
            "0.666667 (0.067344) with: {'optimizer': 'RMSprop', 'optimizer__learning_rate': 0.01}\n",
            "0.714286 (0.067344) with: {'optimizer': 'Adagrad', 'optimizer__learning_rate': 0.01}\n",
            "0.412698 (0.059391) with: {'optimizer': 'Adadelta', 'optimizer__learning_rate': 0.01}\n",
            "0.666667 (0.067344) with: {'optimizer': 'Adam', 'optimizer__learning_rate': 0.01}\n",
            "0.682540 (0.080937) with: {'optimizer': 'Adamax', 'optimizer__learning_rate': 0.01}\n",
            "0.682540 (0.080937) with: {'optimizer': 'Nadam', 'optimizer__learning_rate': 0.01}\n"
          ]
        }
      ],
      "source": [
        "\n",
        "# Function to create model, required for KerasClassifier\n",
        "def create_model():\n",
        "\t# create model\n",
        "\tmodel = Sequential()\n",
        "\tmodel.add(Dense(12, input_shape=(3,), activation='linear'))\n",
        "\tmodel.add(Dense(1, activation='sigmoid'))\n",
        "\t# return model without compile\n",
        "\treturn model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, loss=\"binary_crossentropy\", epochs=100, batch_size=10, verbose=0, metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "optimizer = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']\n",
        "learn_rate = [ 0.01]\n",
        "param_grid = dict(optimizer=optimizer,optimizer__learning_rate=learn_rate)\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train, y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# optional if optimizer uses momentum\n",
        "\n",
        "def create_model():\n",
        " # create model\n",
        " model = Sequential()\n",
        " model.add(Dense(12, input_shape=(3,), activation='linear'))\n",
        " model.add(Dense(1, activation='sigmoid'))\n",
        " return model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, loss=\"binary_crossentropy\", optimizer=\"SGD\", epochs=100, verbose=0, metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "optimizer = ['SGD']\n",
        "momentum = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9]\n",
        "learn_rate = [ 0.01]\n",
        "param_grid = dict(optimizer=optimizer,optimizer__learning_rate=learn_rate, optimizer__momentum=momentum)\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train,y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7CPmC1U4Xmnc",
        "outputId": "2d8e04ed-25bf-4219-92a8-2ebd51970560"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.730159 using {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.682540 (0.080937) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.0}\n",
            "0.698413 (0.097848) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.2}\n",
            "0.714286 (0.067344) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.4}\n",
            "0.730159 (0.044896) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.714286 (0.038881) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.8}\n",
            "0.698413 (0.059391) with: {'optimizer': 'SGD', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.9}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LM0rXKeG9Rsq",
        "outputId": "20d5bdeb-6573-4275-cb03-7cd5334b2f69"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.746032 using {'model__dropout_rate': 0.9, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.634921 (0.022448) with: {'model__dropout_rate': 0.0, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.650794 (0.022448) with: {'model__dropout_rate': 0.1, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.634921 (0.022448) with: {'model__dropout_rate': 0.2, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.666667 (0.038881) with: {'model__dropout_rate': 0.3, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.698413 (0.080937) with: {'model__dropout_rate': 0.4, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.666667 (0.038881) with: {'model__dropout_rate': 0.5, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.698413 (0.059391) with: {'model__dropout_rate': 0.6, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.730159 (0.059391) with: {'model__dropout_rate': 0.7, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.714286 (0.038881) with: {'model__dropout_rate': 0.8, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.080937) with: {'model__dropout_rate': 0.9, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n"
          ]
        }
      ],
      "source": [
        "\n",
        "def create_model(dropout_rate):\n",
        "  # , weight_constraint\n",
        " # create model\n",
        " model = Sequential()\n",
        " model.add(Dense(12, input_shape=(3,), kernel_initializer='uniform', activation='linear'))\n",
        "#  , kernel_constraint=MaxNorm(weight_constraint)\n",
        " model.add(Dropout(dropout_rate))\n",
        " model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))\n",
        " return model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, epochs=100, loss='binary_crossentropy', verbose=0, metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "# weight_constraint = [1.0, 2.0, 3.0, 4.0, 5.0]\n",
        "dropout_rate = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
        "optimizer = ['SGD']\n",
        "momentum = [ 0.6]\n",
        "learn_rate = [ 0.01]\n",
        "param_grid = dict(model__dropout_rate=dropout_rate,optimizer__learning_rate=learn_rate, optimizer__momentum=momentum)\n",
        "# , model__weight_constraint=weight_constraint\n",
        "\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train,y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UAwNhwSOXMtD",
        "outputId": "a36acb6f-1d34-4f9e-8b02-75e6092a492d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.714286 using {'epochs': 10, 'model__dropout_rate': 0.0, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.714286 (0.038881) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.714286 (0.102869) with: {'epochs': 50, 'model__dropout_rate': 0.0, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.634921 (0.022448) with: {'epochs': 100, 'model__dropout_rate': 0.0, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n"
          ]
        }
      ],
      "source": [
        "#  Use scikit-learn to grid search the batch size and epochs\n",
        "\n",
        "# Function to create model, required for KerasClassifier\n",
        "def create_model(dropout_rate):\n",
        " # create model\n",
        " model = Sequential()\n",
        " model.add(Dense(12, input_shape=(3,), activation='linear'))\n",
        " model.add(Dropout(dropout_rate))\n",
        " model.add(Dense(1, activation='sigmoid'))\n",
        " return model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, verbose=0,loss='binary_crossentropy', metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "# batch_size = [10, 20, 40, 60, 80, 100]\n",
        "\n",
        "dropout_rate = [0.0]\n",
        "optimizer = ['SGD']\n",
        "momentum = [ 0.6]\n",
        "learn_rate = [ 0.01]\n",
        "epochs = [10, 50, 100]\n",
        "param_grid = dict( epochs=epochs,model__dropout_rate=dropout_rate,optimizer__learning_rate=learn_rate, optimizer__momentum=momentum)\n",
        "# batch_size=batch_size,\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3,)\n",
        "grid_result = grid.fit(x_train, y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oECNAtOq8tk9",
        "outputId": "747dafca-d6dd-427f-fc54-c03e352e5fbc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.746032 using {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'uniform', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.080937) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'uniform', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.698413 (0.089791) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'lecun_uniform', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.080937) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'normal', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.555556 (0.022448) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'zero', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.698413 (0.044896) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'glorot_normal', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.714286 (0.038881) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'glorot_uniform', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.698413 (0.044896) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'he_normal', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.650794 (0.059391) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__init_mode': 'he_uniform', 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n"
          ]
        }
      ],
      "source": [
        "def create_model(dropout_rate,init_mode='uniform'):\n",
        " # create model\n",
        " model = Sequential()\n",
        " model.add(Dense(12, input_shape=(3,), kernel_initializer=init_mode, activation='linear'))\n",
        " model.add(Dropout(dropout_rate))\n",
        " model.add(Dense(1, kernel_initializer=init_mode, activation='sigmoid'))\n",
        " return model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model,   verbose=0,loss='binary_crossentropy', metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "dropout_rate = [0.0]\n",
        "optimizer = ['SGD']\n",
        "momentum = [ 0.6]\n",
        "learn_rate = [ 0.01]\n",
        "epochs = [10]\n",
        "init_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']\n",
        "param_grid = dict(model__init_mode=init_mode,epochs=epochs,model__dropout_rate=dropout_rate,optimizer__learning_rate=learn_rate, optimizer__momentum=momentum)\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train,y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "7ymQY6PM8HUT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9651671e-4257-46ac-8c63-bfaa140c2f7c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best: 0.746032 using {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 1, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.080937) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 1, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.080937) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 5, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.080937) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 10, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.730159 (0.059391) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 15, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.730159 (0.059391) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 20, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.746032 (0.059391) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 25, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n",
            "0.730159 (0.059391) with: {'epochs': 10, 'model__dropout_rate': 0.0, 'model__neurons': 30, 'optimizer__learning_rate': 0.01, 'optimizer__momentum': 0.6}\n"
          ]
        }
      ],
      "source": [
        "def create_model(neurons,dropout_rate):\n",
        "\t# create model\n",
        "\tmodel = Sequential()\n",
        "\tmodel.add(Dense(neurons, input_shape=(3,), kernel_initializer='uniform', activation='linear'))\n",
        "#  , kernel_constraint=MaxNorm(4)\n",
        "\tmodel.add(Dropout(dropout_rate))\n",
        "\tmodel.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))\n",
        "\treturn model\n",
        "# fix random seed for reproducibility\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, verbose=0,loss='binary_crossentropy', metrics=['accuracy'],random_state=0)\n",
        "# define the grid search parameters\n",
        "neurons = [1, 5, 10, 15, 20, 25, 30]\n",
        "dropout_rate = [0.0]\n",
        "optimizer = ['SGD']\n",
        "momentum = [ 0.6]\n",
        "learn_rate = [ 0.01]\n",
        "epochs = [10]\n",
        "\n",
        "\n",
        "param_grid = dict(model__neurons=neurons,epochs=epochs,model__dropout_rate=dropout_rate,optimizer__learning_rate=learn_rate, optimizer__momentum=momentum)\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train, y_train)\n",
        "# summarize results\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "q-j7wHiZAKU7",
        "outputId": "29b2163e-68b4-4360-b703-ed23749e869b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/joblib/externals/loky/process_executor.py:700: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from tensorflow.keras.constraints import MaxNorm\n",
        "from tensorflow.keras.layers import Dropout\n",
        "def create_model(dropout_rate,neurons,activation='relu',init_mode='uniform'):\n",
        " # create model\n",
        " model = Sequential()\n",
        " model.add(Dense(neurons, input_shape=(3,),kernel_initializer=init_mode, activation=activation))\n",
        " model.add(Dropout(dropout_rate))\n",
        " model.add(Dense(1, kernel_initializer=init_mode, activation='sigmoid'))\n",
        "#  activation=activation\n",
        " # Compile model\n",
        " model.compile(loss='binary_crossentropy',  metrics=['accuracy'])\n",
        " return model\n",
        "\n",
        "seed = 7\n",
        "tf.random.set_seed(seed)\n",
        "\n",
        "model = KerasClassifier(model=create_model, verbose=0)\n",
        "\n",
        "neurons = [1, 5, 10, 15, 20, 25, 30]#8\n",
        "# weight_constraint = [1.0, 2.0, 3.0, 4.0, 5.0] # you may omit it too\n",
        "dropout_rate = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5]#5\n",
        "activation = ['softmax',  'relu', 'tanh', 'sigmoid']#1\n",
        "init_mode = ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform']#7\n",
        "learn_rate = [0.001, 0.01]#2\n",
        "momentum = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9]#4\n",
        "optimizer = ['SGD', 'Adam']#3\n",
        "# batch_size = [10, 20, 40, 60, 80, 100] # omit this hyperparameter\n",
        "epochs = [10, 50, 100]#6\n",
        "\n",
        "param_grid = dict(model__dropout_rate=dropout_rate, model__activation=activation,\n",
        "                  model__init_mode=init_mode,model__neurons=neurons,optimizer__learning_rate=learn_rate, optimizer__momentum=momentum, epochs=epochs,optimizer=optimizer)\n",
        "# batch_size=batch_size,\n",
        "# model__weight_constraint=weight_constraint,\n",
        "\n",
        "\n",
        "\n",
        "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
        "grid_result = grid.fit(x_train,y_train)\n",
        "\n",
        "\n",
        "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}