Run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "from linear_cca import linear_cca\n",
    "from torch.utils.data import BatchSampler, SequentialSampler\n",
    "from DeepCCAModels import DeepCCA\n",
    "from main import Solver\n",
    "from utils import load_data, svm_classify\n",
    "try:\n",
    "    import cPickle as thepickle\n",
    "except ImportError:\n",
    "    import _pickle as thepickle\n",
    "\n",
    "import gzip\n",
    "import numpy as np\n",
    "torch.set_default_tensor_type(torch.DoubleTensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 2 GPUs\n",
      "loading data ...\n",
      "loading data ...\n"
     ]
    }
   ],
   "source": [
    "############\n",
    "# Parameters Section\n",
    "\n",
    "device = torch.device('cuda')\n",
    "print(\"Using\", torch.cuda.device_count(), \"GPUs\")\n",
    "\n",
    "# the path to save the final learned features\n",
    "save_to = './new_features.gz'\n",
    "\n",
    "# the size of the new space learned by the model (number of the new features)\n",
    "outdim_size = 10\n",
    "\n",
    "# size of the input for view 1 and view 2\n",
    "input_shape1 = 784\n",
    "input_shape2 = 784\n",
    "\n",
    "# number of layers with nodes in each one\n",
    "layer_sizes1 = [1024, 1024, 1024, outdim_size]\n",
    "layer_sizes2 = [1024, 1024, 1024, outdim_size]\n",
    "\n",
    "# the parameters for training the network\n",
    "learning_rate = 1e-3\n",
    "epoch_num = 10\n",
    "batch_size = 800\n",
    "\n",
    "# the regularization parameter of the network\n",
    "# seems necessary to avoid the gradient exploding especially when non-saturating activations are used\n",
    "reg_par = 1e-5\n",
    "\n",
    "# specifies if all the singular values should get used to calculate the correlation or just the top outdim_size ones\n",
    "# if one option does not work for a network or dataset, try the other one\n",
    "use_all_singular_values = False\n",
    "\n",
    "# if a linear CCA should get applied on the learned features extracted from the networks\n",
    "# it does not affect the performance on noisy MNIST significantly\n",
    "apply_linear_cca = True\n",
    "# end of parameters section\n",
    "############\n",
    "\n",
    "# Each view is stored in a gzip file separately. They will get downloaded the first time the code gets executed.\n",
    "# Datasets get stored under the datasets folder of user's Keras folder\n",
    "# normally under [Home Folder]/.keras/datasets/\n",
    "data1 = load_data('./noisymnist_view1.gz')\n",
    "data2 = load_data('./noisymnist_view2.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[ INFO : 2020-04-12 12:49:38,131 ] - DataParallel(\n",
      "  (module): DeepCCA(\n",
      "    (model1): MlpNet(\n",
      "      (layers): ModuleList(\n",
      "        (0): Sequential(\n",
      "          (0): Linear(in_features=784, out_features=1024, bias=True)\n",
      "          (1): Sigmoid()\n",
      "          (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "        )\n",
      "        (1): Sequential(\n",
      "          (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (1): Sigmoid()\n",
      "          (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "        )\n",
      "        (2): Sequential(\n",
      "          (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (1): Sigmoid()\n",
      "          (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "        )\n",
      "        (3): Sequential(\n",
      "          (0): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "          (1): Linear(in_features=1024, out_features=10, bias=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (model2): MlpNet(\n",
      "      (layers): ModuleList(\n",
      "        (0): Sequential(\n",
      "          (0): Linear(in_features=784, out_features=1024, bias=True)\n",
      "          (1): Sigmoid()\n",
      "          (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "        )\n",
      "        (1): Sequential(\n",
      "          (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (1): Sigmoid()\n",
      "          (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "        )\n",
      "        (2): Sequential(\n",
      "          (0): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (1): Sigmoid()\n",
      "          (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "        )\n",
      "        (3): Sequential(\n",
      "          (0): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=False, track_running_stats=True)\n",
      "          (1): Linear(in_features=1024, out_features=10, bias=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "[ INFO : 2020-04-12 12:49:38,132 ] - RMSprop (\n",
      "Parameter Group 0\n",
      "    alpha: 0.99\n",
      "    centered: False\n",
      "    eps: 1e-08\n",
      "    lr: 0.001\n",
      "    momentum: 0\n",
      "    weight_decay: 1e-05\n",
      ")\n",
      "[ INFO : 2020-04-12 12:49:45,119 ] - Epoch 1: val_loss improved from 0.0000 to -5.6908, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:45,179 ] - Epoch 1/10 - time: 6.92 - training_loss: -4.9530 - val_loss: -5.6908\n",
      "[ INFO : 2020-04-12 12:49:47,192 ] - Epoch 2: val_loss improved from -5.6908 to -6.2831, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:47,440 ] - Epoch 2/10 - time: 2.26 - training_loss: -5.5974 - val_loss: -6.2831\n",
      "[ INFO : 2020-04-12 12:49:49,578 ] - Epoch 3: val_loss improved from -6.2831 to -6.5218, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:49,820 ] - Epoch 3/10 - time: 2.38 - training_loss: -5.9621 - val_loss: -6.5218\n",
      "[ INFO : 2020-04-12 12:49:51,781 ] - Epoch 4: val_loss improved from -6.5218 to -7.2041, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:52,087 ] - Epoch 4/10 - time: 2.27 - training_loss: -6.3012 - val_loss: -7.2041\n",
      "[ INFO : 2020-04-12 12:49:54,100 ] - Epoch 5: val_loss improved from -7.2041 to -7.2828, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:54,349 ] - Epoch 5/10 - time: 2.26 - training_loss: -6.5927 - val_loss: -7.2828\n",
      "[ INFO : 2020-04-12 12:49:56,401 ] - Epoch 6: val_loss improved from -7.2828 to -7.5330, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:56,668 ] - Epoch 6/10 - time: 2.32 - training_loss: -6.8270 - val_loss: -7.5330\n",
      "[ INFO : 2020-04-12 12:49:58,672 ] - Epoch 7: val_loss improved from -7.5330 to -7.6615, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:49:58,942 ] - Epoch 7/10 - time: 2.27 - training_loss: -7.0269 - val_loss: -7.6615\n",
      "[ INFO : 2020-04-12 12:50:01,009 ] - Epoch 8: val_loss improved from -7.6615 to -7.6812, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:50:01,257 ] - Epoch 8/10 - time: 2.31 - training_loss: -7.2024 - val_loss: -7.6812\n",
      "[ INFO : 2020-04-12 12:50:03,238 ] - Epoch 9: val_loss improved from -7.6812 to -7.8300, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:50:03,492 ] - Epoch 9/10 - time: 2.23 - training_loss: -7.3619 - val_loss: -7.8300\n",
      "[ INFO : 2020-04-12 12:50:05,492 ] - Epoch 10: val_loss improved from -7.8300 to -7.8474, saving model to checkpoint.model\n",
      "[ INFO : 2020-04-12 12:50:05,773 ] - Epoch 10/10 - time: 2.28 - training_loss: -7.5112 - val_loss: -7.8474\n",
      "[ INFO : 2020-04-12 12:50:06,950 ] - loss on validation data: -7.8474\n",
      "[ INFO : 2020-04-12 12:50:07,140 ] - loss on test data: -7.7955\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear CCA started!\n"
     ]
    }
   ],
   "source": [
    "# Building, training, and producing the new features by DCCA\n",
    "model = DeepCCA(layer_sizes1, layer_sizes2, input_shape1,\n",
    "                input_shape2, outdim_size, use_all_singular_values, device=device).double()\n",
    "l_cca = None\n",
    "if apply_linear_cca:\n",
    "    l_cca = linear_cca()\n",
    "solver = Solver(model, l_cca, outdim_size, epoch_num, batch_size,\n",
    "                learning_rate, reg_par, device=device)\n",
    "train1, train2 = data1[0][0], data2[0][0]\n",
    "val1, val2 = data1[1][0], data2[1][0]\n",
    "test1, test2 = data1[2][0], data2[2][0]\n",
    "# val1=None\n",
    "# test1=None\n",
    "solver.fit(train1, train2, val1, val2, test1, test2)\n",
    "# TODO: Save linear_cca model if needed\n",
    "\n",
    "set_size = [0, train1.size(0), train1.size(\n",
    "    0) + val1.size(0), train1.size(0) + val1.size(0) + test1.size(0)]\n",
    "loss, outputs = solver.test(torch.cat([train1, val1, test1], dim=0), torch.cat(\n",
    "    [train2, val2, test2], dim=0), apply_linear_cca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training SVM...\n",
      "Accuracy on view 1 (validation data) is: 95.61\n",
      "Accuracy on view 1 (test data) is: 95.1\n"
     ]
    }
   ],
   "source": [
    "new_data = []\n",
    "# print(outputs)\n",
    "for idx in range(3):\n",
    "    new_data.append([outputs[0][set_size[idx]:set_size[idx + 1], :],\n",
    "                     outputs[1][set_size[idx]:set_size[idx + 1], :], data1[idx][1]])\n",
    "# Training and testing of SVM with linear kernel on the view 1 with new features\n",
    "[test_acc, valid_acc] = svm_classify(new_data, C=0.01)\n",
    "print(\"Accuracy on view 1 (validation data) is:\", valid_acc * 100.0)\n",
    "print(\"Accuracy on view 1 (test data) is:\", test_acc*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving new features ...\n"
     ]
    }
   ],
   "source": [
    "# Saving new features in a gzip pickled file specified by save_to\n",
    "print('saving new features ...')\n",
    "f1 = gzip.open(save_to, 'wb')\n",
    "thepickle.dump(new_data, f1)\n",
    "f1.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<generator object Module.parameters at 0x7fc3f8183050>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = torch.load('checkpoint.model')\n",
    "solver.model.load_state_dict(d)\n",
    "solver.model.parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}