best_I_C_M_to_O.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ef78f349",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path as osp\n",
    "import GPUtil\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "import os\n",
    "from torch_geometric.data import Dataset, download_url\n",
    "from torch_geometric.loader import DataLoader\n",
    "import pandas as pd\n",
    "from performance import performances_val\n",
    "from PIL import Image\n",
    "from torch.nn import Linear\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from torch_geometric.nn import GCNConv\n",
    "from torch_geometric.nn import global_mean_pool, global_max_pool\n",
    "from torch_geometric.nn.norm import BatchNorm\n",
    "from torch_geometric.data import Data\n",
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from torch_geometric.nn import ChebConv, GraphSAGE,GraphUNet ,TransformerConv\n",
    "\n",
    "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "# os.environ[\"CUDA_LAUNCH_BLOCKING\"] = \"1\"\n",
    "seed = 1378\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "landmark_colors = np.load(\"landmark_colors.npy\")\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "one_hot_list=[]\n",
    "enc = OneHotEncoder()\n",
    "enc.fit(np.expand_dims(landmark_colors,axis=1))  \n",
    "for x in landmark_colors:\n",
    "    one_hot_list.append(enc.transform([[x]]).toarray()[0])\n",
    "one_hot_arr  = np.array(one_hot_list)\n",
    "one_hot_arr = torch.FloatTensor(one_hot_arr)\n",
    "class landmark_dataset(Dataset):\n",
    "    def __init__(self, info_list, transform=None, pre_transform=None, pre_filter=None):\n",
    "        super().__init__(None, transform, pre_transform, pre_filter)\n",
    "        \n",
    "        self.graphs = pd.read_csv(info_list, delimiter=\",\", header=None)\n",
    "\n",
    "\n",
    "    def len(self):\n",
    "        return len(self.graphs)\n",
    "\n",
    "    def get(self, idx):\n",
    "        \n",
    "        graph_path = self.graphs.iloc[idx]\n",
    "        graph_path = graph_path[0]\n",
    "        \n",
    "        \n",
    "        data = torch.load(graph_path)\n",
    "        #data.x =data.x.t()\n",
    "        return data \n",
    "class landmark_dataset_train(Dataset):\n",
    "    def __init__(self, info_list, transform=None, pre_transform=None, pre_filter=None):\n",
    "        super().__init__(None, transform, pre_transform, pre_filter)\n",
    "        \n",
    "        self.graphs = pd.read_csv(info_list, delimiter=\",\", header=None)\n",
    "\n",
    "\n",
    "    def len(self):\n",
    "        return len(self.graphs)\n",
    "\n",
    "    def get(self, idx):\n",
    "        \n",
    "        graph_path = self.graphs.iloc[idx]\n",
    "        graph_path = graph_path[0]\n",
    "        \n",
    "        data = torch.load(graph_path)\n",
    "        \n",
    " \n",
    "        \n",
    "        domain_name = graph_path.split(\"/\")[-4]\n",
    "        \n",
    "            \n",
    "        if domain_name == \"casia-ds\" :\n",
    "            \n",
    "            domain_label = torch.FloatTensor([[0,0,1]])\n",
    "            \n",
    "        elif domain_name == \"Replay_attack_dataset\" :\n",
    "            \n",
    "            domain_label = torch.FloatTensor([[0,1,0]])\n",
    "            \n",
    "        elif domain_name == \"MSU_MFSD\" :\n",
    "            \n",
    "            domain_label = torch.FloatTensor([[0,0,1]])\n",
    "        else :\n",
    "            print(graph_path,domain_name )\n",
    "            raise \"error\"\n",
    "            \n",
    "        \n",
    "        data = Data(x=data.x, edge_index=data.edge_index,y =data.y,data = domain_label,y_node = one_hot_arr)\n",
    "\n",
    "        #data.x =data.x.t()\n",
    "        return data \n",
    "train_dataset = landmark_dataset_train(\"/home/hassan-hossein/single_image_graph_face_anti_spoofing/train/graph_train_M_C_I.txt\")\n",
    "train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True,num_workers=16,pin_memory=True)\n",
    "\n",
    "test_dataset = landmark_dataset(\"/home/hassan-hossein/single_image_graph_face_anti_spoofing/train/graph_test_O.txt\")\n",
    "test_loader = DataLoader(test_dataset, batch_size=64,num_workers=16)\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "\n",
    "# فرض کنید که داده‌های ورودی شما یک توالی از ۶۸۰ ویژگی دارد.\n",
    "#sequence_length = 128\n",
    "#feature_dim = 4  # هر ویژگی تک‌بعدی است\n",
    "\n",
    "# نمونه داده ورودی (تعداد نمونه‌ها × طول توالی × بعد ویژگی)\n",
    "#inputs = torch.rand(32, sequence_length, feature_dim)  # 32 نمونه با 680 ویژگی تک‌بعدی\n",
    "\n",
    "# تعریف یک لایه ترنسفورمر انکودر\n",
    "class SimpleTransformer(nn.Module):\n",
    "    def __init__(self, embed_dim, num_layers, nhead,n_out):\n",
    "        super(SimpleTransformer, self).__init__()\n",
    "        self.linear = nn.Linear(4, embed_dim)  # افزایش بعد ویژگی‌ها\n",
    "        self.encoder_layer = TransformerEncoderLayer(d_model=embed_dim, nhead=nhead)\n",
    "        self.transformer_encoder = TransformerEncoder(self.encoder_layer, num_layers=num_layers)\n",
    "        self.fc = nn.Linear(128 * embed_dim, n_out)  # فرضا 10 کلاس خروجی داریم\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)  # تغییر بعد ویژگی‌ها\n",
    "        x = self.transformer_encoder(x)\n",
    "        x = x.view(x.size(0), -1)  # فشرده‌سازی به یک بردار برای طبقه‌بندی\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "# # ایجاد مدل و ارسال ورودی\n",
    "# # model = SimpleTransformer(embed_dim=embed_dim, num_layers=2, nhead=4)\n",
    "# # output = model(inputs)\n",
    "\n",
    "# #print(output.shape)\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "\n",
    "# فرض کنید که داده‌های ورودی شما یک توالی از ۶۸۰ ویژگی دارد.\n",
    "sequence_length = 128\n",
    "feature_dim = 1  # هر ویژگی تک‌بعدی است\n",
    "\n",
    "# نمونه داده ورودی (تعداد نمونه‌ها × طول توالی × بعد ویژگی)\n",
    "#inputs = torch.rand(32, sequence_length, feature_dim)  # 32 نمونه با 680 ویژگی تک‌بعدی\n",
    "\n",
    "# تعریف یک لایه ترنسفورمر انکودر\n",
    "class SimpleTransformer_2(nn.Module):\n",
    "    def __init__(self, embed_dim, num_layers, nhead):\n",
    "        super(SimpleTransformer_2, self).__init__()\n",
    "        self.linear = nn.Linear(feature_dim, embed_dim)  # افزایش بعد ویژگی‌ها\n",
    "        self.encoder_layer = TransformerEncoderLayer(d_model=embed_dim, nhead=nhead)\n",
    "        self.transformer_encoder = TransformerEncoder(self.encoder_layer, num_layers=num_layers)\n",
    "        self.fc = nn.Linear(sequence_length * embed_dim, 7)  # فرضا 10 کلاس خروجی داریم\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.linear(x)  # تغییر بعد ویژگی‌ها\n",
    "        x = self.transformer_encoder(x)\n",
    "        x = x.view(x.size(0), -1)  # فشرده‌سازی به یک بردار برای طبقه‌بندی\n",
    "        x = self.fc(x)\n",
    "        return x\n",
    "\n",
    "# # ایجاد مدل و ارسال ورودی\n",
    "# # model = SimpleTransformer(embed_dim=embed_dim, num_layers=2, nhead=4)\n",
    "# # output = model(inputs)\n",
    "\n",
    "# #print(output.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d408ce67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GCN(\n",
      "  (conv1): ChebConv(640, 128, K=2, normalization=sym)\n",
      "  (conv2): ChebConv(128, 128, K=2, normalization=sym)\n",
      "  (conv3): ChebConv(128, 128, K=2, normalization=sym)\n",
      "  (conv4): ChebConv(128, 128, K=2, normalization=sym)\n",
      "  (conv5): ChebConv(256, 128, K=2, normalization=sym)\n",
      "  (conv6): ChebConv(128, 128, K=2, normalization=sym)\n",
      "  (conv7): ChebConv(128, 128, K=2, normalization=sym)\n",
      "  (conv8): ChebConv(128, 128, K=2, normalization=sym)\n",
      "  (lin): SimpleTransformer(\n",
      "    (linear): Linear(in_features=4, out_features=8, bias=True)\n",
      "    (encoder_layer): TransformerEncoderLayer(\n",
      "      (self_attn): MultiheadAttention(\n",
      "        (out_proj): NonDynamicallyQuantizableLinear(in_features=8, out_features=8, bias=True)\n",
      "      )\n",
      "      (linear1): Linear(in_features=8, out_features=2048, bias=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "      (linear2): Linear(in_features=2048, out_features=8, bias=True)\n",
      "      (norm1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "      (norm2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "      (dropout1): Dropout(p=0.1, inplace=False)\n",
      "      (dropout2): Dropout(p=0.1, inplace=False)\n",
      "    )\n",
      "    (transformer_encoder): TransformerEncoder(\n",
      "      (layers): ModuleList(\n",
      "        (0): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=8, out_features=8, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=8, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=8, bias=True)\n",
      "          (norm1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (1): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=8, out_features=8, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=8, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=8, bias=True)\n",
      "          (norm1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (2): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=8, out_features=8, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=8, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=8, bias=True)\n",
      "          (norm1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (3): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=8, out_features=8, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=8, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=8, bias=True)\n",
      "          (norm1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (4): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=8, out_features=8, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=8, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=8, bias=True)\n",
      "          (norm1): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((8,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (fc): Linear(in_features=1024, out_features=1, bias=True)\n",
      "  )\n",
      "  (drop_1): RandomNodeDropout()\n",
      "  (drop_2): RandomNodeDropout()\n",
      "  (drop_3): RandomNodeDropout()\n",
      "  (drop_4): RandomNodeDropout()\n",
      "  (drop_5): RandomNodeDropout()\n",
      "  (drop_6): RandomNodeDropout()\n",
      "  (b1): BatchNorm(128)\n",
      "  (b2): BatchNorm(128)\n",
      "  (b3): BatchNorm(128)\n",
      "  (b4): BatchNorm(128)\n",
      "  (b5): BatchNorm(128)\n",
      "  (b6): BatchNorm(128)\n",
      "  (b7): BatchNorm(128)\n",
      "  (dis): Discriminator(\n",
      "    (conv1): GraphSAGE(128, 128, num_layers=2)\n",
      "    (conv2): GraphSAGE(128, 128, num_layers=2)\n",
      "    (conv3): GraphSAGE(128, 128, num_layers=2)\n",
      "    (conv4): GraphSAGE(128, 128, num_layers=2)\n",
      "    (drop_1): RandomNodeDropout()\n",
      "    (drop_2): RandomNodeDropout()\n",
      "    (grl_layer): GRL()\n",
      "    (fc2): Linear(in_features=128, out_features=3, bias=True)\n",
      "  )\n",
      "  (conv1_node_class): GraphSAGE(128, 128, num_layers=2)\n",
      "  (conv2_node_class): GraphSAGE(128, 128, num_layers=2)\n",
      "  (lin_node_class): SimpleTransformer_2(\n",
      "    (linear): Linear(in_features=1, out_features=64, bias=True)\n",
      "    (encoder_layer): TransformerEncoderLayer(\n",
      "      (self_attn): MultiheadAttention(\n",
      "        (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "      )\n",
      "      (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "      (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "      (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "      (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "      (dropout1): Dropout(p=0.1, inplace=False)\n",
      "      (dropout2): Dropout(p=0.1, inplace=False)\n",
      "    )\n",
      "    (transformer_encoder): TransformerEncoder(\n",
      "      (layers): ModuleList(\n",
      "        (0): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (1): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (2): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (3): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (4): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (5): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (6): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (7): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (8): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "        (9): TransformerEncoderLayer(\n",
      "          (self_attn): MultiheadAttention(\n",
      "            (out_proj): NonDynamicallyQuantizableLinear(in_features=64, out_features=64, bias=True)\n",
      "          )\n",
      "          (linear1): Linear(in_features=64, out_features=2048, bias=True)\n",
      "          (dropout): Dropout(p=0.1, inplace=False)\n",
      "          (linear2): Linear(in_features=2048, out_features=64, bias=True)\n",
      "          (norm1): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (norm2): LayerNorm((64,), eps=1e-05, elementwise_affine=True)\n",
      "          (dropout1): Dropout(p=0.1, inplace=False)\n",
      "          (dropout2): Dropout(p=0.1, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (fc): Linear(in_features=8192, out_features=7, bias=True)\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class RandomNodeDropout(nn.Module):\n",
    "    def __init__(self, drop_prob=0.3):\n",
    "        super(RandomNodeDropout, self).__init__()\n",
    "        self.drop_prob = drop_prob\n",
    "\n",
    "    def forward(self, x, enabled=True):\n",
    "        if not self.training or not enabled:\n",
    "            return x\n",
    "        \n",
    "        num_rows = x.size(0)\n",
    "        mask = torch.rand(num_rows, device=x.device) > self.drop_prob\n",
    "        mask = mask.float().unsqueeze(1).expand_as(x)\n",
    "        x = x * mask\n",
    "        \n",
    "        return x\n",
    "\n",
    "soft = torch.nn.Softmax(dim=1)\n",
    "\n",
    "num_domain = 3\n",
    "class GRL(nn.Module):\n",
    "\n",
    "    def __init__(self, max_iter):\n",
    "        super(GRL, self).__init__()\n",
    "        self.iter_num = 0\n",
    "        self.alpha = 10\n",
    "        self.low = 0.0\n",
    "        self.high = 1.0\n",
    "        self.max_iter = max_iter\n",
    "\n",
    "    def forward(self, input):\n",
    "        self.iter_num += 1\n",
    "        return input * 1.0\n",
    "\n",
    "    def backward(self, gradOutput):\n",
    "        coeff = np.float(2.0 * (self.high - self.low) / (1.0 + np.exp(-self.alpha * self.iter_num / self.max_iter))\n",
    "                         - (self.high - self.low) + self.low)\n",
    "        return -coeff*10* gradOutput\n",
    "\n",
    "num_node  =train_dataset.num_node_features\n",
    "class Discriminator(torch.nn.Module):\n",
    "    def __init__(self,hidden_channels ,max_iter):\n",
    "        super(Discriminator, self).__init__()\n",
    "        torch.manual_seed(12345)\n",
    "        self.conv1 = GraphSAGE(hidden_channels, hidden_channels,2,dropout=0.5)\n",
    "        self.conv2 = GraphSAGE(hidden_channels, hidden_channels,2,dropout=0.5)\n",
    "        self.conv3 = GraphSAGE(hidden_channels, hidden_channels,2,dropout=0.5)\n",
    "        self.conv4 = GraphSAGE(hidden_channels, hidden_channels,2,dropout=0.5)\n",
    "        self.drop_1 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.drop_2 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.grl_layer = GRL(max_iter)\n",
    "        self.fc2 = nn.Linear(hidden_channels, num_domain)\n",
    "        \n",
    "    def forward(self, x, edge_index, batch,drop_en):\n",
    "        #print(\"graph fea :\",graph_level_feature.shape ,edge_index.shape )\n",
    "        \n",
    "        x = self.grl_layer(x)\n",
    "        x = self.conv1(x, edge_index)\n",
    "        x = x.relu()\n",
    "        x = self.drop_1(x,drop_en)\n",
    "        \n",
    "        x = self.conv2(x, edge_index)\n",
    "        x = x.relu()\n",
    "        x = self.drop_2(x,drop_en)\n",
    "        \n",
    "        x = self.conv3(x, edge_index)\n",
    "        x = x.relu()\n",
    "        x = self.conv4(x, edge_index)\n",
    "        x = x.relu()\n",
    "        \n",
    "        x = global_mean_pool(x, batch)\n",
    "        \n",
    "        x = self.fc2(x)\n",
    "         \n",
    "        return x \n",
    "\n",
    "\n",
    "\n",
    "class GCN(torch.nn.Module):\n",
    "    def __init__(self, hidden_channels,max_iter=4000):\n",
    "        super(GCN, self).__init__()\n",
    "        torch.manual_seed(12345)\n",
    "        self.conv1 = ChebConv(num_node, hidden_channels,2)\n",
    "        self.conv2 = ChebConv(hidden_channels, hidden_channels,2)\n",
    "        self.conv3 = ChebConv(hidden_channels, hidden_channels,2)\n",
    "        self.conv4 = ChebConv(hidden_channels, hidden_channels,2)\n",
    "        self.conv5 = ChebConv(hidden_channels*2, hidden_channels,2)\n",
    "        self.conv6 = ChebConv(hidden_channels, hidden_channels,2)\n",
    "        self.conv7 = ChebConv(hidden_channels, hidden_channels,2)\n",
    "        self.conv8 = ChebConv(hidden_channels, hidden_channels,2)\n",
    "        self.lin = SimpleTransformer(8,5,8,1)\n",
    "        \n",
    "        self.drop_1 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.drop_2 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.drop_3 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.drop_4 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.drop_5 = RandomNodeDropout(drop_prob=0.3)\n",
    "        self.drop_6 = RandomNodeDropout(drop_prob=0.3)\n",
    "        \n",
    "        \n",
    "        self.b1 = BatchNorm(hidden_channels)\n",
    "        self.b2 = BatchNorm(hidden_channels)\n",
    "        self.b3 = BatchNorm(hidden_channels)\n",
    "        self.b4 = BatchNorm(hidden_channels)\n",
    "        self.b5 = BatchNorm(hidden_channels)\n",
    "        self.b6 = BatchNorm(hidden_channels)\n",
    "        self.b7 = BatchNorm(hidden_channels)\n",
    "        \n",
    "        self.dis = Discriminator(hidden_channels,max_iter)\n",
    "        \n",
    "        self.conv1_node_class = GraphSAGE(hidden_channels, hidden_channels,2,dropout=0.5)\n",
    "        \n",
    "        self.conv2_node_class = GraphSAGE(hidden_channels, hidden_channels,2,dropout=0.5)\n",
    "        \n",
    "        self.lin_node_class = SimpleTransformer_2(64,10,8)\n",
    "        \n",
    "\n",
    "        \n",
    "        #self.tcn = Simple1DCNN()\n",
    "        \n",
    "    def forward(self, x, edge_index, batch,drop_en,index_arr):\n",
    "        #print(\"graph fea :\",graph_level_feature.shape ,edge_index.shape )\n",
    "        \n",
    "\n",
    "        x = self.conv1(x, edge_index)\n",
    "        x  = self.b1(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        #x = self.drop_1(x,drop_en)\n",
    "        \n",
    "        \n",
    "\n",
    "        x = self.conv2(x, edge_index)\n",
    "        x  = self.b2(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        #x = self.drop_2(x,drop_en)\n",
    "                \n",
    "        dicriminator = x\n",
    "        \n",
    "        x_1 = global_mean_pool(x, batch)\n",
    "        \n",
    "        x = self.conv3(x, edge_index)\n",
    "        x  = self.b3(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        x = self.drop_3(x,drop_en)\n",
    "        \n",
    "        x_3 = global_mean_pool(x, batch)\n",
    "        \n",
    "        \n",
    "        x = self.conv4(x, edge_index)\n",
    "        x  = self.b4(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        x = self.drop_4(x,drop_en)\n",
    "        \n",
    "        \n",
    "        x = torch.cat((x,dicriminator),dim=1) #skip connection\n",
    "        \n",
    "        x = self.conv5(x, edge_index)\n",
    "        x  = self.b5(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        x = self.drop_5(x,drop_en)\n",
    "        \n",
    "        x_5 = global_mean_pool(x, batch)\n",
    "        \n",
    "        x = self.conv6(x, edge_index)\n",
    "        x  = self.b6(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        x = self.drop_6(x,drop_en)\n",
    "        \n",
    "        x = self.conv7(x, edge_index)\n",
    "        x  = self.b7(x)\n",
    "        x = x.relu()\n",
    "        \n",
    "        \n",
    "        x = self.conv8(x, edge_index)\n",
    "        # 2. Readout layer\n",
    "        x = global_mean_pool(x, batch)  # [batch_size, hidden_channels]\n",
    "        \n",
    "        x = torch.stack([x,x_1,x_5,x_3],dim=2)\n",
    "        \n",
    "        x = F.dropout(x, p=0.5, training=self.training)\n",
    "        \n",
    "        #x = torch.unsqueeze(x,2)\n",
    "        #print(x.shape)\n",
    "        \n",
    "        x = self.lin(x)\n",
    "        \n",
    "        x= x.sigmoid()\n",
    "        \n",
    "        \n",
    "        node_class_hidden = self.conv1_node_class(dicriminator,edge_index)\n",
    "        node_class_hidden = node_class_hidden.relu()\n",
    "        \n",
    "        node_class_hidden = self.conv2_node_class(node_class_hidden,edge_index)\n",
    "        node_class_hidden = node_class_hidden.relu()\n",
    "        \n",
    "        #print(index_arr.shape , node_class_hidden.shape)\n",
    "        \n",
    "        node_class_hidden = node_class_hidden[index_arr]\n",
    "        \n",
    "        #print(node_class_hidden.shape)\n",
    "        node_class_hidden = torch.unsqueeze(node_class_hidden, dim=2)\n",
    "        \n",
    "        node_pre = self.lin_node_class(node_class_hidden)\n",
    "        \n",
    "            \n",
    "        #print(\"hiiii :\" , dicriminator.shape)\n",
    "        dis_invariant = self.dis(dicriminator,edge_index,batch,drop_en)\n",
    "        \n",
    "        return x ,dis_invariant ,node_pre\n",
    "\n",
    "model = GCN(hidden_channels=128).to(device)\n",
    "print(model)\n",
    "\n",
    "def check_folder(log_dir):\n",
    "    if not os.path.exists(log_dir):\n",
    "        os.makedirs(log_dir)\n",
    "    return log_dir\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)\n",
    "#criterion = torch.nn.CrossEntropyLoss()\n",
    "criterion= nn.BCELoss()\n",
    "criterion_node_class=  torch.nn.CrossEntropyLoss()\n",
    "criterion_2 = torch.nn.CrossEntropyLoss()\n",
    "def train(epoch):\n",
    "    model.train()\n",
    "    losses_adv_loss=0\n",
    "    losses_loss_cls=0\n",
    "    total_loss = 0\n",
    "    losses_node_cls_loss = 0\n",
    "    progress_bar = tqdm(train_loader, desc=f'Epoch {epoch+1}', unit='batch')\n",
    "    for graph in progress_bar:  # Iterate in batches over the training dataset.\n",
    "        graph = graph.to(device)  # Move data to the device\n",
    "        \n",
    "        drop_en = bool(np.random.binomial(1, 0.5))\n",
    "        \n",
    "        if graph.x.shape[0] < 29952 :\n",
    "            \n",
    "            if graph.x.shape[0] < 64:\n",
    "            \n",
    "                index_arr = torch.randint(0, int(graph.x.shape[0]), (int(graph.x.shape[0]),)).to(device)\n",
    "            else :\n",
    "                \n",
    "                index_arr = torch.randint(0, int(graph.x.shape[0]), (64,)).to(device)\n",
    "        else :\n",
    "                \n",
    "            index_arr = torch.randint(0, 29952, (64,)).to(device)\n",
    "        \n",
    "        \n",
    "        \n",
    "        out,domain_invariant,node_pre = model(graph.x, graph.edge_index, graph.batch,drop_en,index_arr)  # Perform a single forward pass.\n",
    "        \n",
    "        adv_loss = criterion_2(domain_invariant, graph.data)\n",
    "        \n",
    "        node_cls_loss = criterion_node_class(node_pre.squeeze(),graph.y_node.float()[index_arr])\n",
    "        #print(out.shape , graph.y.long().shape)\n",
    "        loss_cls = criterion(out.squeeze(), graph.y.float())  # Compute the loss.\n",
    "        \n",
    "        loss_all = 0.1* adv_loss + 0.8*loss_cls + 0.1*node_cls_loss\n",
    "        loss_all.backward()  # Derive gradients.\n",
    "        optimizer.step()  # Update parameters based on gradients.\n",
    "        optimizer.zero_grad()  # Clear gradients.\n",
    "        total_loss = loss_all + total_loss\n",
    "        losses_adv_loss = adv_loss + losses_adv_loss\n",
    "        losses_loss_cls = loss_cls + losses_loss_cls\n",
    "        losses_node_cls_loss = node_cls_loss + losses_node_cls_loss\n",
    "\n",
    "\n",
    "    print(\"/////////////////////////\")\n",
    "    print(\"adverserial loss : \" ,losses_adv_loss/len(train_loader.dataset))  \n",
    "    print(\"classification loss : \" , losses_loss_cls/len(train_loader.dataset))\n",
    "    print(\"classification node loss : \" , losses_node_cls_loss/len(train_loader.dataset))\n",
    "    print(\"total loss : \" ,total_loss/len(train_loader.dataset) )\n",
    "    print(\"////////////////////////////////////\")\n",
    "def test(loader,epoch):\n",
    "    score_path = os.path.join(\"/home/hassan-hossein/single_image_graph_face_anti_spoofing/train/Scores\", \"epoch_{}\".format(epoch+1))\n",
    "    check_folder(score_path)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        scores_list = []\n",
    "\n",
    "        correct = 0\n",
    "        s= 0\n",
    "        progress_bar = tqdm(loader, desc=f'Epoch {epoch+1}', unit='batch')\n",
    "        for graph in progress_bar:  # Iterate in batches over the training/test dataset.\n",
    "            if graph.x.shape[0] < 29952 :\n",
    "\n",
    "                if graph.x.shape[0] < 64:\n",
    "\n",
    "                    index_arr = torch.randint(0, int(graph.x.shape[0]), (int(graph.x.shape[0]),)).to(device)\n",
    "                else :\n",
    "\n",
    "                    index_arr = torch.randint(0, int(graph.x.shape[0]), (64,)).to(device)\n",
    "            else :\n",
    "\n",
    "                index_arr = torch.randint(0, 29952, (64,)).to(device)\n",
    "                \n",
    "            graph = graph.to(device)  # Move data to the device\n",
    "            logit,_ ,_= model(graph.x, graph.edge_index, graph.batch,False,index_arr)\n",
    "            live_label = graph.y.float()\n",
    "            for i in range(len(logit)):\n",
    "                        scores_list.append(\"{} {}\\n\".format(logit[i].item(), live_label[i].item()))\n",
    "  # Check against ground-truth labels.\n",
    "    map_score_val_filename = os.path.join(score_path, \"score.txt\")\n",
    "    with open(map_score_val_filename, 'w') as file:\n",
    "                file.writelines(scores_list)\n",
    "    print(\"score: write test scores to {}\".format(map_score_val_filename))\n",
    "    test_ACC, fpr, FRR, HTER, auc_test, test_err, tpr = performances_val(map_score_val_filename)\n",
    "    print(\"epoch:{:d}, test:  val_ACC={:.4f}, HTER={:.4f}, AUC={:.4f}, val_err={:.4f}, ACC={:.4f}, TPR={:.4f}\".format(\n",
    "        epoch + 1, test_ACC, HTER, auc_test, test_err, test_ACC, tpr))  \n",
    "    return auc_test\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "36095ee9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|███████████████████████████| 1560/1560 [05:54<00:00,  4.40batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/////////////////////////\n",
      "adverserial loss :  tensor(0.0012, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification loss :  tensor(0.0007, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification node loss :  tensor(0.0129, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "total loss :  tensor(0.0020, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "////////////////////////////////////\n",
      "***************************\n",
      "test : \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 1: 100%|███████████████████████████| 2131/2131 [15:26<00:00,  2.30batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: write test scores to /home/hassan-hossein/single_image_graph_face_anti_spoofing/train/Scores/epoch_1/score.txt\n",
      "epoch:1, test:  val_ACC=0.7923, HTER=0.2078, AUC=0.8539, val_err=0.2079, ACC=0.7923, TPR=0.1455\n",
      "improve acc .. .. ..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|███████████████████████████| 1560/1560 [11:41<00:00,  2.22batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/////////////////////////\n",
      "adverserial loss :  tensor(2.4943e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification loss :  tensor(7.1381e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification node loss :  tensor(0.0053, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "total loss :  tensor(0.0006, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "////////////////////////////////////\n",
      "***************************\n",
      "test : \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 2: 100%|███████████████████████████| 2131/2131 [15:34<00:00,  2.28batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: write test scores to /home/hassan-hossein/single_image_graph_face_anti_spoofing/train/Scores/epoch_2/score.txt\n",
      "epoch:2, test:  val_ACC=0.8427, HTER=0.1573, AUC=0.9136, val_err=0.1573, ACC=0.8427, TPR=0.5211\n",
      "improve acc .. .. ..\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|███████████████████████████| 1560/1560 [11:37<00:00,  2.24batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/////////////////////////\n",
      "adverserial loss :  tensor(2.6317e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification loss :  tensor(4.8982e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification node loss :  tensor(0.0043, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "total loss :  tensor(0.0005, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "////////////////////////////////////\n",
      "***************************\n",
      "test : \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 3: 100%|███████████████████████████| 2131/2131 [15:31<00:00,  2.29batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: write test scores to /home/hassan-hossein/single_image_graph_face_anti_spoofing/train/Scores/epoch_3/score.txt\n",
      "epoch:3, test:  val_ACC=0.9886, HTER=0.2441, AUC=0.7647, val_err=0.4304, ACC=0.9886, TPR=0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|███████████████████████████| 1560/1560 [07:57<00:00,  3.27batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/////////////////////////\n",
      "adverserial loss :  tensor(2.3625e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification loss :  tensor(4.5898e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification node loss :  tensor(0.0037, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "total loss :  tensor(0.0004, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "////////////////////////////////////\n",
      "***************************\n",
      "test : \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 4: 100%|███████████████████████████| 2131/2131 [04:47<00:00,  7.42batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: write test scores to /home/hassan-hossein/single_image_graph_face_anti_spoofing/train/Scores/epoch_4/score.txt\n",
      "epoch:4, test:  val_ACC=0.9858, HTER=0.1749, AUC=0.8410, val_err=0.2779, ACC=0.9858, TPR=0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5: 100%|███████████████████████████| 1560/1560 [03:49<00:00,  6.80batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/////////////////////////\n",
      "adverserial loss :  tensor(5.9433e-06, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification loss :  tensor(2.6858e-05, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "classification node loss :  tensor(0.0032, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "total loss :  tensor(0.0003, device='cuda:0', grad_fn=<DivBackward0>)\n",
      "////////////////////////////////////\n",
      "***************************\n",
      "test : \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 5: 100%|███████████████████████████| 2131/2131 [04:46<00:00,  7.44batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score: write test scores to /home/hassan-hossein/single_image_graph_face_anti_spoofing/train/Scores/epoch_5/score.txt\n",
      "epoch:5, test:  val_ACC=0.9974, HTER=0.3189, AUC=0.6822, val_err=0.6245, ACC=0.9974, TPR=0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 6:  20%|█████▋                      | 318/1560 [00:50<03:18,  6.25batch/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m best_auc \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1000\u001b[39m):\n\u001b[0;32m----> 5\u001b[0m     \u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43mepoch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m***************************\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;66;03m#print(\"train : \")\u001b[39;00m\n\u001b[1;32m      8\u001b[0m     \u001b[38;5;66;03m#train_auc = test(train_loader,epoch)\u001b[39;00m\n",
      "Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36mtrain\u001b[0;34m(epoch)\u001b[0m\n\u001b[1;32m    263\u001b[0m loss_all \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0.1\u001b[39m\u001b[38;5;241m*\u001b[39m adv_loss \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m0.8\u001b[39m\u001b[38;5;241m*\u001b[39mloss_cls \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m0.1\u001b[39m\u001b[38;5;241m*\u001b[39mnode_cls_loss\n\u001b[1;32m    264\u001b[0m loss_all\u001b[38;5;241m.\u001b[39mbackward()  \u001b[38;5;66;03m# Derive gradients.\u001b[39;00m\n\u001b[0;32m--> 265\u001b[0m \u001b[43moptimizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# Update parameters based on gradients.\u001b[39;00m\n\u001b[1;32m    266\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()  \u001b[38;5;66;03m# Clear gradients.\u001b[39;00m\n\u001b[1;32m    267\u001b[0m total_loss \u001b[38;5;241m=\u001b[39m loss_all \u001b[38;5;241m+\u001b[39m total_loss\n",
      "File \u001b[0;32m~/anaconda3/envs/Wav2Lip/lib/python3.8/site-packages/torch/optim/optimizer.py:113\u001b[0m, in \u001b[0;36mOptimizer._hook_for_profile.<locals>.profile_hook_step.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    111\u001b[0m profile_name \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOptimizer.step#\u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m.step\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(obj\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m)\n\u001b[1;32m    112\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mautograd\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mrecord_function(profile_name):\n\u001b[0;32m--> 113\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/Wav2Lip/lib/python3.8/site-packages/torch/autograd/grad_mode.py:27\u001b[0m, in \u001b[0;36m_DecoratorContextManager.__call__.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     25\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_context\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclone():\n\u001b[0;32m---> 27\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/Wav2Lip/lib/python3.8/site-packages/torch/optim/adam.py:157\u001b[0m, in \u001b[0;36mAdam.step\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    153\u001b[0m                 max_exp_avg_sqs\u001b[38;5;241m.\u001b[39mappend(state[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmax_exp_avg_sq\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m    155\u001b[0m             state_steps\u001b[38;5;241m.\u001b[39mappend(state[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstep\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[0;32m--> 157\u001b[0m     \u001b[43madam\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparams_with_grad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    158\u001b[0m \u001b[43m         \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    159\u001b[0m \u001b[43m         \u001b[49m\u001b[43mexp_avgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    160\u001b[0m \u001b[43m         \u001b[49m\u001b[43mexp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    161\u001b[0m \u001b[43m         \u001b[49m\u001b[43mmax_exp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    162\u001b[0m \u001b[43m         \u001b[49m\u001b[43mstate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    163\u001b[0m \u001b[43m         \u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mamsgrad\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    164\u001b[0m \u001b[43m         \u001b[49m\u001b[43mbeta1\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta1\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    165\u001b[0m \u001b[43m         \u001b[49m\u001b[43mbeta2\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta2\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    166\u001b[0m \u001b[43m         \u001b[49m\u001b[43mlr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mlr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    167\u001b[0m \u001b[43m         \u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mweight_decay\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    168\u001b[0m \u001b[43m         \u001b[49m\u001b[43meps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43meps\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    169\u001b[0m \u001b[43m         \u001b[49m\u001b[43mmaximize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mmaximize\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    170\u001b[0m \u001b[43m         \u001b[49m\u001b[43mforeach\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mforeach\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    171\u001b[0m \u001b[43m         \u001b[49m\u001b[43mcapturable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgroup\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mcapturable\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    173\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m loss\n",
      "File \u001b[0;32m~/anaconda3/envs/Wav2Lip/lib/python3.8/site-packages/torch/optim/adam.py:213\u001b[0m, in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, foreach, capturable, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize)\u001b[0m\n\u001b[1;32m    210\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    211\u001b[0m     func \u001b[38;5;241m=\u001b[39m _single_tensor_adam\n\u001b[0;32m--> 213\u001b[0m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    214\u001b[0m \u001b[43m     \u001b[49m\u001b[43mgrads\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    215\u001b[0m \u001b[43m     \u001b[49m\u001b[43mexp_avgs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    216\u001b[0m \u001b[43m     \u001b[49m\u001b[43mexp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    217\u001b[0m \u001b[43m     \u001b[49m\u001b[43mmax_exp_avg_sqs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    218\u001b[0m \u001b[43m     \u001b[49m\u001b[43mstate_steps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    219\u001b[0m \u001b[43m     \u001b[49m\u001b[43mamsgrad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mamsgrad\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    220\u001b[0m \u001b[43m     \u001b[49m\u001b[43mbeta1\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta1\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    221\u001b[0m \u001b[43m     \u001b[49m\u001b[43mbeta2\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbeta2\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    222\u001b[0m \u001b[43m     \u001b[49m\u001b[43mlr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlr\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    223\u001b[0m \u001b[43m     \u001b[49m\u001b[43mweight_decay\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mweight_decay\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    224\u001b[0m \u001b[43m     \u001b[49m\u001b[43meps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    225\u001b[0m \u001b[43m     \u001b[49m\u001b[43mmaximize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmaximize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    226\u001b[0m \u001b[43m     \u001b[49m\u001b[43mcapturable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcapturable\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/Wav2Lip/lib/python3.8/site-packages/torch/optim/adam.py:305\u001b[0m, in \u001b[0;36m_single_tensor_adam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps, maximize, capturable)\u001b[0m\n\u001b[1;32m    303\u001b[0m     denom \u001b[38;5;241m=\u001b[39m (max_exp_avg_sqs[i]\u001b[38;5;241m.\u001b[39msqrt() \u001b[38;5;241m/\u001b[39m bias_correction2_sqrt)\u001b[38;5;241m.\u001b[39madd_(eps)\n\u001b[1;32m    304\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 305\u001b[0m     denom \u001b[38;5;241m=\u001b[39m (\u001b[43mexp_avg_sq\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msqrt\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;241m/\u001b[39m bias_correction2_sqrt)\u001b[38;5;241m.\u001b[39madd_(eps)\n\u001b[1;32m    307\u001b[0m param\u001b[38;5;241m.\u001b[39maddcdiv_(exp_avg, denom, value\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m-\u001b[39mstep_size)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#model.load_state_dict(torch.load(\"M_test_V_10_without_pipline_domain_generalization_tuning.pth\",map_location='cuda'))\n",
    "#best_auc = 0.9862\n",
    "best_auc = 0\n",
    "for epoch in range(0, 1000):\n",
    "    train(epoch)\n",
    "    print(\"***************************\")\n",
    "    #print(\"train : \")\n",
    "    #train_auc = test(train_loader,epoch)\n",
    "    print(\"test : \")\n",
    "    test_auc = test(test_loader,epoch)\n",
    "    if test_auc > best_auc:\n",
    "        print(\"improve acc .. .. ..\")\n",
    "        torch.save(model.state_dict(), 'O_test_V_10_without_pipline_domain_generalization_tuning.pth')\n",
    "        best_auc = test_auc\n",
    "        continue "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef09fc96",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (Wav2Lip)",
   "language": "python",
   "name": "wav2lip"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}