Zero-shot-testing-results-models (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4aec42f2-4bcd-499d-a0f7-1f5394735050",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting scikit-learn\n",
      "  Downloading scikit_learn-1.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
      "Requirement already satisfied: numpy<2.0,>=1.17.3 in /home/016651544/anaconda3/lib/python3.8/site-packages (from scikit-learn) (1.24.3)\n",
      "Requirement already satisfied: scipy>=1.5.0 in /home/016651544/anaconda3/lib/python3.8/site-packages (from scikit-learn) (1.10.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /home/016651544/anaconda3/lib/python3.8/site-packages (from scikit-learn) (1.2.0)\n",
      "Collecting threadpoolctl>=2.0.0 (from scikit-learn)\n",
      "  Using cached threadpoolctl-3.4.0-py3-none-any.whl.metadata (13 kB)\n",
      "Downloading scikit_learn-1.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.1/11.1 MB\u001b[0m \u001b[31m56.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m0:01\u001b[0m\n",
      "\u001b[?25hUsing cached threadpoolctl-3.4.0-py3-none-any.whl (17 kB)\n",
      "Installing collected packages: threadpoolctl, scikit-learn\n",
      "Successfully installed scikit-learn-1.3.2 threadpoolctl-3.4.0\n"
     ]
    }
   ],
   "source": [
    "# !pip install scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "9536e7e3-ca99-44b6-b9a4-533aa2b0e301",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>These results suggest that the C1772T polymorp...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In our setting, @DISEASE$ among alcoholic indi...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MPO genotype GG is associated with @DISEASE$ i...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>These three studies do not provide consistent ...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Our prospective findings suggest that individu...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  actual_output  \\\n",
       "0  These results suggest that the C1772T polymorp...           True   \n",
       "1  In our setting, @DISEASE$ among alcoholic indi...           True   \n",
       "2  MPO genotype GG is associated with @DISEASE$ i...           True   \n",
       "3  These three studies do not provide consistent ...           True   \n",
       "4  Our prospective findings suggest that individu...           True   \n",
       "\n",
       "  predicted_output  \n",
       "0             True  \n",
       "1             True  \n",
       "2             True  \n",
       "3             True  \n",
       "4             True  "
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "gad_out = pd.read_csv(\"/home/016651544/llama2/test_df_base_llama_full_test.csv\")\n",
    "gad_out.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "160d903b-f52f-4ed5-8975-1e5b1b7ebbd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import numpy as np\n",
    "# def process_predictions(x):\n",
    "#     if x is np.nan:\n",
    "#         return None\n",
    "#     return x.split(\"\\n\")[0]\n",
    "\n",
    "# def extract_boolean(sentence):\n",
    "#     if sentence is np.nan:\n",
    "#         return None\n",
    "#     sentence_start = sentence[:10]  # Check within the first 10 characters\n",
    "#     if \"TRUE\" in sentence_start.upper():\n",
    "#         return True\n",
    "#     elif \"FALSE\" in sentence_start.upper():\n",
    "#         return False\n",
    "#     return np.nan  # or None, depending on how you want to handle non-matches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "faf0bbe1-3bc2-452b-ab66-aa58b2c35c70",
   "metadata": {},
   "outputs": [],
   "source": [
    "gad_out.predicted_output = gad_out.predicted_output.apply(lambda x: extract_boolean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "bf6132fa-ef9b-47f0-94db-bf7464925244",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(actual_output\n",
       " True     2801\n",
       " False    2529\n",
       " Name: count, dtype: int64,\n",
       " predicted_output\n",
       " True     5245\n",
       " False       1\n",
       " Name: count, dtype: int64)"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gad_out.actual_output.value_counts(),gad_out.predicted_output.value_counts(),"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "249d0307-2bd9-4c48-8a7e-81c092e6e5dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_nan(row):\n",
    "    \n",
    "    if pd.isna(row[2]):# or row[2].lower() == \"nan\":\n",
    "        return not row[1]\n",
    "    else:\n",
    "        return row[2]\n",
    "gad_out.predicted_output = gad_out.apply(lambda x: change_nan(x),axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "1eb11b7f-11fe-46a0-9a07-84d2abc88b60",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>These results suggest that the C1772T polymorp...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In our setting, @DISEASE$ among alcoholic indi...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MPO genotype GG is associated with @DISEASE$ i...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>These three studies do not provide consistent ...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Our prospective findings suggest that individu...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5325</th>\n",
       "      <td>Despite anatomical similarities between @GENE$...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5326</th>\n",
       "      <td>The combination of the tt genotype and the neg...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5327</th>\n",
       "      <td>The @GENE$ @DISEASE$ polymorphism does not pla...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5328</th>\n",
       "      <td>Our data suggest both TNF and @GENE$ genes are...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5329</th>\n",
       "      <td>These results differ from previous reports inv...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5330 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence  actual_output  \\\n",
       "0     These results suggest that the C1772T polymorp...           True   \n",
       "1     In our setting, @DISEASE$ among alcoholic indi...           True   \n",
       "2     MPO genotype GG is associated with @DISEASE$ i...           True   \n",
       "3     These three studies do not provide consistent ...           True   \n",
       "4     Our prospective findings suggest that individu...           True   \n",
       "...                                                 ...            ...   \n",
       "5325  Despite anatomical similarities between @GENE$...          False   \n",
       "5326  The combination of the tt genotype and the neg...          False   \n",
       "5327  The @GENE$ @DISEASE$ polymorphism does not pla...          False   \n",
       "5328  Our data suggest both TNF and @GENE$ genes are...          False   \n",
       "5329  These results differ from previous reports inv...          False   \n",
       "\n",
       "      predicted_output  \n",
       "0                 True  \n",
       "1                 True  \n",
       "2                 True  \n",
       "3                 True  \n",
       "4                 True  \n",
       "...                ...  \n",
       "5325             False  \n",
       "5326              True  \n",
       "5327              True  \n",
       "5328              True  \n",
       "5329              True  \n",
       "\n",
       "[5330 rows x 3 columns]"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gad_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "a5199848-877b-401d-98b8-0a7d906cc40d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       FALSE       0.00      0.03      0.00        37\n",
      "        TRUE       0.99      0.52      0.68      5293\n",
      "\n",
      "    accuracy                           0.52      5330\n",
      "   macro avg       0.49      0.27      0.34      5330\n",
      "weighted avg       0.98      0.52      0.68      5330\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.27470754336425973, 0.4937714302735184, 0.342000781541748)"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "from sklearn.metrics import classification_report\n",
    "# Convert actual and predicted columns to boolean\n",
    "predicted = gad_out.predicted_output.astype(str).str.upper()\n",
    "actual = gad_out.actual_output.astype(str).str.upper()\n",
    "report = classification_report(predicted,actual)\n",
    "print(report)\n",
    "# Calculate Precision, Recall, and F1 score\n",
    "precision = precision_score(actual, predicted, average='macro')\n",
    "recall = recall_score(actual, predicted, average='macro')\n",
    "f1 = f1_score(actual, predicted, average='macro')\n",
    "\n",
    "precision, recall, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abba5bcb-e2ed-4b81-9f37-4570700bd349",
   "metadata": {},
   "source": [
    "### gemma-2b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "db86b41a-86b4-4e6a-885b-639fc62bb055",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>These results suggest that the C1772T polymorp...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In our setting, @DISEASE$ among alcoholic indi...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MPO genotype GG is associated with @DISEASE$ i...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>These three studies do not provide consistent ...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Our prospective findings suggest that individu...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  actual_output  \\\n",
       "0  These results suggest that the C1772T polymorp...           True   \n",
       "1  In our setting, @DISEASE$ among alcoholic indi...           True   \n",
       "2  MPO genotype GG is associated with @DISEASE$ i...           True   \n",
       "3  These three studies do not provide consistent ...           True   \n",
       "4  Our prospective findings suggest that individu...           True   \n",
       "\n",
       "  predicted_output  \n",
       "0           TRUE\\n  \n",
       "1           TRUE\\n  \n",
       "2           TRUE\\n  \n",
       "3           TRUE\\n  \n",
       "4           TRUE\\n  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gad_out_gemma = pd.read_csv(\"/home/016651544/gemma/test_df_gad_base_gemma_without_FT.csv\")\n",
    "gad_out_gemma.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "27cad20f-1a49-4a00-9e17-4a551e284d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "# gad_out_gemma.predicted_output = gad_out_gemma.predicted_output.apply(lambda x: extract_boolean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "ecb9a2ee-467b-4e79-acb7-e28760eba329",
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_nan(row):\n",
    "    if pd.isna(row[2]) :# or row[2].lower() == \"nan\":\n",
    "        return not row[1]\n",
    "    elif \"true\" in row[2].lower():\n",
    "        return True\n",
    "    elif \"false\" in row[2].lower():\n",
    "        return False\n",
    "    else:\n",
    "        return not row[1]\n",
    "gad_out_gemma.predicted_output = gad_out_gemma.apply(lambda x: change_nan(x),axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "66953466-30af-4b78-bbbb-02f1c78d7567",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# euadr_out_gemma.predicted_output = euadr_out_gemma.predicted_output.apply(lambda x: process_predictions(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "6cc3473b-277b-4fc9-9227-e21087b9f554",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predicted_output\n",
       "True     4794\n",
       "False     536\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gad_out_gemma.predicted_output.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "4d835620-8e0d-4775-bf6c-5b8ed932c7e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       FALSE       0.00      0.01      0.00       536\n",
      "        TRUE       0.81      0.47      0.60      4794\n",
      "\n",
      "    accuracy                           0.43      5330\n",
      "   macro avg       0.41      0.24      0.30      5330\n",
      "weighted avg       0.73      0.43      0.54      5330\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.2403813224241745, 0.4058247428720099, 0.3000542341864389)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "# Convert actual and predicted columns to boolean\n",
    "predicted = gad_out_gemma.predicted_output.astype(str).str.upper()\n",
    "actual = gad_out_gemma.actual_output.astype(str).str.upper()\n",
    "report = classification_report(predicted,actual)\n",
    "\n",
    "print(report)\n",
    "# Calculate Precision, Recall, and F1 score\n",
    "precision = precision_score(actual, predicted, average='macro')\n",
    "recall = recall_score(actual, predicted, average='macro')\n",
    "f1 = f1_score(actual, predicted, average='macro')\n",
    "\n",
    "precision, recall, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfcbf4a9-5648-4ed0-bb2c-5044dc84bd0b",
   "metadata": {},
   "source": [
    "### gemma-7b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37ba39fd-cac5-4c2c-8223-5ea1b4765652",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "cc88f293-6f4b-4afd-9968-03aa1836a1db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>These results suggest that the C1772T polymorp...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The sentence suggests that the C1772...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>In our setting, @DISEASE$ among alcoholic indi...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The sentence suggests that the prese...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>MPO genotype GG is associated with @DISEASE$ i...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The sentence suggests that the MPO g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>These three studies do not provide consistent ...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The sentence suggests that the prese...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Our prospective findings suggest that individu...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The presence of the @GENE$ C282Y mut...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  actual_output  \\\n",
       "0  These results suggest that the C1772T polymorp...           True   \n",
       "1  In our setting, @DISEASE$ among alcoholic indi...           True   \n",
       "2  MPO genotype GG is associated with @DISEASE$ i...           True   \n",
       "3  These three studies do not provide consistent ...           True   \n",
       "4  Our prospective findings suggest that individu...           True   \n",
       "\n",
       "                                    predicted_output  \n",
       "0  **True.** The sentence suggests that the C1772...  \n",
       "1  **True.** The sentence suggests that the prese...  \n",
       "2  **True.** The sentence suggests that the MPO g...  \n",
       "3  **True.** The sentence suggests that the prese...  \n",
       "4  **True.** The presence of the @GENE$ C282Y mut...  "
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gad_out_gemma = pd.read_csv(\"/home/016651544/gemma/test_df_gad_base_gemma_7b_without_FT.csv\")\n",
    "gad_out_gemma.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "6bcb4f3c-cd3d-4c6f-a9e8-242f9f370927",
   "metadata": {},
   "outputs": [],
   "source": [
    "# gad_out_gemma.predicted_output = gad_out_gemma.predicted_output.apply(lambda x: extract_boolean(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "id": "45c8f95f-edbb-464f-a4e6-b9b5333babd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def change_nan(row):\n",
    "    if pd.isna(row[2]) :# or row[2].lower() == \"nan\":\n",
    "        return not row[1]\n",
    "    elif \"true\" in row[2].lower():\n",
    "        return True\n",
    "    elif \"false\" in row[2].lower():\n",
    "        return False\n",
    "    else:\n",
    "        return not row[1]\n",
    "gad_out_gemma.predicted_output = gad_out_gemma.apply(lambda x: change_nan(x),axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "36174e7d-0b47-4d68-818b-1e1b67347b0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# euadr_out_gemma.predicted_output = euadr_out_gemma.predicted_output.apply(lambda x: process_predictions(x))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "id": "4cbb195f-736a-4c12-9dd9-3c66b74eb6c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predicted_output\n",
       "True     5239\n",
       "False      91\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gad_out_gemma.predicted_output.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "id": "4d24187e-a2db-4698-9d7a-120a843b5c38",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       FALSE       0.02      0.42      0.03        91\n",
      "        TRUE       0.98      0.52      0.68      5239\n",
      "\n",
      "    accuracy                           0.52      5330\n",
      "   macro avg       0.50      0.47      0.36      5330\n",
      "weighted avg       0.96      0.52      0.67      5330\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.47105499959097974, 0.49805194411022785, 0.35629486157001256)"
      ]
     },
     "execution_count": 192,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "# Convert actual and predicted columns to boolean\n",
    "predicted = gad_out_gemma.predicted_output.astype(str).str.upper()\n",
    "actual = gad_out_gemma.actual_output.astype(str).str.upper()\n",
    "report = classification_report(predicted,actual)\n",
    "\n",
    "print(report)\n",
    "# Calculate Precision, Recall, and F1 score\n",
    "precision = precision_score(actual, predicted, average='macro')\n",
    "recall = recall_score(actual, predicted, average='macro')\n",
    "f1 = f1_score(actual, predicted, average='macro')\n",
    "\n",
    "precision, recall, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "532308be-fdea-4c39-9254-96bd0446414d",
   "metadata": {},
   "source": [
    "### EUADR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d0d0ddd-6ffc-4179-879a-b6f33d95dd02",
   "metadata": {},
   "source": [
    "### llama2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "e04096d2-af83-413a-948e-cfb86d63d713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Our work supported @GENE$ genetic variants as ...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Especially, the SNP @GENE$ and its strongly as...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Especially, the SNP rs491347 and its strongly ...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The HBS1L-MYB intergenic region on chromosome ...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fetal haemoglobin (@GENE$) level modifies the ...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  actual_output  \\\n",
       "0  Our work supported @GENE$ genetic variants as ...           True   \n",
       "1  Especially, the SNP @GENE$ and its strongly as...           True   \n",
       "2  Especially, the SNP rs491347 and its strongly ...           True   \n",
       "3  The HBS1L-MYB intergenic region on chromosome ...          False   \n",
       "4  Fetal haemoglobin (@GENE$) level modifies the ...           True   \n",
       "\n",
       "  predicted_output  \n",
       "0             True  \n",
       "1             True  \n",
       "2             True  \n",
       "3             True  \n",
       "4             True  "
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "euadr_out = pd.read_csv(\"/home/016651544/llama2_euadr/test_df_base_llama_euadr.csv\")\n",
    "euadr_out.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "f4f0ea32-b98c-4a13-a20f-940a5f0b3954",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predicted_output\n",
       "True     346\n",
       "False      9\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def change_nan(row):\n",
    "    if pd.isna(row[2]) :# or row[2].lower() == \"nan\":\n",
    "        return not row[1]\n",
    "    elif \"true\" in row[2].lower():\n",
    "        return True\n",
    "    elif \"false\" in row[2].lower():\n",
    "        return False\n",
    "    else:\n",
    "        return not row[1]\n",
    "euadr_out.predicted_output = euadr_out.apply(lambda x: change_nan(x),axis=1)\n",
    "euadr_out.predicted_output.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "fcd6e8eb-16c1-499e-be9d-aac351f05b0f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       FALSE       0.00      0.00      0.00         9\n",
      "        TRUE       0.97      0.73      0.83       346\n",
      "\n",
      "    accuracy                           0.71       355\n",
      "   macro avg       0.48      0.37      0.42       355\n",
      "weighted avg       0.94      0.71      0.81       355\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.36560693641618497, 0.48282442748091603, 0.4161184210526316)"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "# Convert actual and predicted columns to boolean\n",
    "predicted = euadr_out.predicted_output.astype(str).str.upper()\n",
    "actual = euadr_out.actual_output.astype(str).str.upper()\n",
    "report = classification_report(predicted,actual)\n",
    "\n",
    "print(report)\n",
    "# Calculate Precision, Recall, and F1 score\n",
    "precision = precision_score(actual, predicted, average='macro')\n",
    "recall = recall_score(actual, predicted, average='macro')\n",
    "f1 = f1_score(actual, predicted, average='macro')\n",
    "\n",
    "precision, recall, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ee0b181-1f35-48fd-b1bf-57517f2276f0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf91e5b-e338-452e-95ef-cddccf7e2dfd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "64e6002c-c2da-4806-b686-28f0af3e2f4c",
   "metadata": {},
   "source": [
    "### GEMMA 2b EUADR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "6670fed2-6cab-487a-8998-eff962e0c810",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Our work supported @GENE$ genetic variants as ...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Especially, the SNP @GENE$ and its strongly as...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Especially, the SNP rs491347 and its strongly ...</td>\n",
       "      <td>True</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The HBS1L-MYB intergenic region on chromosome ...</td>\n",
       "      <td>False</td>\n",
       "      <td>TRUE\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fetal haemoglobin (@GENE$) level modifies the ...</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  actual_output  \\\n",
       "0  Our work supported @GENE$ genetic variants as ...           True   \n",
       "1  Especially, the SNP @GENE$ and its strongly as...           True   \n",
       "2  Especially, the SNP rs491347 and its strongly ...           True   \n",
       "3  The HBS1L-MYB intergenic region on chromosome ...          False   \n",
       "4  Fetal haemoglobin (@GENE$) level modifies the ...           True   \n",
       "\n",
       "  predicted_output  \n",
       "0           TRUE\\n  \n",
       "1           TRUE\\n  \n",
       "2           TRUE\\n  \n",
       "3           TRUE\\n  \n",
       "4              NaN  "
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "euadr_out_gemma_2b = pd.read_csv(\"/home/016651544/gemma/euadr/test_df_euadr_base_gemma_2b_without_FT.csv\")\n",
    "euadr_out_gemma_2b.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "cccaa653-30fe-4e03-a2b6-0b67b7d6f758",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predicted_output\n",
       "True     303\n",
       "False     52\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "euadr_out_gemma_2b.predicted_output = euadr_out_gemma_2b.apply(lambda x: change_nan(x),axis=1)\n",
    "euadr_out_gemma_2b.predicted_output.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "ce884c1b-3db0-4b8d-aed5-75dea5bf4d2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       FALSE       0.00      0.00      0.00        52\n",
      "        TRUE       0.80      0.69      0.74       303\n",
      "\n",
      "    accuracy                           0.59       355\n",
      "   macro avg       0.40      0.35      0.37       355\n",
      "weighted avg       0.68      0.59      0.63       355\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.3465346534653465, 0.40076335877862596, 0.37168141592920345)"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "# Convert actual and predicted columns to boolean\n",
    "predicted = euadr_out_gemma_2b.predicted_output.astype(str).str.upper()\n",
    "actual = euadr_out_gemma_2b.actual_output.astype(str).str.upper()\n",
    "report = classification_report(predicted,actual)\n",
    "print(report)\n",
    "# Calculate Precision, Recall, and F1 score\n",
    "precision = precision_score(actual, predicted, average='macro')\n",
    "recall = recall_score(actual, predicted, average='macro')\n",
    "f1 = f1_score(actual, predicted, average='macro')\n",
    "\n",
    "precision, recall, f1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38f8abe1-e336-470f-b7a9-d2bd905e6d71",
   "metadata": {},
   "source": [
    "### gemma 7b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "eea10ebd-b2b7-4baa-b3d5-19a438453188",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>actual_output</th>\n",
       "      <th>predicted_output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Our work supported @GENE$ genetic variants as ...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The sentence suggests that the genet...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Especially, the SNP @GENE$ and its strongly as...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The presence of strongly associated ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Especially, the SNP rs491347 and its strongly ...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The presence of strongly associated ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The HBS1L-MYB intergenic region on chromosome ...</td>\n",
       "      <td>False</td>\n",
       "      <td>**True.** The presence of a quantitative trait...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Fetal haemoglobin (@GENE$) level modifies the ...</td>\n",
       "      <td>True</td>\n",
       "      <td>**True.** The sentence suggests that the level...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  actual_output  \\\n",
       "0  Our work supported @GENE$ genetic variants as ...           True   \n",
       "1  Especially, the SNP @GENE$ and its strongly as...           True   \n",
       "2  Especially, the SNP rs491347 and its strongly ...           True   \n",
       "3  The HBS1L-MYB intergenic region on chromosome ...          False   \n",
       "4  Fetal haemoglobin (@GENE$) level modifies the ...           True   \n",
       "\n",
       "                                    predicted_output  \n",
       "0  **True.** The sentence suggests that the genet...  \n",
       "1  **True.** The presence of strongly associated ...  \n",
       "2  **True.** The presence of strongly associated ...  \n",
       "3  **True.** The presence of a quantitative trait...  \n",
       "4  **True.** The sentence suggests that the level...  "
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "euadr_out_gemma = pd.read_csv(\"/home/016651544/gemma/euadr/test_df_euadr_base_gemma_7b_without_FT.csv\")\n",
    "euadr_out_gemma.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "126dd905-896e-4a2c-b76d-cd963fcfd6ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "euadr_out_gemma.predicted_output = euadr_out_gemma.apply(lambda x: change_nan(x),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "d0485c81-7f4e-4df5-a0ed-7a57ed05441d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "predicted_output\n",
       "True     354\n",
       "False      1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "euadr_out_gemma.predicted_output.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "704d85ea-b7d2-48a0-a0ae-01075c0d1d71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False       0.00      0.00      0.00         1\n",
      "        True       1.00      0.74      0.85       354\n",
      "\n",
      "    accuracy                           0.74       355\n",
      "   macro avg       0.50      0.37      0.42       355\n",
      "weighted avg       0.99      0.74      0.85       355\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.3686440677966102, 0.49809160305343514, 0.42370129870129875)"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "# Convert actual and predicted columns to boolean\n",
    "predicted = euadr_out_gemma.predicted_output.astype(str)\n",
    "actual = euadr_out_gemma.actual_output.astype(str)\n",
    "report = classification_report(predicted,actual)\n",
    "print(report)\n",
    "# Calculate Precision, Recall, and F1 score\n",
    "precision = precision_score(actual, predicted, average='macro')\n",
    "recall = recall_score(actual, predicted, average='macro')\n",
    "f1 = f1_score(actual, predicted, average='macro')\n",
    "\n",
    "precision, recall, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be1f5342-1adb-499a-a86d-165a2845f5e3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c64edcf-0c3c-4672-aa03-cf5217c4f868",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9207e2d-b36a-4cc3-882f-047d3744304e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87509d53-dbe9-479b-9ba3-3fab8780d37b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "612bd2da-ecf3-4f2c-b4ad-5c893ca38dab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}