reviewing_info_extract_emphysema.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "327a9bfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import traceback\n",
    "import copy\n",
    "import pickle\n",
    "\n",
    "#Statistics\n",
    "from scipy.special import ndtri\n",
    "from statsmodels.stats.contingency_tables import mcnemar\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "import scipy.stats as stats #For Mann-Whitney U test\n",
    "\n",
    "import warnings #To stop pandas warnings \n",
    "warnings.simplefilter(action='ignore', category=Warning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f565aa1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Paths with folders containing subfolders named by each participant id. \n",
    "#These subfolders contain txt files and images of discrepancies reviewed by radiologists (information provided in txt files)\n",
    "path_noemph=os.getcwd()+'/no_emphysema_reviewed'\n",
    "path_emph=os.getcwd()+'/emphysema_reviewed'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8cf98639",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Read excel files with data\n",
    "mod=pd.read_excel(os.getcwd()+'/emphysema_exp_files'+\"\\\\moderate_manual.xlsx\")\n",
    "conf=pd.read_excel(os.getcwd()+'/emphysema_exp_files'+\"\\\\moderate_manual.xlsx\")\n",
    "adv=pd.read_excel(os.getcwd()+'/emphysema_exp_files'+\"\\\\advanced_manual.xlsx\")\n",
    "noemph=pd.read_excel(os.getcwd()+'/emphysema_exp_files'+\"\\\\noemphysema_manual.xlsx\")\n",
    "\n",
    "#We also need a folder named 'emph_csv' with REDCap exports, one for each degree of emphysema - see below"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "62e278ab",
   "metadata": {},
   "source": [
    "Function to check results of radiologists' review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7a87a407",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_information_of_review(path):\n",
    "    \n",
    "    'Gets the path of a folder with subfolders containing images and txt files of results of nodule review.'\n",
    "    'These results should be of the following format: \"nodule\"/\"no nodule\" and then description and a confidence score'\n",
    "    'It prints the participant_id, the txt file (with the slice number and if it is a FP or FN), the confidence score,'\n",
    "    'and a description of the finding given by the radiologists.'\n",
    "    'Returns dictionaries with participant id and nodule ids of findings belonging to each of the nodule/non-nodule'\n",
    "    'categories (two dictionaries for each category, one with FPs and one with FNs). Moreover, it returns 4 dictionaries,'\n",
    "    '2 containing the participant with the correct nodule ids for each of the FPs and FNs and 2 with the wrong ones.'\n",
    "    'Moreover, we get 4 more dictionaries, 2 containing only lymph nodes and 2 containing only nodule ids (FP and FN again).'\n",
    "    'At last, we get 4 dictionaries with non-nodule categories, 2 with FPs and 2 with FNs. Each of them has lung and non-lung findings.'\n",
    "    \n",
    "\n",
    "    uncertain=0 #Unsure of what the finding is\n",
    "    nodule_all=0 #Count all nodules, FPs, and FNs\n",
    "    total_files=0 #Total files\n",
    "    excluded=[] #Files not taken into account\n",
    "    tp_mistakes=0 #For TP accidentaly considered as discrepancies during review - happened probably only once\n",
    "    \n",
    "    \n",
    "    #All possible non-nodule categories - based on new definition   \n",
    "    fibr_scar_pleural=0 \n",
    "    other=0\n",
    "    \n",
    "    #FPs and FNs for non-nodule categories\n",
    "    fibr_FP=0\n",
    "    fibr_FN=0\n",
    "    other_FP=0\n",
    "    other_FN=0\n",
    "    \n",
    "    #Possible TPs (errors) for non-nodule categories\n",
    "    fibr_TP=0\n",
    "    other_TP=0\n",
    "    \n",
    "    \n",
    "    #Nodule categories\n",
    "    cal_nod=0\n",
    "    pleu_nod=0\n",
    "    other_nod=0\n",
    "    subgrou_nod=0\n",
    "    canc_nod=0\n",
    "    \n",
    "    atypical_triang=0 #This and the next are typically benign so less important if AI would miss them\n",
    "    peri_fissur=0\n",
    "    bronchperi=0 \n",
    "    \n",
    "    #TP (errors) for nodules\n",
    "    other_nod_TP=0\n",
    "    cal_TP=0\n",
    "    pleu_TP=0\n",
    "    subgrou_TP=0\n",
    "    canc_TP=0\n",
    "    \n",
    "    atypical_TP=0\n",
    "    peri_TP=0\n",
    "    bronchperi_TP=0\n",
    "    \n",
    "    #FPs and FNs for nodule categories\n",
    "    other_nod_FP=0\n",
    "    cal_FP=0\n",
    "    pleu_FP=0\n",
    "    subgrou_FP=0\n",
    "    canc_FP=0\n",
    "    other_nod_FN=0\n",
    "    cal_FN=0\n",
    "    pleu_FN=0\n",
    "    subgrou_FN=0\n",
    "    canc_FN=0\n",
    "    \n",
    "    atypical_FP=0\n",
    "    atypical_FN=0\n",
    "    peri_FP=0\n",
    "    peri_FN=0\n",
    "    bronchperi_FP=0\n",
    "    bronchperi_FN=0\n",
    "    \n",
    "    #Dictionaries to be filled participant_ids and nodule_ids that belong to a given category\n",
    "    atyp_FN={}\n",
    "    per_FN={}\n",
    "    bronchioperi_FN={}\n",
    "    pleural_FN={}\n",
    "    calcif_FN={}\n",
    "    sub_ground_FN={}\n",
    "    cancer_FN={}\n",
    "    other_nodules_FN={}\n",
    "    \n",
    "    other_nonodules_FN={}\n",
    "    fibrosis_FN={}\n",
    "    other_nonodules_FN_lung={}\n",
    "    other_nonodules_FN_nolung={}\n",
    "    \n",
    "    atyp_FP={}\n",
    "    per_FP={}\n",
    "    bronchioperi_FP={}\n",
    "    pleural_FP={}\n",
    "    calcif_FP={}\n",
    "    sub_ground_FP={}\n",
    "    cancer_FP={}\n",
    "    other_nodules_FP={}\n",
    "    \n",
    "    #Non-nodule categories\n",
    "    other_nonodules_FP={}\n",
    "    fibrosis_FP={}\n",
    "    other_nonodules_FP_lung={}\n",
    "    other_nonodules_FP_nolung={}\n",
    "\n",
    "    peri=0\n",
    "    \n",
    "    #Initialize empty dictionaries to keep track FP and FN slices\n",
    "    \n",
    "    #These are for both nodules (+lymph nodes) and non-nodules\n",
    "    dict_FP_correct={}\n",
    "    dict_FN_correct={}\n",
    "    dict_FP_wrong={}\n",
    "    dict_FN_wrong={}\n",
    "\n",
    "    #Only for lymph nodes\n",
    "    lymph_FN_correct={}\n",
    "    lymph_FP_wrong={}\n",
    "\n",
    "    #Only for nodules\n",
    "    nod_FN_correct={}\n",
    "    nod_FP_wrong={}\n",
    "    \n",
    "    \n",
    "    for dirpath, dirnames, filenames in os.walk(path): #Loop over folders and subfolders\n",
    "        for folder in dirnames: #For each folder (has participant name) in the above directory\n",
    "            for file in os.listdir(dirpath+'/'+folder): #For each file in the above folder\n",
    "\n",
    "                if file.endswith('.txt'): #If it's a txt print it (contains the review) along with the folder name (ID)\n",
    "                    print(dirpath,':',folder,':',file)\n",
    "\n",
    "                    with open(dirpath+'/'+folder+'/'+file) as f: #Read txt file\n",
    "                        lines = f.readlines()\n",
    "\n",
    "                    folder_pat=folder[:6] #keep only first 6 letters that correspond to participant id\n",
    "                        \n",
    "                    #Get confidence score - the only number in the text\n",
    "                    confidence=[num for line in lines for num in line if num.isdigit()] \n",
    "            \n",
    "                    if len(confidence)==1: #If there are more numbers it should be checked for errors\n",
    "                        print('Confidence is',int(confidence[0]))\n",
    "                    else:\n",
    "                        print(\"ERROR in confidence level of file\",file)\n",
    "\n",
    "                        \n",
    "                    no_nodules=[line for line in lines if 'no ' in line.lower()] #if this string in txt then no nodule\n",
    "\n",
    "                    if len(no_nodules)!=0: #Confirm that above non-empty list\n",
    "                        \n",
    "                        total_files=total_files+1 #Increase total number of files taken into account\n",
    "                        print('Finding is NOT a nodule (or it is a lymph node)')\n",
    "                        \n",
    "                        information=[info.split('nodule',1) for info in no_nodules][0] #split only on first occurence \n",
    "                        details=[elem for elem in information if len(elem)>5] #Since we may also have an element with 'no'\n",
    "                        \n",
    "                        if len(details)>0: #If we have a description of finding\n",
    "\n",
    "                            #Perform some replacements to delete '\\n','-', empty spaces and confidence score\n",
    "                            detailed_info=details[0].replace('-', '').replace('\\n','').replace(confidence[0],'').strip()\n",
    "\n",
    "                            print(detailed_info.replace(':',''), 'was written in the txt file')\n",
    "                            \n",
    "#                             if int(confidence[0])>=4: #Only take into account confident predictions\n",
    "                                \n",
    "                            #Below categories for non-nodules\n",
    "\n",
    "                            #For atypical and perifissural we noted them as non-nodules while actually want to be detected\n",
    "                            #We will consider them as nodules - that's why we changed to 'fn_correct', 'fp_wrong' for them\n",
    "                            if ('atypical' in detailed_info.lower() or 'triangular' in detailed_info.lower()): \n",
    "\n",
    "                                print('atypical/triangular lymph node')\n",
    "                                atypical_triang=atypical_triang+1 #Count them\n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')\n",
    "                                    atypical_TP=atypical_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower() and 'fn' not in file.lower():\n",
    "                                    atypical_FP=atypical_FP+1 #Count them\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_wrong: #Add participant id to dictionary - list only with slice numbers\n",
    "                                        dict_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))] \n",
    "                                    else:\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    #Atypical lymph nodes added to nodule group\n",
    "                                    # if folder_pat not in lymph_FP_wrong: #Add it to dictionary with lymph nodes\n",
    "                                    #     lymph_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    # else:\n",
    "                                    #     lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    if folder_pat not in nod_FP_wrong:\n",
    "                                        nod_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        nod_FP_wrong[folder_pat]=nod_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "        \n",
    "\n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower(): #AI manually extracted image names have different naming conventions\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        # lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        nod_FP_wrong[folder_pat]=nod_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "    \n",
    "                                    if int(folder_pat) in atyp_FP: #Add it to corresponding category dictionary\n",
    "                                        atyp_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                    else:\n",
    "                                        atyp_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                elif 'fn' in file.lower(): #Similarly as above for FNs\n",
    "                                    atypical_FN=atypical_FN+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_correct:\n",
    "                                        dict_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    # if folder_pat not in lymph_FN_correct:\n",
    "                                    #     lymph_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    # else:\n",
    "                                    #     lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]    \n",
    "                                    if folder_pat not in nod_FN_correct:\n",
    "                                        nod_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        nod_FN_correct[folder_pat]=nod_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]    \n",
    "                                       \n",
    "                                        \n",
    "                                    if '_fn' in file.lower(): #To address some issues with manually created images\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        # lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        nod_FN_correct[folder_pat]=nod_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "            \n",
    "                                    if int(folder_pat) in atyp_FN:\n",
    "                                        atyp_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                    else:\n",
    "                                        atyp_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                else:\n",
    "                                    print('ERROR in atypical')\n",
    "\n",
    "\n",
    "                            elif ('periphysural' in detailed_info.lower() or 'fissural' in detailed_info.lower() \n",
    "                                  or 'fiscu' in detailed_info.lower() or 'pfn' in detailed_info.lower()):\n",
    "\n",
    "                                print('periphysural/fissural/PFN')\n",
    "                                peri_fissur=peri_fissur+1  \n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')\n",
    "                                    peri_TP=peri_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower() and 'fn' not in file.lower():\n",
    "                                    peri_FP=peri_FP+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_wrong:\n",
    "                                        dict_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                                                                                 \n",
    "                                    if folder_pat not in lymph_FP_wrong:\n",
    "                                        lymph_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                        \n",
    "                        \n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in per_FP:\n",
    "                                        per_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                    else:\n",
    "                                        per_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                elif 'fn' in file.lower():\n",
    "                                    peri_FN=peri_FN+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_correct:\n",
    "                                        dict_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if folder_pat not in lymph_FN_correct:\n",
    "                                        lymph_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if '_fn' in file.lower():\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "                                    if int(folder_pat) in per_FN:\n",
    "                                        per_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                    else:\n",
    "                                        per_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                else:\n",
    "                                    print('ERROR in periphysural')\n",
    "\n",
    "\n",
    "                            elif ('fibrosis' in detailed_info.lower() or 'scar' in detailed_info.lower() \n",
    "                                  or 'thick' in detailed_info.lower()):\n",
    "\n",
    "                                print('fibrosis/scar/pleural thickening')\n",
    "                                fibr_scar_pleural=fibr_scar_pleural+1 \n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')\n",
    "                                    fibr_TP=fibr_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower() and 'fn' not in file.lower():\n",
    "                                    fibr_FP=fibr_FP+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_correct:\n",
    "                                        dict_FP_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FP_correct[folder_pat]=dict_FP_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                        dict_FP_correct[folder_pat]=dict_FP_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in fibrosis_FP:\n",
    "                                        fibrosis_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                    else:\n",
    "                                        fibrosis_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                elif 'fn' in file.lower():\n",
    "                                    fibr_FN=fibr_FN+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_wrong:\n",
    "                                        dict_FN_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FN_wrong[folder_pat]=dict_FN_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    if '_fn' in file.lower():\n",
    "                                        dict_FN_wrong[folder_pat]=dict_FN_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in fibrosis_FN:\n",
    "                                        fibrosis_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                    else:\n",
    "                                        fibrosis_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                else:\n",
    "                                    print('ERROR IN fibrosis')\n",
    "\n",
    "\n",
    "                            elif ('bronch' in detailed_info.lower() or 'peribronchial' in detailed_info.lower() or 'pbv' in detailed_info.lower()):\n",
    "\n",
    "                                print('peribronchial/bronchiovascular')\n",
    "                                bronchperi=bronchperi+1\n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')\n",
    "                                    bronchperi_TP=bronchperi_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "\n",
    "                                #In case peribronchial lymph nodes are excluded from analysis, activate below\n",
    "                                # elif 'peribronchial' in detailed_info.lower() or 'pbv' in detailed_info.lower():\n",
    "                                #     print('peribronchial that will not be considered - Need to confirm manually that vol<100mm3.')\n",
    "                                #     print(folder,file)\n",
    "                                #     nodule_all=nodule_all+1\n",
    "                                #     peri=peri+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower() and 'fn' not in file.lower():\n",
    "                                    bronchperi_FP=bronchperi_FP+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_wrong:\n",
    "                                        dict_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "\n",
    "                                    if folder_pat not in lymph_FP_wrong:\n",
    "                                        lymph_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                        \n",
    "                        \n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in bronchioperi_FP:\n",
    "                                        bronchioperi_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                    else:\n",
    "                                        bronchioperi_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                elif 'fn' in file.lower():\n",
    "                                    bronchperi_FN=bronchperi_FN+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_correct:\n",
    "                                        dict_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if folder_pat not in lymph_FN_correct:\n",
    "                                        lymph_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if '_fn' in file.lower():\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if int(folder_pat) in bronchioperi_FN:\n",
    "                                        bronchioperi_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                    else:\n",
    "                                        bronchioperi_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "                                else:\n",
    "                                    print('ERROR IN peribronchial')\n",
    "\n",
    "\n",
    "                            elif 'lymph' in detailed_info.lower(): \n",
    "                                #Seperate from above since sometimes it may start with 'fissural lymph node'\n",
    "                                #and therefore being a different category - this checked first in the 'if' above\n",
    "\n",
    "                                print('atypical/triangular lymph node')\n",
    "                                atypical_triang=atypical_triang+1\n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')\n",
    "                                    atypical_TP=atypical_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower() and 'fn' not in file.lower():\n",
    "                                    atypical_FP=atypical_FP+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_wrong:\n",
    "                                        dict_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "\n",
    "                                    # if folder_pat not in lymph_FP_wrong:\n",
    "                                    #     lymph_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    # else:\n",
    "                                    #     lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    if folder_pat not in nod_FP_wrong:\n",
    "                                        nod_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        nod_FP_wrong[folder_pat]=nod_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                        \n",
    "\n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                        dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        # lymph_FP_wrong[folder_pat]=lymph_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        nod_FP_wrong[folder_pat]=nod_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "                                    \n",
    "                                    if int(folder_pat) in atyp_FP:\n",
    "                                        atyp_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                    else:\n",
    "                                        atyp_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                elif 'fn' in file.lower():\n",
    "                                    atypical_FN=atypical_FN+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_correct:\n",
    "                                        dict_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                   \n",
    "                                    # if folder_pat not in lymph_FN_correct:\n",
    "                                    #     lymph_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    # else:\n",
    "                                    #     lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    if folder_pat not in nod_FN_correct:\n",
    "                                        nod_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        nod_FN_correct[folder_pat]=nod_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]    \n",
    "                                 \n",
    "\n",
    "\n",
    "                                    if '_fn' in file.lower():\n",
    "                                        dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        # lymph_FN_correct[folder_pat]=lymph_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                        nod_FN_correct[folder_pat]=nod_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "                                    if int(folder_pat) in atyp_FN:\n",
    "                                        atyp_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                    else:\n",
    "                                        atyp_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                else:\n",
    "                                    print('ERROR IN atypical')\n",
    "\n",
    "\n",
    "                            else: #Here when we have description but it's other non-nods (eg. atelectasis)\n",
    "                                print('other')\n",
    "                                other=other+1\n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')\n",
    "                                    other_TP=other_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "                                    nodule_all=nodule_all+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower() and 'fn' not in file.lower():\n",
    "                                    other_FP=other_FP+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_correct:\n",
    "                                        dict_FP_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FP_correct[folder_pat]=dict_FP_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                        dict_FP_correct[folder_pat]=dict_FP_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in other_nonodules_FP:\n",
    "                                        other_nonodules_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "\n",
    "                                        res=detailed_info.lower() #Get information about type of non-nodule\n",
    "\n",
    "                                        if 'atele' in res or 'infe' in res or 'conso' in res or 'mucu' in res or 'vess' in res: #Without parenthesis always get in!\n",
    "                                            try: #Lung findings\n",
    "                                                other_nonodules_FP_lung[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FP_lung[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        elif 'bone' in res or 'osis' in res or 'fat' in res or 'tiss' in res: #Non-lung\n",
    "                                            try:\n",
    "                                                other_nonodules_FP_nolung[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FP_nolung[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "                                        else:\n",
    "                                            print(\"Cannot classify it as lung/non-lung based on description\")\n",
    "\n",
    "                                    else:\n",
    "                                        other_nonodules_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        res=detailed_info.lower()\n",
    "\n",
    "                                        if 'atele' in res or 'infe' in res or 'conso' in res or 'mucu' in res or 'vess' in res:\n",
    "                                            try:\n",
    "                                                other_nonodules_FP_lung[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FP_lung[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        elif 'bone' in res or 'osis' in res or 'fat' in res or 'tiss' in res:\n",
    "                                            try:\n",
    "                                                other_nonodules_FP_nolung[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FP_nolung[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "                                        else:\n",
    "                                            print(\"Cannot classify it as lung/non-lung based on description\")\n",
    "                                        \n",
    "                                elif 'fn' in file.lower():\n",
    "                                    other_FN=other_FN+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_wrong:\n",
    "                                        dict_FN_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:  \n",
    "                                        dict_FN_wrong[folder_pat]=dict_FN_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if '_fn' in file.lower():\n",
    "                                        dict_FN_wrong[folder_pat]=dict_FN_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in other_nonodules_FN:\n",
    "                                        other_nonodules_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "\n",
    "                                        res=detailed_info.lower()\n",
    "\n",
    "                                        if 'atele' in res or 'infe' in res or 'conso' in res or 'mucu' in res or 'vess' in res:\n",
    "                                            try:\n",
    "                                                other_nonodules_FN_lung[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FN_lung[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        elif 'bone' in res or 'osis' in res or 'fat' in res or 'tiss' in res:\n",
    "\n",
    "                                            try:\n",
    "                                                other_nonodules_FN_nolung[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FN_nolung[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "                                        else:\n",
    "                                            print(\"Cannot classify it as lung/non-lung based on description\")\n",
    "\n",
    "                                    else:\n",
    "                                        other_nonodules_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        res=detailed_info.lower()\n",
    "\n",
    "                                        if 'atele' in res or 'infe' in res or 'conso' in res or 'mucu' in res or 'vess' in res:\n",
    "                                            try:\n",
    "                                                other_nonodules_FN_lung[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FN_lung[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        elif 'bone' in res or 'osis' in res or 'fat' in res or 'tiss' in res:\n",
    "                                            try:\n",
    "                                                other_nonodules_FN_nolung[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            except:\n",
    "                                                other_nonodules_FN_nolung[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "                                        else:\n",
    "                                            print(\"Cannot classify it as lung/non-lung based on description\")\n",
    "\n",
    "                                else:\n",
    "                                    print('ERROR IN other')\n",
    "                                       \n",
    "\n",
    "                                        \n",
    "#                             else:\n",
    "#                                 print('Low confidence <=3 - excluded from analysis')\n",
    "#                                 excluded.append(folder+':'+file)\n",
    "                                \n",
    "                                \n",
    "                        else: #If we don't have a description of the finding - we add those 'non-nodule' in 'other'\n",
    "                            \n",
    "#                             if int(confidence[0])>=4:\n",
    "\n",
    "                                print('No information for non-nodule file',dirpath,':',folder,':',file)\n",
    "                                other=other+1\n",
    "\n",
    "                                if 'tp' in file.lower():\n",
    "                                    print('This will not be considered')                    \n",
    "                                    other_TP=other_TP+1\n",
    "                                    tp_mistakes=tp_mistakes+1\n",
    "\n",
    "                                elif 'fp' in file.lower() or 'ai' in file.lower():\n",
    "                                    other_FP=other_FP+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FP_correct:\n",
    "                                        dict_FP_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FP_correct[folder_pat]=dict_FP_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                        dict_FP_correct[folder_pat]=dict_FP_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in other_nonodules_FP:\n",
    "                                        other_nonodules_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                    else:\n",
    "                                        other_nonodules_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        \n",
    "                                elif 'fn' in file.lower():\n",
    "                                    other_FN=other_FN+1\n",
    "                                    \n",
    "                                    if folder_pat not in dict_FN_wrong:\n",
    "                                        dict_FN_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    else:\n",
    "                                        dict_FN_wrong[folder_pat]=dict_FN_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                        \n",
    "                                    if '_fn' in file.lower():\n",
    "                                        dict_FN_wrong[folder_pat]=dict_FN_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    if int(folder_pat) in other_nonodules_FN:\n",
    "                                        other_nonodules_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                    else:\n",
    "                                        other_nonodules_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "                                        \n",
    "                                else:\n",
    "                                    print('ERROR IN other')\n",
    "\n",
    "\n",
    "#                             else:\n",
    "#                                 print('Low confidence <=3 - excluded from analysis')\n",
    "#                                 excluded.append(folder+':'+file)\n",
    "                                \n",
    "                                \n",
    "                        print('\\n')\n",
    "\n",
    "                                \n",
    "\n",
    "                    else: #If it's not a non-nodule, it will be either 'unsure' or 'nodule'\n",
    "                        \n",
    "                        total_files=total_files+1 #Increase total number of files taken into account\n",
    "                        unsure=[line for line in lines if 'unsure' in line.lower()] #If line contains 'unsure'\n",
    "                        \n",
    "                        if len(unsure)!=0: #If it's not empty, then unsure about finding\n",
    "                            print('Unsure about what this finding is')\n",
    "                            uncertain=uncertain+1\n",
    "#                             print('Low confidence <=3 - excluded from analysis')\n",
    "#                             excluded.append(folder+':'+file)\n",
    "                            \n",
    "                        else: #Otherwise it's a nodule\n",
    "                            print('Finding is a nodule')\n",
    "                            \n",
    "                            nodules=[line for line in lines if 'nodule' in line.lower()] #Confirm 'nodule' in line\n",
    "                            information=[info.split('nodule',1) for info in nodules][0] #split only on first occurence\n",
    "                            details=[elem for elem in information if len(elem)>5] #similar as above\n",
    "                            \n",
    "                            if len(details)>0: #If we have a description of finding \n",
    "                                    \n",
    "                                    #Clean as above plus ':'\n",
    "                                    nod_desc=details[0].lower().replace('nodule','').replace('-', '').replace('\\n','').replace(confidence[0],'').replace(':','').strip()\n",
    "                                    print(nod_desc)\n",
    "            \n",
    "#                                     if int(confidence[0])>=4:\n",
    "                    \n",
    "                                    #Below categories for nodules\n",
    "\n",
    "                                    if 'calc' in nod_desc:\n",
    "                                        cal_nod=cal_nod+1\n",
    "                                        print('Calcified nodule added')\n",
    "\n",
    "                                        if 'tp' in file.lower():\n",
    "                                            print('This will not be considered')\n",
    "                                            cal_TP=cal_TP+1\n",
    "                                            tp_mistakes=tp_mistakes+1\n",
    "\n",
    "                                        elif 'fp' in file.lower() or 'ai' in file.lower():\n",
    "                                            cal_FP=cal_FP+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in calcif_FP:\n",
    "                                                calcif_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            else:\n",
    "                                                calcif_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        elif 'fn' in file.lower():\n",
    "                                            cal_FN=cal_FN+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in calcif_FN:\n",
    "                                                calcif_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            else:\n",
    "                                                calcif_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        else:\n",
    "                                            print('ERROR IN calcified')\n",
    "\n",
    "\n",
    "                                    elif 'pleu' in nod_desc:\n",
    "                                        pleu_nod=pleu_nod+1\n",
    "                                        print('pleural nodule added')\n",
    "\n",
    "                                        if 'tp' in file.lower():\n",
    "                                            print('This will not be considered')\n",
    "                                            pleu_TP=pleu_TP+1\n",
    "                                            tp_mistakes=tp_mistakes+1\n",
    "\n",
    "                                        elif 'fp' in file.lower() or 'ai' in file.lower():\n",
    "                                            pleu_FP=pleu_FP+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in pleural_FP:\n",
    "                                                pleural_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            else:\n",
    "                                                pleural_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        elif 'fn' in file.lower():\n",
    "                                            pleu_FN=pleu_FN+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in pleural_FN: \n",
    "                                                pleural_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            else:\n",
    "                                                pleural_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        else:\n",
    "                                            print('ERROR IN pleural nodules')\n",
    "\n",
    "\n",
    "                                    elif 'sub' in nod_desc or 'grou' in nod_desc:\n",
    "                                        subgrou_nod=subgrou_nod+1\n",
    "                                        print('subsolid/ground glass nodule added')\n",
    "\n",
    "                                        if 'tp' in file.lower():\n",
    "                                            print('This will not be considered')\n",
    "                                            subgrou_TP=subgrou_TP+1\n",
    "                                            tp_mistakes=tp_mistakes+1\n",
    "\n",
    "                                        elif 'fp' in file.lower() or 'ai' in file.lower():\n",
    "                                            subgrou_FP=subgrou_FP+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in sub_ground_FP:\n",
    "                                                sub_ground_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            else:\n",
    "                                                sub_ground_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                        elif 'fn' in file.lower():\n",
    "                                            subgrou_FN=subgrou_FN+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in sub_ground_FN:\n",
    "                                                sub_ground_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            else:\n",
    "                                                sub_ground_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        else:\n",
    "                                            print('ERROR IN subsolid/ground class nodules')\n",
    "\n",
    "\n",
    "                                    elif 'canc' in nod_desc:\n",
    "                                        canc_nod=canc_nod+1\n",
    "                                        print('cancer added')\n",
    "\n",
    "                                        if 'tp' in file.lower():\n",
    "                                            print('This will not be considered')\n",
    "                                            canc_TP=canc_TP+1\n",
    "                                            tp_mistakes=tp_mistakes+1\n",
    "\n",
    "                                        elif 'fp' in file.lower() or 'ai' in file.lower():\n",
    "                                            canc_FP=canc_FP+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in cancer_FP:\n",
    "                                                cancer_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                            else:\n",
    "                                                cancer_FP[int(folder_pat)]=[file.lower().split('fp')[0]]                                           \n",
    "\n",
    "                                        elif 'fn' in file.lower():\n",
    "                                            canc_FN=canc_FN+1\n",
    "                                            \n",
    "                                            if int(folder_pat) in cancer_FN:\n",
    "                                                cancer_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                            else:\n",
    "                                                cancer_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                        else:\n",
    "                                            print('ERROR IN cancer')\n",
    "\n",
    "                                                \n",
    "#                                     else:\n",
    "#                                         print('Low confidence <=3 - excluded from analysis')\n",
    "#                                         excluded.append(folder+':'+file)\n",
    "                        \n",
    "                        \n",
    "                            else:\n",
    "                                \n",
    "#                                 if int(confidence[0])>=4:\n",
    "                                    \n",
    "                                    other_nod=other_nod+1\n",
    "                                    print('No information for file with nodule:',dirpath,':',folder,':',file)\n",
    "\n",
    "\n",
    "                                    if 'tp' in file.lower():\n",
    "                                        print('This will not be considered')\n",
    "                                        other_nod_TP=other_nod_TP+1\n",
    "                                        tp_mistakes=tp_mistakes+1\n",
    "\n",
    "                                    elif 'fp' in file.lower() or 'ai' in file.lower():\n",
    "                                        other_nod_FP=other_nod_FP+1\n",
    "                                        \n",
    "                                        if int(folder_pat) in other_nodules_FP:\n",
    "                                            other_nodules_FP[int(folder_pat)].append(file.lower().split('fp')[0])\n",
    "                                        else:     \n",
    "                                            other_nodules_FP[int(folder_pat)]=[file.lower().split('fp')[0]]\n",
    "\n",
    "                                    elif 'fn' in file.lower():\n",
    "                                        other_nod_FN=other_nod_FN+1\n",
    "                                        \n",
    "                                        if int(folder_pat) in other_nodules_FN:\n",
    "                                            other_nodules_FN[int(folder_pat)].append(file.lower().split('fn')[0])\n",
    "                                        else:\n",
    "                                            other_nodules_FN[int(folder_pat)]=[file.lower().split('fn')[0]]\n",
    "\n",
    "                                    else:\n",
    "                                        print('ERROR IN other')\n",
    "                                        \n",
    "                                        \n",
    "#                                 else:\n",
    "#                                     print('Low confidence <=3 - excluded from analysis')\n",
    "#                                     excluded.append(folder+':'+file)\n",
    "                                    \n",
    "                                    \n",
    "                            \n",
    "#                             if int(confidence[0])>=4:\n",
    "                            \n",
    "                            nodule_all=nodule_all+1                                      \n",
    "\n",
    "                            if 'fn' in file.lower() and 'fp' not in file.lower(): \n",
    "                            #Ensure that it was FN - second condition to confirm it\n",
    "                        \n",
    "                                if folder_pat not in dict_FN_correct:\n",
    "                                    dict_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                else:\n",
    "                                    #First letters pick up the slice number\n",
    "                                    dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                    \n",
    "                                    \n",
    "                                if folder_pat not in nod_FN_correct:\n",
    "                                    nod_FN_correct[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                else:\n",
    "                                    nod_FN_correct[folder_pat]=nod_FN_correct[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]    \n",
    "                                    \n",
    "                                    \n",
    "                                if '_fn' in file.lower():\n",
    "                                    dict_FN_correct[folder_pat]=dict_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    nod_FN_correct[folder_pat]=nod_FN_correct[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "                            elif ('fp' in file.lower() or 'ai' in file.lower()) and 'fn' not in file.lower():\n",
    "                               \n",
    "                                if folder_pat not in dict_FP_wrong:\n",
    "                                    dict_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                else:\n",
    "                                    dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                    \n",
    "                                if folder_pat not in nod_FP_wrong:\n",
    "                                    nod_FP_wrong[folder_pat]=[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                                else:\n",
    "                                    nod_FP_wrong[folder_pat]=nod_FP_wrong[folder_pat]+[int(''.join([x for x in file[:5] if x.isdigit()]))]\n",
    "                        \n",
    "\n",
    "                                if 'ai' in file.lower() and 'fp' not in file.lower():\n",
    "                                    dict_FP_wrong[folder_pat]=dict_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "                                    nod_FP_wrong[folder_pat]=nod_FP_wrong[folder_pat][:-1]+[int(''.join([x for x in file[-7:-4] if x.isdigit()]))]\n",
    "\n",
    "                        print('\\n')\n",
    "\n",
    "                                                                                                 \n",
    "    print('Num of uncertainties',uncertain)\n",
    "    print('\\n')\n",
    "        \n",
    "    print(\"From nodules, there were {} FP, {} FN, and {} TP calcified nodules\".format(cal_FP,cal_FN,cal_TP))    \n",
    "    print(\"From nodules, there were {} FP, {} FN, and {} TP pleural nodules\".format(pleu_FP,pleu_FN,pleu_TP))\n",
    "    print(\"From nodules, there were {} FP, {} FN, and {} TP 'other' nodules\".format(other_nod_FP,other_nod_FN,other_nod_TP))\n",
    "    print(\"From nodules, there were {} FP, {} FN, and {} TP subsolid/ground class nodules\".format(subgrou_FP,subgrou_FN,subgrou_TP))\n",
    "    print(\"From nodules, there were {} FP, {} FN, and {} TP cancer cases\".format(canc_FP,canc_FN,canc_TP))\n",
    "    print(\"There are {} FP, {} FN, and {} TP atypical PFN and/or triangular lymph nodes\".format(atypical_FP,atypical_FN,atypical_TP))\n",
    "    print(\"There are {} FP, {} FN, and {} TP periphysural/fissural/PFN\".format(peri_FP,peri_FN,peri_TP))\n",
    "    print(\"There are {} FP, {} FN, and {} TP bronchiovascular lymph nodes\".format(bronchperi_FP,bronchperi_FN,bronchperi_TP))\n",
    "    print(\"There are {} peribronchial (excluded) lymph nodes - both FP and FN\".format(peri))\n",
    "    print('\\n')\n",
    "\n",
    "    print(\"There are {} FP, {} FN, and {} TP fibrosis/scar/pleural thickening\".format(fibr_FP,fibr_FN,fibr_TP))\n",
    "    print(\"There are {} FP, {} FN, and {} TP other findings (bone, tissue, mucus, arthosis, vessel, consolidation, infection, fat, atelectasis, etc. )\".format(other_FP,other_FN,other_TP))\n",
    "    print('\\n')\n",
    "\n",
    "    print(\"Total number of files is \",total_files)\n",
    "    print(\"From those, there were {} files excluded due to low confidence <=3 and their names are: {}\".format(len(excluded),excluded))\n",
    "\n",
    "    #Confirm that non-nodules found properly\n",
    "    assert fibr_scar_pleural==fibr_FP+fibr_FN+fibr_TP\n",
    "\n",
    "    #Confirm that nodules found properly\n",
    "    assert cal_nod==cal_FP+cal_FN+cal_TP\n",
    "    assert pleu_nod==pleu_FP+pleu_FN+pleu_TP\n",
    "    assert subgrou_nod==subgrou_FP+subgrou_FN+subgrou_TP\n",
    "    assert canc_nod==canc_FP+canc_FN+canc_TP\n",
    "    \n",
    "    assert atypical_triang==atypical_FP+atypical_FN+atypical_TP\n",
    "    assert peri_fissur==peri_FP+peri_FN+peri_TP\n",
    "    # assert bronchperi==bronchperi_FP+bronchperi_FN+bronchperi_TP\n",
    "    \n",
    "    #Lists with nodule findings moved from non-nodule categories initially \n",
    "    assert nodule_all==cal_nod+pleu_nod+other_nod+subgrou_nod+canc_nod+atypical_triang+peri_fissur+bronchperi\n",
    "    \n",
    "    return (atyp_FN,per_FN,pleural_FN,calcif_FN,sub_ground_FN,cancer_FN,other_nodules_FN,other_nonodules_FN,fibrosis_FN,\n",
    "            bronchioperi_FN,atyp_FP,per_FP,pleural_FP,calcif_FP,sub_ground_FP,cancer_FP,other_nodules_FP,\n",
    "            other_nonodules_FP,fibrosis_FP,bronchioperi_FP,dict_FP_correct,dict_FP_wrong,dict_FN_correct,dict_FN_wrong,\n",
    "            lymph_FP_wrong,lymph_FN_correct, nod_FP_wrong, nod_FN_correct,\n",
    "            other_nonodules_FN_lung,other_nonodules_FN_nolung, other_nonodules_FP_lung,other_nonodules_FP_nolung)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "99e25b2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "### All confidence levels were included for now - in paper just mentioned those with <=3\n",
    "#unsure for 1 finding - nothing reported - included after reviewed again and category reported\n",
    "\n",
    "# Total files excluded in noemphysema due to low confidence <=3 are 13 (14 with last review) and their names are:\n",
    "#['199391:199391_AI1_272.txt', '225858:368fpAIlastfinal_gray0.txt', '369762:228fnlastfinal_gray0.txt', \n",
    "#'429703:132fnlastfinal_gray0.txt','435703:129fpAIlastfinal_gray0.txt', '591162:123fnlastfinal_gray0.txt', \n",
    "#'673634:095fnlastfinal_gray0.txt','673634:096fnlastfinal_gray0.txt', '673634:138fnlastfinal_gray0.txt',\n",
    "#'673634:173fnlastfinal_gray0.txt', '673634:287fpAIlastfinal_gray0.txt', '845334:845334_ai6_265.txt', \n",
    "#'951248:284fpAIlastfinal_gray00.txt']\n",
    "#from those there are 6 nodules, 5 lymph nodes and 1 non-nodules (1 case unsure)\n",
    "\n",
    "# Toal files excluded in emphysema due to low confidence <=3 are 3 and their names are: \n",
    "#['282528:330fpAIlastfinal_gray0.txt', '609065:609065_112fp.txt', '991277:251fpAIlastfinal_gray0.txt']\n",
    "#from those there are 1 nodule and 2 non-nodules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "733d41ad",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# %%capture cap --no-stderr\n",
    "\n",
    "(atyp_FN_noemph,per_FN_noemph,pleural_FN_noemph,calcif_FN_noemph,sub_ground_FN_noemph,cancer_FN_noemph,\n",
    " other_nodules_FN_noemph,other_nonodules_FN_noemph,fibrosis_FN_noemph,bronchioperi_FN_noemph,atyp_FP_noemph,per_FP_noemph,\n",
    " pleural_FP_noemph,calcif_FP_noemph,sub_ground_FP_noemph,cancer_FP_noemph,other_nodules_FP_noemph,other_nonodules_FP_noemph,\n",
    " fibrosis_FP_noemph,bronchioperi_FP_noemph,dict_FP_correct_noemph,dict_FP_wrong_noemph,dict_FN_correct_noemph,\n",
    " dict_FN_wrong_noemph,lymph_FP_wrong_noemph,lymph_FN_correct_noemph, nod_FP_wrong_noemph, nod_FN_correct_noemph,\n",
    " other_nonodules_FN_lung_noemph,other_nonodules_FN_nolung_noemph,\n",
    " other_nonodules_FP_lung_noemph,other_nonodules_FP_nolung_noemph)=show_information_of_review(path_noemph)\n",
    "\n",
    "#If we want to save the cell output to txt use below and activate above\n",
    "# with open('no_emph_review.txt', 'w') as f:\n",
    "#     f.write(cap.stdout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "67f4c35a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%capture cap --no-stderr\n",
    "\n",
    "(atyp_FN_emph,per_FN_emph,pleural_FN_emph,calcif_FN_emph,sub_ground_FN_emph,cancer_FN_emph,other_nodules_FN_emph,\n",
    " other_nonodules_FN_emph,fibrosis_FN_emph,bronchioperi_FN_emph,atyp_FP_emph,per_FP_emph,pleural_FP_emph,calcif_FP_emph,\n",
    " sub_ground_FP_emph,cancer_FP_emph,other_nodules_FP_emph,other_nonodules_FP_emph,fibrosis_FP_emph,bronchioperi_FP_emph,\n",
    " dict_FP_correct_emph,dict_FP_wrong_emph,dict_FN_correct_emph,dict_FN_wrong_emph,\n",
    " lymph_FP_wrong_emph,lymph_FN_correct_emph, nod_FP_wrong_emph, nod_FN_correct_emph,\n",
    " other_nonodules_FN_lung_emph,other_nonodules_FN_nolung_emph,\n",
    " other_nonodules_FP_lung_emph,other_nonodules_FP_nolung_emph)=show_information_of_review(path_emph)\n",
    "\n",
    "# with open('emph_review.txt', 'w') as f:\n",
    "#     f.write(cap.stdout) "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d0011253",
   "metadata": {},
   "source": [
    "### Convert slices to ids - Use manually checked annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3120501a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Manual corrections in dictionaries to be used\n",
    "\n",
    "# #Non-emphysema\n",
    "# print(dict_FP_correct_noemph)\n",
    "# print(dict_FP_wrong_noemph)\n",
    "# print(dict_FN_correct_noemph)\n",
    "# print(dict_FN_wrong_noemph)\n",
    "\n",
    "# print(\"\\n\")\n",
    "\n",
    "# #Emphysema\n",
    "# print(dict_FP_correct_emph)\n",
    "# print(dict_FP_wrong_emph)\n",
    "# print(dict_FN_correct_emph)\n",
    "# print(dict_FN_wrong_emph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0cb73015",
   "metadata": {},
   "outputs": [],
   "source": [
    "mod=mod[:28] #These are the indices in df of those with moderate emphysema (if only nodules [:13])\n",
    "mod=mod.reset_index(drop=True) #Reset indices\n",
    "\n",
    "conf=conf[30:38] #Get patients with confluent emphysema from df (it contains both mod+conf patients) (if only nodules [33:])\n",
    "conf=conf[conf['participant_id']!=592863]\n",
    "conf=conf.reset_index(drop=True) #Reset indices\n",
    "\n",
    "adv=adv[:5] #Select participants with advanced emphysema\n",
    "adv=adv.reset_index(drop=True) #Reset indices\n",
    "\n",
    "noemph=noemph[:88] #Select participants with nodules and without emphysema (if only nodules [:67])\n",
    "noemph=noemph.reset_index(drop=True) #Reset indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "449a2666",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set all columns with 300+ nodules to nan since they will not be used\n",
    "mod['300+ tp'] = np.nan\n",
    "mod['300+ fp']=np.nan\n",
    "mod['300+ fn']=np.nan\n",
    "\n",
    "adv['300+ tp']=np.nan\n",
    "adv['300+ fp']=np.nan\n",
    "adv['300+ fn']=np.nan\n",
    "\n",
    "conf['300+ tp']=np.nan\n",
    "conf['300+ fp']=np.nan\n",
    "conf['300+ fn']=np.nan\n",
    "\n",
    "noemph['300+ tp']=np.nan\n",
    "noemph['300+ fp']=np.nan\n",
    "noemph['300+ fn']=np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e9215b44",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "vol_cols=[col for col in noemph.columns if 'V' in col] #Get name of columns containing volumes of AI nodules\n",
    "\n",
    "emph_deg=['noemph_fp','adv_fp','mod_fp','conf_fp']\n",
    "\n",
    "for deg in emph_deg: #Loop over emphysema degrees\n",
    "    for col in vol_cols: #Loop over columns with volumes\n",
    "        #If the volume is less than 30mm3 or more than 300mm3 we should ignore them - set it along with the corresponding AI nod to '-'\n",
    "        #This can be done since we get TP and FN from other file - This only considers FPs\n",
    "        for ind,val in eval(deg[:-3]+\"[(\"+deg[:-3]+\"['\"+col+\"']<=30) | (\"+deg[:-3]+\"['\"+col+\"']>300)]['\"+col+\"'].items()\"): \n",
    "            exec(deg[:-3]+\"['\"+col+\"'].iloc[ind]=np.nan\") #was '-' instead of nan\n",
    "            exec(deg[:-3]+\"['AI_nod\"+str(col[1:])+\"'].iloc[ind]=np.nan\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "112f55d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Select rows where we have at least one FP in any of the 0-100 or 100-300 volume subgroup\n",
    "mod_fp=mod[(mod['100-300fp'].notnull() | mod['0-100fp'].notnull() ) & mod['participant_id'].notnull()]\n",
    "conf_fp=conf[(conf['100-300fp'].notnull() | conf['0-100fp'].notnull() ) & conf['participant_id'].notnull()]\n",
    "adv_fp=adv[(adv['100-300fp'].notnull() | adv['0-100fp'].notnull() ) & adv['participant_id'].notnull()]\n",
    "noemph_fp=noemph[(noemph['100-300fp'].notnull() | noemph['0-100fp'].notnull() ) & noemph['participant_id'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "aa1f91b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Initialize empty dicts in the form {'pat_id1':[],'pat_id2':[],...}\n",
    "noemph_dict=dict.fromkeys([str(numeric_string) for numeric_string in noemph_fp['participant_id'].values], [])\n",
    "noemph_dict=[[key[:6],[]] for (key, value) in noemph_dict.items()]\n",
    "noemph_dict = {item[0]: item[1] for item in noemph_dict}\n",
    "noemph_fp['participant_id']=list(noemph_dict.keys())\n",
    "\n",
    "mod_dict=dict.fromkeys([str(numeric_string) for numeric_string in mod_fp['participant_id'].values], [])\n",
    "mod_dict=[[key[:6],[]] for (key, value) in mod_dict.items()]\n",
    "mod_dict = {item[0]: item[1] for item in mod_dict}\n",
    "mod_fp['participant_id']=list(mod_dict.keys())\n",
    "\n",
    "conf_dict=dict.fromkeys([str(numeric_string) for numeric_string in conf_fp['participant_id'].values], [])\n",
    "conf_dict=[[key[:6],[]] for (key, value) in conf_dict.items()]\n",
    "conf_dict = {item[0]: item[1] for item in conf_dict}\n",
    "conf_fp['participant_id']=list(conf_dict.keys())\n",
    "\n",
    "adv_dict=dict.fromkeys([str(numeric_string) for numeric_string in adv_fp['participant_id'].values], [])\n",
    "adv_dict=[[key[:6],[]] for (key, value) in adv_dict.items()]\n",
    "adv_dict = {item[0]: item[1] for item in adv_dict}\n",
    "adv_fp['participant_id']=list(adv_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "29623898",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Volume dictionaries - Normal copies don't work properly. This is why deepcopy is used\n",
    "noemph_dict_vol=copy.deepcopy(noemph_dict)\n",
    "mod_dict_vol=copy.deepcopy(mod_dict)\n",
    "conf_dict_vol=copy.deepcopy(conf_dict)\n",
    "adv_dict_vol=copy.deepcopy(adv_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "20e567f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "AI_cols=[col for col in noemph_fp.columns if 'AI_nod' in col] #Get name of columns containing AI nodules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a4f15fd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "noemph_fp\n",
      "adv_fp\n",
      "mod_fp\n",
      "conf_fp\n"
     ]
    }
   ],
   "source": [
    "emph_deg=['noemph_fp','adv_fp','mod_fp','conf_fp'] #list with strings of dfs to loop\n",
    "\n",
    "for deg in emph_deg: #Loop over emphysema degrees\n",
    "    print(deg)\n",
    "    \n",
    "    for ind_col,col in enumerate(AI_cols): #Loop over AI nodule columns\n",
    "        \n",
    "        #Following line to change nan with '-' since otherwise cannot check for string with 'L' below\n",
    "        exec(deg[:-3]+\"_fp['\"+col+\"']=\"+deg[:-3]+\"_fp['\"+col+\"'].fillna('-')\")\n",
    "        exec(deg[:-3]+\"_fp['\"+str(col)+\"'] = \"+deg[:-3]+\"_fp['\"+str(col)+\"'].astype(str)\") #Convert to string type to use below\n",
    "        \n",
    "        #Create variables storing only those rows of df that a specific AI_nod col contains 'L' (denotes a TP)-or not those\n",
    "        exec('temp='+deg[:-3]+'_fp[~'+deg[:-3]+\"_fp['\"+str(col)+\"'].str.contains('L')]\") #FPs\n",
    "        exec('temp_tp='+deg[:-3]+'_fp['+deg[:-3]+\"_fp['\"+str(col)+\"'].str.contains('L')]\") #TPs\n",
    "\n",
    "        if not temp.empty: #If we have FP for that participant\n",
    "\n",
    "            for ind,pat in enumerate(temp['participant_id']): #Loop over all participants with FP in a specific AI col\n",
    "\n",
    "                try: #To ensure that there are no errors\n",
    "                    nod_id=temp.iloc[ind,ind_col+1][temp.iloc[ind,ind_col+1].find('L')+1:] #Get id\n",
    "                    nod_id=nod_id.split(' ')[0] #To get actual id\n",
    "                    vol=temp.iloc[ind,ind_col+11] #To get the value of the volume\n",
    "                    \n",
    "                    exec(deg[:-3]+'_dict'+\"['\"+str(pat)+\"'].append('\"+nod_id+\"')\") #Add that to the dictionary\n",
    "                    \n",
    "                    if pd.isnull(vol): #When there is no volume - is nan\n",
    "                        exec(deg[:-3]+'_dict_vol'+\"['\"+str(pat)+\"'].append('-')\") #Same for volume dictionary\n",
    "                    else:\n",
    "                        exec(deg[:-3]+'_dict_vol'+\"['\"+str(pat)+\"'].append('\"+str(vol)+\"')\") #Same for volume dictionary\n",
    "                except:\n",
    "                    print(traceback.print_exc()) #print error\n",
    "                    \n",
    "                    \n",
    "        if not temp_tp.empty: #If we have TP for that participant\n",
    "\n",
    "            for ind,pat in enumerate(temp_tp['participant_id']): #Loop over all participants with TP in a specific AI col\n",
    "\n",
    "                try: #To ensure that there are no errors\n",
    "                    exec(deg[:-3]+'_dict'+\"['\"+str(pat)+\"'].append('\"+\"-\"+\"')\") #Add that to the dictionary\n",
    "                    exec(deg[:-3]+'_dict_vol'+\"['\"+str(pat)+\"'].append('\"+\"-\"+\"')\") #Same for volume dictionary\n",
    "                except:\n",
    "                    print(traceback.print_exc())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a3b3a9f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "noemph_dict"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a8f074c7",
   "metadata": {},
   "source": [
    "#### Check below IDs again - They have nodules not reviewed by radiologists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "04e110ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Confirm all participants' nodules counted - These are nodules not reviewed - Should be checked again\n",
    "\n",
    "print(\"All participants below have nodules not reviewed yet. We should not have any participants printed below at the end. \\n\\\n",
    "For now we do, since we manually deleted those in which AI detected a finding >30mm3 but manually measured <30mm3.\")\n",
    "print(\"\\n\")\n",
    "\n",
    "for deg in emph_deg: #Loop over emphysema degrees\n",
    "    \n",
    "    for pat in eval(deg[:-3]+\"_dict\"): #For each participant in a given degree\n",
    "        \n",
    "        temp=[x for x in eval(deg[:-3]+\"_dict['\"+pat+\"']\") if x!='-'] #Get how many FP we have - ignore '-' values\n",
    "        counted=0 #Initialize an index to 0\n",
    "        \n",
    "        if deg!='noemph_fp': #Since we have emph/noemph dictionaries below\n",
    "            temp_deg='emph_tp'\n",
    "        else:\n",
    "            temp_deg=deg\n",
    "            \n",
    "        #If the participant is in any of the FP dictionaries increase the count by the values of that dictionary list\n",
    "        if pat in eval(\"dict_FP_wrong_\"+temp_deg[:-3]): \n",
    "            counted=counted+len(eval(\"dict_FP_wrong_\"+temp_deg[:-3]+\"['\"+pat+\"']\"))\n",
    "\n",
    "        if pat in eval(\"dict_FP_correct_\"+temp_deg[:-3]):\n",
    "            counted=counted+len(eval(\"dict_FP_correct_\"+temp_deg[:-3]+\"['\"+pat+\"']\"))\n",
    "            \n",
    "        #Check if the counts match - If not then we missed some slices\n",
    "        try:\n",
    "            assert len(temp)==counted\n",
    "        except:\n",
    "            print('Missing FP slice(s) for participant',pat,'with',temp_deg[:-3]+'ysema')  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f954e319",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create copies of dictionaries with FP to be filled with the corresponding ids\n",
    "dict_FP_wrong_noemph_ids=copy.deepcopy(dict_FP_wrong_noemph)\n",
    "dict_FP_correct_noemph_ids=copy.deepcopy(dict_FP_correct_noemph)\n",
    "dict_FP_wrong_emph_ids=copy.deepcopy(dict_FP_wrong_emph)\n",
    "dict_FP_correct_emph_ids=copy.deepcopy(dict_FP_correct_emph)\n",
    "\n",
    "#Same for dictionaries with lymph nodes only and nodule only\n",
    "lymph_FP_wrong_noemph_ids=copy.deepcopy(lymph_FP_wrong_noemph)\n",
    "lymph_FP_wrong_emph_ids=copy.deepcopy(lymph_FP_wrong_emph)\n",
    "nod_FP_wrong_noemph_ids=copy.deepcopy(nod_FP_wrong_noemph)\n",
    "nod_FP_wrong_emph_ids=copy.deepcopy(nod_FP_wrong_emph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1efe41c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create new dictionaries 'correct' and 'wrong' with FP indices\n",
    "\n",
    "pat_manual_check=[] #Initialize a list to be filled with participants in whom nodules should be filled in manually\n",
    "\n",
    "for deg in emph_deg: #Loop over emphysema degree\n",
    "    \n",
    "    deg=deg[:-3] #Keep name of emphysema degree only, without '_fp'\n",
    "    \n",
    "    for pat in eval(deg+\"['participant_id']\"): #Loop over participants in each emphysema degree\n",
    "        \n",
    "        if isinstance(pat,str): #required conversions to only keep first 6 digits of participant_id\n",
    "            try:\n",
    "                pat=int(pat[:6])\n",
    "            except:\n",
    "                pass\n",
    "        else: #If participant_id consists only of numbers, then we assume that this is the 6 digit participant_id\n",
    "            try:\n",
    "                pat=int(pat)\n",
    "            except:\n",
    "                pass\n",
    "        \n",
    "        \n",
    "    #Initialize a list to keep track of slices and ensure that there is no overlap between them - Unique mapping to ids\n",
    "        all_slices=[] \n",
    "        \n",
    "        #There are a lot of key errors since a participant might not exist in the FP dictionaries - only TP and/or FNs\n",
    "        try: \n",
    "            for elem in eval(deg+\"_dict['\"+str(pat)+\"']\"): #Loop over all participants findings\n",
    "                if elem!='-': #If there is a value in that finding\n",
    "                    \n",
    "                    for i in range(int(elem.split('-')[0]),int(elem.split('-')[1])): #To get slices of TP or FP\n",
    "                        if i not in all_slices: #If this slice not in the list of slices add it\n",
    "                            all_slices.append(i)\n",
    "                        else: #if exists\n",
    "                            if pat in pat_manual_check: #If that participant has already been added in list to check \n",
    "                                pass\n",
    "                            else: #If not in the list to check, add it\n",
    "                                pat_manual_check.append(pat)\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "        \n",
    "        if deg!='noemph': #This is used below for dictionary names\n",
    "            temp_deg='emph'\n",
    "        else:\n",
    "            temp_deg=deg\n",
    "        \n",
    "        #For those that there is a unique mapping make the conversion:\n",
    "        try:\n",
    "            for AI_ind,elem in enumerate(eval(deg+\"_dict['\"+str(pat)+\"']\")): #Loop over participants of given degree\n",
    "                if elem!='-' and (pat not in pat_manual_check): #For participants with a 1-to-1 mapping\n",
    "                    \n",
    "                    for i in range(int(elem.split('-')[0]),int(elem.split('-')[1])): #Loop over the range of slices\n",
    "                        \n",
    "                        #Then loop over the unique slices in the FP dictionaries\n",
    "                        try: #Again avoid key errors\n",
    "                            for ind,slice_FP_wrong in enumerate(eval(\"dict_FP_wrong_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\")): \n",
    "                                    if slice_FP_wrong==i: #If we found that slice replace it with id\n",
    "                                        exec(\"dict_FP_wrong_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\"+\"[\"+str(ind)+\"]=\"+str(AI_ind+1))\n",
    "                        except:\n",
    "                            pass\n",
    "                        \n",
    "                        try:\n",
    "                            for ind,slice_FP_wrong in enumerate(eval(\"dict_FP_correct_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\")): \n",
    "                                    if slice_FP_wrong==i: #If we found that slice replace it with id\n",
    "                                        exec(\"dict_FP_correct_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\"+\"[\"+str(ind)+\"]=\"+str(AI_ind+1))   \n",
    "                        except:\n",
    "                            pass\n",
    "                        \n",
    "                                            \n",
    "                        #Again avoid key errors for lymph nodes and nodules\n",
    "                        try: \n",
    "                            for ind,slice_FP_wrong in enumerate(eval(\"lymph_FP_wrong_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\")): \n",
    "                                    if slice_FP_wrong==i: #If we found that slice replace it with id\n",
    "                                        exec(\"lymph_FP_wrong_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\"+\"[\"+str(ind)+\"]=\"+str(AI_ind+1))\n",
    "                        except:\n",
    "                            pass\n",
    "                        \n",
    "                        try:\n",
    "                            for ind,slice_FP_wrong in enumerate(eval(\"nod_FP_wrong_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\")): \n",
    "                                    if slice_FP_wrong==i: #If we found that slice replace it with id\n",
    "                                        exec(\"nod_FP_wrong_\"+str(temp_deg)+\"_ids['\"+str(pat)+\"']\"+\"[\"+str(ind)+\"]=\"+str(AI_ind+1))   \n",
    "                        except:\n",
    "                            pass\n",
    "                        \n",
    "        except:\n",
    "            pass\n",
    "\n",
    "print(\"The following participants have to be checked manually to map slices to ids:\",pat_manual_check)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "74cb4e89",
   "metadata": {},
   "source": [
    "Check if the FP in the dictionaries are as expected until now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cc6b2574",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for pat in dict_FP_correct_noemph_ids: #Loop over participants in no emphysema FP_correct\n",
    "    for id in dict_FP_correct_noemph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_wrong_noemph_ids[pat]: #If this ID is also in FP_wrong\n",
    "                print(\"No emphysema and FP_correct\",pat,id,\" should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in dict_FP_wrong_noemph_ids: #Loop over participants in no emphysema FP_wrong\n",
    "    for id in dict_FP_wrong_noemph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_correct_noemph_ids[pat]: #If this ID is also in FP_correct\n",
    "                print(\"No emphysema and FP_wrong\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "for pat in dict_FP_correct_emph_ids: #Loop over participants in emphysema FP_correct\n",
    "    for id in dict_FP_correct_emph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_wrong_emph_ids[pat]: #If this ID is also in FP_wrong\n",
    "                print(\"Emphysema FP_correct\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in dict_FP_wrong_emph_ids: #Loop over participants in emphysema FP_wrong\n",
    "    for id in dict_FP_wrong_emph_ids[pat]: \n",
    "        try:\n",
    "            if id in dict_FP_correct_emph_ids[pat]: #If this ID is also in FP_correct\n",
    "                print(\"Emphysema FP_wrong\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b82420ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Delete participants with volumes <30mm3 or >300mm3 or those cases above that mapping is not possible\n",
    "for deg in emph_deg: #Loop over emphysema degree\n",
    "    print(\"Below are for \",deg[:-3],\"emphysema\")\n",
    "    \n",
    "    deg=deg[:-3] #Keep name of emphysema degree only, without '_fp'\n",
    "    \n",
    "    if deg!='noemph': #This is used below for dictionary names\n",
    "        temp_deg='emph'\n",
    "    else:\n",
    "        temp_deg=deg\n",
    "    \n",
    "    for key,values in eval(\"dict_FP_wrong_\"+str(temp_deg)+\"_ids.items()\"): #Loop over participants and their id\n",
    "        for ind,val in enumerate(values):\n",
    "\n",
    "            try: #Since we will loop many times in the 'emph' and so, we won't be able to evaluate '-' value as integer\n",
    "                if int(val)>10: #For cases with errors\n",
    "                    exec(\"dict_FP_wrong_\"+str(temp_deg)+\"_ids['\"+key+\"'][\"+str(ind)+\"]=\"+\"'-'\")\n",
    "            except:\n",
    "                print('dict_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'not deleted')\n",
    "                pass\n",
    "                \n",
    "    for key,values in eval(\"dict_FP_correct_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        for ind,val in enumerate(values):\n",
    "            try:\n",
    "                if int(val)>10:\n",
    "                    exec(\"dict_FP_correct_\"+str(temp_deg)+\"_ids['\"+key+\"'][\"+str(ind)+\"]=\"+\"'-'\")\n",
    "            except:\n",
    "                print('dict_FP_correct_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'not deleted')\n",
    "                pass       \n",
    "\n",
    "\n",
    "    #Delete participants with only '-' values\n",
    "    del_keys=[]\n",
    "    for key,values in eval(\"dict_FP_wrong_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        if np.unique(values)[0]=='-': #If a participant has '-' which denotes an error\n",
    "            del_keys.append(key)\n",
    "            print('For dict_FP_wrong we have',len(values),'\"-\" values for',key)\n",
    "            \n",
    "    for key in del_keys:\n",
    "        try:\n",
    "            exec(\"del dict_FP_wrong_\"+str(temp_deg)+\"_ids['\"+key+\"']\") #Delete that participants and its values from dict\n",
    "            print('dict_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'deleted')\n",
    "        except:\n",
    "            print('dict_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+' not deleted')\n",
    "            pass\n",
    "        \n",
    "    #Similar for dict_FP_correct    \n",
    "    del_keys=[]\n",
    "    for key,values in eval(\"dict_FP_correct_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        if np.unique(values)[0]=='-':\n",
    "            del_keys.append(key)\n",
    "            print('For dict_FP_correct we have',len(values),'\"-\" values for',key)\n",
    "       \n",
    "    for key in del_keys:\n",
    "        try:\n",
    "            exec(\"del dict_FP_correct_\"+str(temp_deg)+\"_ids['\"+key+\"']\") \n",
    "            print('dict_FP_correct_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'deleted')\n",
    "        except:\n",
    "            print('dict_FP_correct_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+' not deleted')\n",
    "            pass\n",
    "        \n",
    "        \n",
    "        \n",
    "    #Similar for lymph nodes only and nod only dictionaries\n",
    "    for key,values in eval(\"lymph_FP_wrong_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        for ind,val in enumerate(values):\n",
    "\n",
    "            try: #Since we will loop many times in the 'emph' and so, we won't be able to evaluate '-' value as integer\n",
    "                if int(val)>10:\n",
    "                    exec(\"lymph_FP_wrong_\"+str(temp_deg)+\"_ids['\"+key+\"'][\"+str(ind)+\"]=\"+\"'-'\")\n",
    "            except:\n",
    "                print(\"lymph_FP_wrong_\"+str(temp_deg)+\"_ids[\"+key+\"] not deleted\")\n",
    "                pass\n",
    "                \n",
    "    for key,values in eval(\"nod_FP_wrong_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        for ind,val in enumerate(values):\n",
    "            try:\n",
    "                if int(val)>10:\n",
    "                    exec(\"nod_FP_wrong_\"+str(temp_deg)+\"_ids['\"+key+\"'][\"+str(ind)+\"]=\"+\"'-'\")\n",
    "            except:\n",
    "                print(\"nod_FP_wrong_\"+str(temp_deg)+\"_ids[\"+key+\"] not deleted\")\n",
    "                pass\n",
    "            \n",
    "            \n",
    "    #Delete participants with only '-' values\n",
    "    del_keys=[]\n",
    "    for key,values in eval(\"lymph_FP_wrong_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        if np.unique(values)[0]=='-':\n",
    "            del_keys.append(key)            \n",
    "       \n",
    "    for key in del_keys:\n",
    "        try:\n",
    "            exec(\"del lymph_FP_wrong_\"+str(temp_deg)+\"_ids['\"+key+\"']\") \n",
    "            print('lymph_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'deleted' )\n",
    "\n",
    "        except:\n",
    "            print('lymph_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'not deleted')\n",
    "            pass\n",
    "        \n",
    "    del_keys=[]\n",
    "    for key,values in eval(\"nod_FP_wrong_\"+str(temp_deg)+\"_ids.items()\"): \n",
    "        if np.unique(values)[0]=='-':\n",
    "            del_keys.append(key)\n",
    "       \n",
    "    for key in del_keys:\n",
    "        try:\n",
    "            exec(\"del nod_FP_wrong_\"+str(temp_deg)+\"_ids['\"+key+\"']\") \n",
    "            print('nod_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+'deleted')\n",
    "        except:\n",
    "            print('nod_FP_wrong_'+str(temp_deg)+\"_ids['\"+str(key)+\"']\"+' not deleted')\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a084dee",
   "metadata": {},
   "outputs": [],
   "source": [
    "pat_manual_check #These are the participants that should be checked manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fc0acd20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Checks again - Copy pasted from above\n",
    "for pat in dict_FP_correct_noemph_ids: #Loop over participants in low BMI FP_correct\n",
    "    for id in dict_FP_correct_noemph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_wrong_noemph_ids[pat]: #If this ID is also in FP_wrong\n",
    "                print(\"No emphysema and FP_correct\",pat,id,\" should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in dict_FP_wrong_noemph_ids: #Loop over participants in low BMI FP_wrong\n",
    "    for id in dict_FP_wrong_noemph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_correct_noemph_ids[pat]: #If this ID is also in FP_correct\n",
    "                print(\"No emphysema and FP_wrong\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "for pat in dict_FP_correct_emph_ids: #Loop over participants in high BMI FP_correct\n",
    "    for id in dict_FP_correct_emph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_wrong_emph_ids[pat]: #If this ID is also in FP_wrong\n",
    "                print(\"Emphysema FP_correct\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in dict_FP_wrong_emph_ids: #Loop over participants in high BMI FP_wrong\n",
    "    for id in dict_FP_wrong_emph_ids[pat]: \n",
    "        try:\n",
    "            if id in dict_FP_correct_emph_ids[pat]: #If this ID is also in FP_correct\n",
    "                print(\"Emphysema FP_wrong\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "327b4cc9",
   "metadata": {},
   "source": [
    "Further checks based on number of nodules, lymph nodes, and non-nodules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "fe85d790",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emphysema Cases\n",
      "Non-nodules: 20\n",
      "Lymph nodes: 3\n",
      "Nodules 19\n",
      "All findings: 42\n"
     ]
    }
   ],
   "source": [
    "print(\"Emphysema Cases\")\n",
    "all=0\n",
    "for pat in dict_FP_correct_emph:\n",
    "    all=all+len(dict_FP_correct_emph[pat])\n",
    "print('Non-nodules:',all)\n",
    "\n",
    "FP_lymph_wrong=0\n",
    "for pat in lymph_FP_wrong_emph:\n",
    "    FP_lymph_wrong=FP_lymph_wrong+len(lymph_FP_wrong_emph[pat])\n",
    "print(\"Lymph nodes:\",FP_lymph_wrong) \n",
    "\n",
    "FP_nod_wrong=0\n",
    "for pat in nod_FP_wrong_emph:\n",
    "    FP_nod_wrong=FP_nod_wrong+len(nod_FP_wrong_emph[pat])\n",
    "print(\"Nodules\",FP_nod_wrong)\n",
    "\n",
    "for pat in dict_FP_wrong_emph:\n",
    "    all=all+len(dict_FP_wrong_emph[pat])\n",
    "print(\"All findings:\",all) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "41a0f3a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-emphysema cases\n",
      "Non-nodules: 18\n",
      "Lymph nodes: 7\n",
      "Nodules 20\n",
      "All findings: 45\n"
     ]
    }
   ],
   "source": [
    "print(\"Non-emphysema cases\")\n",
    "all=0\n",
    "for pat in dict_FP_correct_noemph:\n",
    "    all=all+len(dict_FP_correct_noemph[pat])\n",
    "print('Non-nodules:',all)\n",
    "\n",
    "FP_lymph_wrong=0\n",
    "for pat in lymph_FP_wrong_noemph:\n",
    "    FP_lymph_wrong=FP_lymph_wrong+len(lymph_FP_wrong_noemph[pat])\n",
    "print(\"Lymph nodes:\",FP_lymph_wrong) \n",
    "\n",
    "FP_nod_wrong=0\n",
    "for pat in nod_FP_wrong_noemph:\n",
    "    FP_nod_wrong=FP_nod_wrong+len(nod_FP_wrong_noemph[pat])\n",
    "print(\"Nodules\",FP_nod_wrong)\n",
    "\n",
    "for pat in dict_FP_wrong_noemph:\n",
    "    all=all+len(dict_FP_wrong_noemph[pat])\n",
    "print(\"All findings:\",all) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "99873680",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total FP wrong for emph are: 22\n",
      "Total FP wrong for noemph are: 27\n",
      "Total FP correct for emph are: 20\n",
      "Total FP correct for noemph are: 18\n"
     ]
    }
   ],
   "source": [
    "FP_wrong_emph=np.sum([len(x) for x in dict_FP_wrong_emph.values()])\n",
    "FP_wrong_noemph=np.sum([len(x) for x in dict_FP_wrong_noemph.values()])\n",
    "print(\"Total FP wrong for emph are:\",FP_wrong_emph)\n",
    "print(\"Total FP wrong for noemph are:\",FP_wrong_noemph)\n",
    "\n",
    "FP_correct_emph=np.sum([len(x) for x in dict_FP_correct_emph.values()])\n",
    "FP_correct_noemph=np.sum([len(x) for x in dict_FP_correct_noemph.values()])\n",
    "print(\"Total FP correct for emph are:\",FP_correct_emph)\n",
    "print(\"Total FP correct for noemph are:\",FP_correct_noemph)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "99424ab8",
   "metadata": {},
   "source": [
    "Manually add participants for whom a unique mapping wasn't possible - Atypical lymph nodes considered as nodules below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b2333f84",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Manually add participants for who a unique mapping wasn't possible\n",
    "dict_FP_correct_noemph['673634']=[287]#,287,306,224] #correct if radiologists said 'not nodule' (but not lymph node) \n",
    "dict_FP_correct_noemph_ids['673634']=[1]#,5,6,7] #Last 3 changed from row below to here since they should be treated as FP (nonods) due to manually measured vol<30mm3\n",
    "dict_FP_wrong_noemph['673634']=[254] #wrong if radiologists said 'nodule' or lymph node\n",
    "dict_FP_wrong_noemph_ids['673634']=[4]\n",
    "\n",
    "dict_FP_wrong_emph['971099']=[89,352,382] \n",
    "dict_FP_wrong_emph_ids['971099']=[8,6,5]\n",
    "dict_FP_correct_emph['971099']=[129,233,255,363] \n",
    "dict_FP_correct_emph_ids['971099']=[3,4,2,7] \n",
    "\n",
    "dict_FP_correct_emph['845594']=[128,99,337,369] \n",
    "dict_FP_correct_emph_ids['845594']=[3,4,5,6] \n",
    "\n",
    "#Same for the lymph node only and nodules only dictionaries\n",
    "nod_FP_wrong_noemph['673634']=[254]\n",
    "nod_FP_wrong_noemph_ids['673634']=[4] #Same change here as above due to manually measured vol<30mm3\n",
    "\n",
    "# lymph_FP_wrong_emph['971099']=[89,352,382] #These are atypical lymph nodes that are considered as nodules\n",
    "# lymph_FP_wrong_emph_ids['971099']=[8,6,5]\n",
    "nod_FP_wrong_emph['971099']=[89,352,382]\n",
    "nod_FP_wrong_emph_ids['971099']=[8,6,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2b717973",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Checks again - Copy pasted from above\n",
    "for pat in dict_FP_correct_noemph_ids: #Loop over participants in low BMI FP_correct\n",
    "    for id in dict_FP_correct_noemph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_wrong_noemph_ids[pat]: #If this ID is also in FP_wrong\n",
    "                print(\"No emphysema and FP_correct\",pat,id,\" should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in dict_FP_wrong_noemph_ids: #Loop over participants in low BMI FP_wrong\n",
    "    for id in dict_FP_wrong_noemph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_correct_noemph_ids[pat]: #If this ID is also in FP_correct\n",
    "                print(\"No emphysema and FP_wrong\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "print(\"\\n\")\n",
    "\n",
    "for pat in dict_FP_correct_emph_ids: #Loop over participants in high BMI FP_correct\n",
    "    for id in dict_FP_correct_emph_ids[pat]:\n",
    "        try:\n",
    "            if id in dict_FP_wrong_emph_ids[pat]: #If this ID is also in FP_wrong\n",
    "                print(\"Emphysema FP_correct\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in dict_FP_wrong_emph_ids: #Loop over participants in high BMI FP_wrong\n",
    "    for id in dict_FP_wrong_emph_ids[pat]: \n",
    "        try:\n",
    "            if id in dict_FP_correct_emph_ids[pat]: #If this ID is also in FP_correct\n",
    "                print(\"Emphysema FP_wrong\",pat,id,\"should be checked manually\") #Should fix that manually\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "\n",
    "#For lymph nodes and nodules\n",
    "for pat in lymph_FP_wrong_emph_ids: #Loop over participants in low BMI FP_wrong\n",
    "    for id in lymph_FP_wrong_emph_ids[pat]:\n",
    "        try:\n",
    "            if id in nod_FP_wrong_emph_ids[pat]: #If this ID is also in nod_FP_wrong\n",
    "                print(\"Emphysema lymph_FP_wrong\",pat,'with ID',id,\"should be checked manually\")\n",
    "        except: #might not exist in above dict\n",
    "            pass\n",
    "\n",
    "for pat in nod_FP_wrong_emph_ids: #Loop over participants in low BMI nod_FP_wrong\n",
    "    for id in nod_FP_wrong_emph_ids[pat]:\n",
    "        try:\n",
    "            if id in lymph_FP_wrong_emph_ids[pat]: #If this ID is also in lymph_FP_wrong\n",
    "                print(\"Emphysema nod_FP_wrong\",pat,'with ID',id,\"should be checked manually\")\n",
    "        except: #might not exist in above dict\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ad20ad94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-nodules: 20\n",
      "Lymph nodes: 3\n",
      "Nodules 19\n",
      "All findings: 42\n"
     ]
    }
   ],
   "source": [
    "all=0\n",
    "for pat in dict_FP_correct_emph:\n",
    "    all=all+len(dict_FP_correct_emph[pat])\n",
    "print('Non-nodules:',all)\n",
    "\n",
    "FP_lymph_wrong=0\n",
    "for pat in lymph_FP_wrong_emph:\n",
    "    FP_lymph_wrong=FP_lymph_wrong+len(lymph_FP_wrong_emph[pat])\n",
    "print(\"Lymph nodes:\",FP_lymph_wrong) \n",
    "\n",
    "FP_nod_wrong=0\n",
    "for pat in nod_FP_wrong_emph:\n",
    "    FP_nod_wrong=FP_nod_wrong+len(nod_FP_wrong_emph[pat])\n",
    "print(\"Nodules\",FP_nod_wrong)\n",
    "\n",
    "for pat in dict_FP_wrong_emph:\n",
    "    all=all+len(dict_FP_wrong_emph[pat])\n",
    "print(\"All findings:\",all) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "710b0e20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total FP wrong for emph are: 22\n",
      "Total FP wrong for noemph are: 27\n",
      "Total FP correct for emph are: 20\n",
      "Total FP correct for noemph are: 18\n"
     ]
    }
   ],
   "source": [
    "FP_wrong_emph=np.sum([len(x) for x in dict_FP_wrong_emph.values()])\n",
    "FP_wrong_noemph=np.sum([len(x) for x in dict_FP_wrong_noemph.values()])\n",
    "print(\"Total FP wrong for emph are:\",FP_wrong_emph)\n",
    "print(\"Total FP wrong for noemph are:\",FP_wrong_noemph)\n",
    "\n",
    "FP_correct_emph=np.sum([len(x) for x in dict_FP_correct_emph.values()])\n",
    "FP_correct_noemph=np.sum([len(x) for x in dict_FP_correct_noemph.values()])\n",
    "print(\"Total FP correct for emph are:\",FP_correct_emph)\n",
    "print(\"Total FP correct for noemph are:\",FP_correct_noemph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "60cba74c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#lists of dictionaries with participant and nodule ids that belong to a given category\n",
    "all_categories=[atyp_FN_noemph,per_FN_noemph,pleural_FN_noemph,calcif_FN_noemph,sub_ground_FN_noemph,\n",
    "                cancer_FN_noemph,other_nodules_FN_noemph,other_nonodules_FN_noemph,fibrosis_FN_noemph,\n",
    "                bronchioperi_FN_noemph,atyp_FP_noemph,per_FP_noemph,pleural_FP_noemph,calcif_FP_noemph,\n",
    "                sub_ground_FP_noemph,cancer_FP_noemph,other_nodules_FP_noemph,other_nonodules_FP_noemph,\n",
    "                fibrosis_FP_noemph,bronchioperi_FP_noemph,  #Until here no emphysema\n",
    "                atyp_FN_emph,per_FN_emph,pleural_FN_emph,calcif_FN_emph,sub_ground_FN_emph,cancer_FN_emph,\n",
    "                other_nodules_FN_emph,other_nonodules_FN_emph,fibrosis_FN_emph,bronchioperi_FN_emph,atyp_FP_emph,\n",
    "                per_FP_emph,pleural_FP_emph,calcif_FP_emph,sub_ground_FP_emph,cancer_FP_emph,\n",
    "                other_nodules_FP_emph,other_nonodules_FP_emph,fibrosis_FP_emph,bronchioperi_FP_emph] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "dd957a54",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Same as above with the names as strings\n",
    "name_cats=['atyp_FN_noemph','per_FN_noemph','pleural_FN_noemph','calcif_FN_noemph','sub_ground_FN_noemph',\n",
    "           'cancer_FN_noemph','other_nodules_FN_noemph','other_nonodules_FN_noemph','fibrosis_FN_noemph',\n",
    "           'bronchioperi_FN_noemph','atyp_FP_noemph','per_FP_noemph','pleural_FP_noemp','calcif_FP_noemph',\n",
    "           'sub_ground_FP_noemph','cancer_FP_noemph','other_nodules_FP_noemph','other_nonodules_FP_noemph',\n",
    "           'fibrosis_FP_noemph','bronchioperi_FP_noemph', #Until here no emphysema\n",
    "           'atyp_FN_emph','per_FN_emph','pleural_FN_emph','calcif_FN_emph','sub_ground_FN_emph','cancer_FN_emph',\n",
    "           'other_nodules_FN_emph','other_nonodules_FN_emph','fibrosis_FN_emph','bronchioperi_FN_emph','atyp_FP_emph',\n",
    "           'per_FP_emph','pleural_FP_emph','calcif_FP_emph','sub_ground_FP_emph','cancer_FP_emph',\n",
    "           'other_nodules_FP_emph','other_nonodules_FP_emph','fibrosis_FP_emph','bronchioperi_FP_emph']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "32074c79",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Save dictionaries to be used from 'patient_selection_emphysema_experiment.ipynb' file to match slices with ids\n",
    "\n",
    "with open('dict_FN_wrong_emph.pickle','wb') as f:\n",
    "    pickle.dump(dict_FN_wrong_emph,f)\n",
    "\n",
    "with open('dict_FN_correct_emph.pickle','wb') as f:\n",
    "    pickle.dump(dict_FN_correct_emph,f)    \n",
    "\n",
    "with open('dict_FN_wrong_noemph.pickle','wb') as f:\n",
    "    pickle.dump(dict_FN_wrong_noemph,f)\n",
    "\n",
    "with open('dict_FN_correct_noemph.pickle','wb') as f:\n",
    "    pickle.dump(dict_FN_correct_noemph,f) \n",
    "    \n",
    "#Same for lymph nodes only and nodules only dictionaries\n",
    "with open('lymph_FN_correct_emph.pickle','wb') as f:\n",
    "    pickle.dump(lymph_FN_correct_emph,f)    \n",
    "\n",
    "with open('lymph_FN_correct_noemph.pickle','wb') as f:\n",
    "    pickle.dump(lymph_FN_correct_noemph,f) \n",
    "    \n",
    "with open('nod_FN_correct_emph.pickle','wb') as f:\n",
    "    pickle.dump(nod_FN_correct_emph,f)    \n",
    "\n",
    "with open('nod_FN_correct_noemph.pickle','wb') as f:\n",
    "    pickle.dump(nod_FN_correct_noemph,f) "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7524dba0",
   "metadata": {},
   "source": [
    "### Get volume subgroups for nodules/non-nodules for each of emphysema/non-emphysema\n",
    "\n",
    "##### AI found, reader missed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "62b1527a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get numbers for nodules (+lymph nodes) vs no nodules\n",
    "#Nodules can also be found be just adding nodules only + lymph nodes only from below)\n",
    "\n",
    "ai_nonods_noemph_30_100=0\n",
    "ai_nonods_noemph_100_300=0\n",
    "ai_nonods_noemph_300=0\n",
    "\n",
    "ai_nonods_emph_30_100=0\n",
    "ai_nonods_emph_100_300=0\n",
    "ai_nonods_emph_300=0\n",
    "\n",
    "#Similarly get numbers for nodules only and for lymph nodes only\n",
    "ai_only_nods_noemph_30_100=0\n",
    "ai_only_nods_noemph_100_300=0\n",
    "ai_only_nods_noemph_300=0\n",
    "\n",
    "ai_lymph_noemph_30_100=0\n",
    "ai_lymph_noemph_100_300=0\n",
    "ai_lymph_noemph_300=0\n",
    "\n",
    "ai_only_nods_emph_30_100=0\n",
    "ai_only_nods_emph_100_300=0\n",
    "ai_only_nods_emph_300=0\n",
    "\n",
    "ai_lymph_emph_30_100=0\n",
    "ai_lymph_emph_100_300=0\n",
    "ai_lymph_emph_300=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "fda77b52",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Similarly for volume of nodules (if comparison between groups with Mann-Whitney U test is used below)\n",
    "\n",
    "#Get detailed list of volume of nodules (+lymph nodes) vs no nodules\n",
    "#Nodules can also be found be just adding nodules only + lymph nodes only from below)\n",
    "\n",
    "ai_nonods_noemph_30_100_vols=[]\n",
    "ai_nonods_noemph_100_300_vols=[]\n",
    "ai_nonods_noemph_300_vols=[]\n",
    "\n",
    "ai_nonods_emph_30_100_vols=[]\n",
    "ai_nonods_emph_100_300_vols=[]\n",
    "ai_nonods_emph_300_vols=[]\n",
    "\n",
    "#Similarly get numbers for nodules only and for lymph nodes only\n",
    "ai_only_nods_noemph_30_100_vols=[]\n",
    "ai_only_nods_noemph_100_300_vols=[]\n",
    "ai_only_nods_noemph_300_vols=[]\n",
    "\n",
    "ai_lymph_noemph_30_100_vols=[]\n",
    "ai_lymph_noemph_100_300_vols=[]\n",
    "ai_lymph_noemph_300_vols=[]\n",
    "\n",
    "ai_only_nods_emph_30_100_vols=[]\n",
    "ai_only_nods_emph_100_300_vols=[]\n",
    "ai_only_nods_emph_300_vols=[]\n",
    "\n",
    "ai_lymph_emph_30_100_vols=[]\n",
    "ai_lymph_emph_100_300_vols=[]\n",
    "ai_lymph_emph_300_vols=[]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e7538cc0",
   "metadata": {},
   "source": [
    "Get numbers of nodules in each volume subgroup for : lymph node only, nodules only, and non-nodule categories in emphysema/non-emphysema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "d81c134d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total lymph nodes in emph group is 3\n",
      "Total lymph nodes in noemph group is 7\n"
     ]
    }
   ],
   "source": [
    "#For lymph node subgroup with/without emphysema\n",
    "\n",
    "for emph in ['emph','noemph']:\n",
    "    total=0 #count total number\n",
    "\n",
    "    for pat in eval('lymph_FP_wrong_'+emph+'_ids'): #loop over participants\n",
    "\n",
    "        for nod_id in eval('lymph_FP_wrong_'+emph+'_ids[pat]'): #Loop over nodule ids\n",
    "\n",
    "            if emph=='noemph':\n",
    "                vol=float(eval(noemph_dict_vol[pat][nod_id-1])) #Get volume of that nodule id\n",
    "            else: #For emphysema groups volume will be taken from the corresponding degree of that participant\n",
    "                try:\n",
    "                    vol=float(mod_dict_vol[pat][nod_id-1])\n",
    "                except:\n",
    "                    try:\n",
    "                        vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "            #Increase the number of findings of a specific volume subgroup depending on volume of finding - Add volume to the corresponding variable\n",
    "            if vol>=30 and vol<=100:\n",
    "                exec('ai_lymph_'+emph+'_30_100=ai_lymph_'+emph+'_30_100+1')\n",
    "                exec('ai_lymph_'+emph+'_30_100_vols.append(vol)')\n",
    "                total=total+1\n",
    "            elif vol>100 and vol<=300:\n",
    "                exec('ai_lymph_'+emph+'_100_300=ai_lymph_'+emph+'_100_300+1')\n",
    "                exec('ai_lymph_'+emph+'_100_300_vols.append(vol)')\n",
    "                total=total+1\n",
    "            elif vol>300:\n",
    "                exec('ai_lymph_'+emph+'_300=ai_lymph_'+emph+'_300+1') #Should be 0\n",
    "                exec('ai_lymph_'+emph+'_300_vols.append(vol)')\n",
    "            else:\n",
    "                print('For participant {} volume is smaller than 30mm3',pat)\n",
    "\n",
    "    print('Total lymph nodes in {} group is {}'.format(emph,total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8a35d39d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total nodules in emph group is 19\n",
      "Total nodules in noemph group is 20\n"
     ]
    }
   ],
   "source": [
    "#Similarly for nodule only group with/without emphysema - from discrepancies\n",
    "\n",
    "for emph in ['emph','noemph']:\n",
    "\n",
    "    total=0\n",
    "\n",
    "    for pat in eval('nod_FP_wrong_'+emph+'_ids'):\n",
    "\n",
    "        for nod_id in eval('nod_FP_wrong_'+emph+'_ids[pat]'):\n",
    "\n",
    "            if emph=='noemph':\n",
    "                vol=float(noemph_dict_vol[pat][nod_id-1])\n",
    "            else:\n",
    "                try:\n",
    "                    vol=float(mod_dict_vol[pat][nod_id-1])\n",
    "                except:\n",
    "                    try:\n",
    "                        vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "            if vol>=30 and vol<=100:\n",
    "                exec('ai_only_nods_'+emph+'_30_100=ai_only_nods_'+emph+'_30_100+1')\n",
    "                exec('ai_only_nods_'+emph+'_30_100_vols.append(vol)')\n",
    "                total=total+1\n",
    "            elif vol>100 and vol<=300:\n",
    "                exec('ai_only_nods_'+emph+'_100_300=ai_only_nods_'+emph+'_100_300+1')\n",
    "                exec('ai_only_nods_'+emph+'_100_300_vols.append(vol)')\n",
    "                total=total+1\n",
    "            elif vol>300:\n",
    "                exec('ai_only_nods_'+emph+'_300=ai_only_nods_'+emph+'_300+1') #Should be 0\n",
    "                exec('ai_only_nods_'+emph+'_300_vols.append(vol)')\n",
    "            else:\n",
    "                print('For participant {} volume is smaller than 30mm3',pat)\n",
    "                \n",
    "    print('Total nodules in {} group is {}'.format(emph,total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "03d2c002",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total non-nodules in emph group is 20\n",
      "Total non-nodules in noemph group is 18\n"
     ]
    }
   ],
   "source": [
    "#Similarly for non-nodule emphysema/non-emphysema groups\n",
    "\n",
    "for emph in ['emph','noemph']:\n",
    "    total=0\n",
    "\n",
    "    for pat in eval('dict_FP_correct_'+emph+'_ids'):\n",
    "\n",
    "        for nod_id in eval('dict_FP_correct_'+emph+'_ids[pat]'):\n",
    "\n",
    "            if emph=='noemph':\n",
    "                vol=float(noemph_dict_vol[pat][nod_id-1])\n",
    "            else:\n",
    "                try:\n",
    "                    vol=float(mod_dict_vol[pat][nod_id-1])\n",
    "                except:\n",
    "                    try:\n",
    "                        vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "            if vol>=30 and vol<=100:\n",
    "                exec('ai_nonods_'+emph+'_30_100=ai_nonods_'+emph+'_30_100+1')\n",
    "                exec('ai_nonods_'+emph+'_30_100_vols.append(vol)')\n",
    "                total=total+1\n",
    "            elif vol>100 and vol<=300:\n",
    "                exec('ai_nonods_'+emph+'_100_300=ai_nonods_'+emph+'_100_300+1')\n",
    "                exec('ai_nonods_'+emph+'_100_300_vols.append(vol)')\n",
    "                total=total+1\n",
    "            elif vol>300:\n",
    "                exec('ai_nonods_'+emph+'_300=ai_nonods_'+emph+'_300+1') #Should be 0\n",
    "                exec('ai_nonods_'+emph+'_300_vols.append(vol)')\n",
    "            else:\n",
    "                print('For participant {} volume is smaller than 30mm3',pat)\n",
    "\n",
    "    print('Total non-nodules in {} group is {}'.format(emph,total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "0d911039",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Confirm everything worked as expected\n",
    "assert len(np.unique(list(mod_dict_vol.keys())))+len(np.unique(list(conf_dict_vol.keys())))+len(np.unique(list(adv_dict_vol.keys())))==len(np.unique(list(mod_dict_vol.keys())+list(conf_dict_vol.keys())+list(adv_dict_vol.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "819080b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Total number of nodules in each of the emphysema/non-emphysema groups is the sum of the nodules and lymph nodes in those\n",
    "ai_nods_noemph_30_100=ai_only_nods_noemph_30_100+ai_lymph_noemph_30_100\n",
    "ai_nods_noemph_100_300=ai_only_nods_noemph_100_300+ai_lymph_noemph_100_300\n",
    "ai_nods_noemph_300=ai_only_nods_noemph_300+ai_lymph_noemph_300 #Should be 0\n",
    "\n",
    "ai_nods_emph_30_100=ai_only_nods_emph_30_100+ai_lymph_emph_30_100\n",
    "ai_nods_emph_100_300=ai_only_nods_emph_100_300+ai_lymph_emph_100_300\n",
    "ai_nods_emph_300=ai_only_nods_emph_300+ai_lymph_emph_300 #Should be 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "4be8fe28",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ai_nods_emph_300==ai_nods_noemph_300==ai_nonods_emph_300==ai_nonods_noemph_300==0"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "78eac366",
   "metadata": {},
   "source": [
    "Statistics for FPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8f63377b",
   "metadata": {},
   "outputs": [],
   "source": [
    "noemph=noemph.dropna(axis=1, how='all') #There are many columns only with nans - Some strange error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "3c179ea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Select rows with participant ids and create new cols with the total number of FPs and FNs for each participant\n",
    "mod_all=mod[~mod['participant_id'].isnull()]\n",
    "mod_all['fp_all']=0\n",
    "mod_all['fn_all']=0\n",
    "mod_all['fp_30_100']=0\n",
    "mod_all['fp_100_300']=0\n",
    "mod_all['fn_30_100']=0\n",
    "mod_all['fn_100_300']=0\n",
    "\n",
    "adv_all=adv[~adv['participant_id'].isnull()]\n",
    "adv_all['fp_all']=0\n",
    "adv_all['fn_all']=0\n",
    "adv_all['fp_30_100']=0\n",
    "adv_all['fp_100_300']=0\n",
    "adv_all['fn_30_100']=0\n",
    "adv_all['fn_100_300']=0\n",
    "\n",
    "conf_all=conf[~conf['participant_id'].isnull()]\n",
    "conf_all['fp_all']=0\n",
    "conf_all['fn_all']=0\n",
    "conf_all['fp_30_100']=0\n",
    "conf_all['fp_100_300']=0\n",
    "conf_all['fn_30_100']=0\n",
    "conf_all['fn_100_300']=0\n",
    "\n",
    "noemph_all=noemph[~noemph['participant_id'].isnull()]\n",
    "noemph_all['fp_all']=0\n",
    "noemph_all['fn_all']=0\n",
    "noemph_all['fp_30_100']=0\n",
    "noemph_all['fp_100_300']=0\n",
    "noemph_all['fn_30_100']=0\n",
    "noemph_all['fn_100_300']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "04e559ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop rows without participant IDs\n",
    "noemph_all=noemph_all.drop(index=noemph_all[noemph_all['participant_id'].str.contains('below|BELOW')==True].index)\n",
    "\n",
    "#Keep only valid IDs\n",
    "noemph_all['participant_id']=[int(str(pat)[:6]) for pat in noemph_all['participant_id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1d145a3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loop through all participants and add the number of FPs for AI for each participant (based on consensus review)\n",
    "for ind,pat in enumerate(mod['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in list(other_nonodules_FP_emph.keys()):            \n",
    "            mod_all.loc[ind,'fp_all']=mod_all.loc[ind,'fp_all']+len(other_nonodules_FP_emph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FP_emph.keys()):\n",
    "            mod_all.loc[ind,'fp_all']=mod_all.loc[ind,'fp_all']+len(fibrosis_FP_emph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        try:\n",
    "            if int(str(pat)[:6]) in list(fibrosis_FP_emph.keys()):\n",
    "                mod_all.loc[ind,'fp_all']=mod_all.loc[ind,'fp_all']+len(fibrosis_FP_emph[int(str(pat)[:6])])\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "for ind,pat in enumerate(adv['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FP_emph.keys())):            \n",
    "            adv_all.loc[ind,'fp_all']=adv_all.loc[ind,'fp_all']+len(other_nonodules_FP_emph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FP_emph.keys()):\n",
    "            adv_all.loc[ind,'fp_all']=adv_all.loc[ind,'fp_all']+len(fibrosis_FP_emph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        try:\n",
    "            if int(str(pat)[:6]) in list(fibrosis_FP_emph.keys()):\n",
    "                adv_all.loc[ind,'fp_all']=adv_all.loc[ind,'fp_all']+len(fibrosis_FP_emph[int(str(pat)[:6])])\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "\n",
    "for ind,pat in enumerate(conf['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FP_emph.keys())):            \n",
    "            conf_all.loc[ind,'fp_all']=conf_all.loc[ind,'fp_all']+len(other_nonodules_FP_emph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FP_emph.keys()):\n",
    "            conf_all.loc[ind,'fp_all']=conf_all.loc[ind,'fp_all']+len(fibrosis_FP_emph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        try:\n",
    "            if int(str(pat)[:6]) in list(fibrosis_FP_emph.keys()):\n",
    "                conf_all.loc[ind,'fp_all']=conf_all.loc[ind,'fp_all']+len(fibrosis_FP_emph[int(str(pat)[:6])])\n",
    "        except:\n",
    "            pass\n",
    "\n",
    "\n",
    "for ind,pat in enumerate(noemph['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FP_noemph.keys())):\n",
    "            noemph_all.loc[ind,'fp_all']=noemph_all.loc[ind,'fp_all']+len(other_nonodules_FP_noemph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FP_noemph.keys()):\n",
    "            noemph_all.loc[ind,'fp_all']=noemph_all.loc[ind,'fp_all']+len(fibrosis_FP_noemph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        try:\n",
    "            if int(str(pat)[:6]) in list(fibrosis_FP_noemph.keys()):\n",
    "                noemph_all.loc[ind,'fp_all']=noemph_all.loc[ind,'fp_all']+len(fibrosis_FP_noemph[int(str(pat)[:6])])\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "d7f7ec2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loop through all participants and add the number of FPs for the reader for each participant (based on consensus review)\n",
    "for ind,pat in enumerate(mod['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FN_emph.keys())):            \n",
    "            mod_all.loc[ind,'fn_all']=mod_all.loc[ind,'fn_all']+len(other_nonodules_FN_emph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FN_emph.keys()):\n",
    "            mod_all.loc[ind,'fn_all']=mod_all.loc[ind,'fn_all']+len(fibrosis_FN_emph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "for ind,pat in enumerate(adv['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FN_emph.keys())):            \n",
    "            adv_all.loc[ind,'fn_all']=adv_all.loc[ind,'fn_all']+len(other_nonodules_FN_emph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FN_emph.keys()):\n",
    "            adv_all.loc[ind,'fn_all']=adv_all.loc[ind,'fn_all']+len(fibrosis_FN_emph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "for ind,pat in enumerate(conf['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FN_emph.keys())):            \n",
    "            conf_all.loc[ind,'fn_all']=conf_all.loc[ind,'fn_all']+len(other_nonodules_FN_emph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FN_emph.keys()):\n",
    "            conf_all.loc[ind,'fn_all']=conf_all.loc[ind,'fn_all']+len(fibrosis_FN_emph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "\n",
    "for ind,pat in enumerate(noemph['participant_id']):\n",
    "    try: #avoid nan\n",
    "        if int(str(pat)[:6]) in (list(other_nonodules_FN_noemph.keys())):\n",
    "            noemph_all.loc[ind,'fn_all']=noemph_all.loc[ind,'fn_all']+len(other_nonodules_FN_noemph[int(str(pat)[:6])])\n",
    "        if int(str(pat)[:6]) in list(fibrosis_FN_noemph.keys()):\n",
    "            noemph_all.loc[ind,'fn_all']=noemph_all.loc[ind,'fn_all']+len(fibrosis_FN_noemph[int(str(pat)[:6])])\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "41ecdd6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of FPs for AI in emphysema cases: 20\n",
      "Num of FPs for AI in non-emphysema cases: 18\n",
      "Num of FPs for reader in emphysema cases: 6\n",
      "Num of FPs for reader in non-emphysema cases: 22\n"
     ]
    }
   ],
   "source": [
    "emph_all=adv_all.append(conf_all).append(mod_all) #Merge all emphysema dfs to one\n",
    "emph_all['participant_id']=[int(str(pat)[:6]) for pat in emph_all['participant_id']]\n",
    "\n",
    "emph_all.reset_index(inplace=True,drop=True) #Reset index\n",
    "noemph_all.reset_index(inplace=True,drop=True) #Reset index\n",
    "\n",
    "print(\"Num of FPs for AI in emphysema cases:\",np.sum(emph_all['fp_all']))\n",
    "print(\"Num of FPs for AI in non-emphysema cases:\",np.sum(noemph_all['fp_all']))\n",
    "\n",
    "print(\"Num of FPs for reader in emphysema cases:\",np.sum(emph_all['fn_all']))\n",
    "print(\"Num of FPs for reader in non-emphysema cases:\",np.sum(noemph_all['fn_all']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "33fea88c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Not used for now\n",
    "# print('Paired T-test')\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.ttest_rel(noemph_all['fn_all'], noemph_all['fp_all']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.ttest_rel(emph_all['fn_all'], emph_all['fp_all']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "# print('T-test of independent samples')\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.ttest_ind(noemph_all['fn_all'], noemph_all['fp_all']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.ttest_ind(emph_all['fn_all'], emph_all['fp_all']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "# print(\"Below only possible is independent samples t-test. Paired t-test does not make sense here.\")\n",
    "# print(\"Emphysema vs non-emphysema for reader\",stats.ttest_ind(noemph_all['fn_all'], emph_all['fn_all']).pvalue)\n",
    "# print(\"Emphysema vs non-emphysema for AI\",stats.ttest_ind(noemph_all['fp_all'], emph_all['fp_all']).pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b641b8f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emphysema vs non-emphysema for reader 0.2336660600138144\n",
      "Emphysema vs non-emphysema for AI 0.028342555964237917\n",
      "Reader vs AI for emphysema 0.019995826013995247\n",
      "Reader vs AI for non-emphysema 0.45823529886716086\n"
     ]
    }
   ],
   "source": [
    "# # conduct the Wilcoxon-Signed Rank Test\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.wilcoxon(noemph_all['fn_all'], noemph_all['fp_all']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.wilcoxon(emph_all['fn_all'], emph_all['fp_all']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "#For unequal sample size Mann-Whitney U test is used\n",
    "print(\"Emphysema vs non-emphysema for reader\",stats.mannwhitneyu(noemph_all['fn_all'], emph_all['fn_all']).pvalue)\n",
    "print(\"Emphysema vs non-emphysema for AI\",stats.mannwhitneyu(noemph_all['fp_all'], emph_all['fp_all']).pvalue)\n",
    "\n",
    "print(\"Reader vs AI for emphysema\",stats.mannwhitneyu(emph_all['fn_all'], emph_all['fp_all']).pvalue)\n",
    "print(\"Reader vs AI for non-emphysema\",stats.mannwhitneyu(noemph_all['fp_all'], noemph_all['fn_all']).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "8b9915ef",
   "metadata": {},
   "source": [
    "Statistics for FP for AI volume subgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bd513031",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "emph\n",
      "noemph\n"
     ]
    }
   ],
   "source": [
    "for emph in ['emph','noemph']:\n",
    "    print(emph)\n",
    "\n",
    "    for pat in eval('dict_FP_correct_'+emph+'_ids'):\n",
    "\n",
    "            for nod_id in eval('dict_FP_correct_'+emph+'_ids[pat]'):\n",
    "\n",
    "                if emph=='noemph':\n",
    "                    vol=float(noemph_dict_vol[pat][nod_id-1])\n",
    "                else:\n",
    "                    try:\n",
    "                        vol=float(mod_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        try:\n",
    "                            vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                        except:\n",
    "                            vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "                if vol>=30 and vol<=100:\n",
    "                    exec(\"index=\"+emph+\"_all[\"+emph+\"_all['participant_id']==int(pat)].index[0]\")\n",
    "                    exec(emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fp_30_100')]=\"+emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fp_30_100')]+1\")\n",
    "\n",
    "                elif vol>100 and vol<=300:\n",
    "                    exec(\"index=\"+emph+\"_all[\"+emph+\"_all['participant_id']==int(pat)].index[0]\")\n",
    "                    exec(emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fp_100_300')]=\"+emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fp_100_300')]+1\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9007b12f",
   "metadata": {},
   "source": [
    "##### AI missed, reader found\n",
    "\n",
    "Before running part below we should execute the other file ('patient_selection_emphysema_experiment.ipynb') to get dictionaries containing information about the ids of FNs. We need REDCap information to extract those \n",
    "\n",
    "Up until here there are 8 files generated that will be used by the other notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bf91d625",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Run other notebook and continue execution on next cell if it gives error\n",
    "#This requires the path 'emph_csv' that contains 6 different excel files, one for each degree of emphysema. Those are the REDCap exports\n",
    "\n",
    "try: #To ignore error and continue in next cell we need try-except and 'no raise error' flag\n",
    "    %run ./patient_selection_emphysema_experiment.ipynb --no-raise-error\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "74053509",
   "metadata": {},
   "source": [
    "FP for reader's volume subgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "a4b9375a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "emph\n",
      "noemph\n"
     ]
    }
   ],
   "source": [
    "for emph in ['emph','noemph']: \n",
    "    print(emph)\n",
    "    temp_emph=emph\n",
    "\n",
    "    for pat in eval('dict_FN_wrong_'+emph+'_ids'):\n",
    "\n",
    "            for nod_id,_ in enumerate(eval('dict_FN_wrong_'+emph+'_ids[pat]')):\n",
    "\n",
    "                vol=float(eval('dict_FN_wrong_'+emph+'_vols[pat][nod_id]'))\n",
    "\n",
    "\n",
    "                if vol>=30 and vol<=100:\n",
    "\n",
    "                    if emph!='noemph':\n",
    "                        emph='emph'\n",
    "\n",
    "                    exec(\"index=\"+emph+\"_all[\"+emph+\"_all['participant_id']==int(pat)].index[0]\")\n",
    "                    exec(emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fn_30_100')]=\"+emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fn_30_100')]+1\")\n",
    "\n",
    "                elif vol>100 and vol<=300:\n",
    "\n",
    "                    if emph!='noemph':\n",
    "                        emph='emph'\n",
    "\n",
    "                    exec(\"index=\"+emph+\"_all[\"+emph+\"_all['participant_id']==int(pat)].index[0]\")\n",
    "                    exec(emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fn_100_300')]=\"+emph+\"_all.iloc[index,\"+emph+\"_all.columns.get_loc('fn_100_300')]+1\")\n",
    "\n",
    "                emph=temp_emph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3439db89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FP AI emph 20\n",
      "FP AI noemph 18\n",
      "FN read emph 6\n",
      "FN read noemph 22\n"
     ]
    }
   ],
   "source": [
    "assert list(emph_all['fp_all'])==list(emph_all['fp_30_100']+emph_all['fp_100_300'])\n",
    "assert list(emph_all['fn_all'])==list(emph_all['fn_30_100']+emph_all['fn_100_300'])\n",
    "assert list(noemph_all['fp_all'])==list(noemph_all['fp_30_100']+noemph_all['fp_100_300'])\n",
    "assert list(noemph_all['fn_all'])==list(noemph_all['fn_30_100']+noemph_all['fn_100_300'])\n",
    "\n",
    "assert np.sum(emph_all['fp_all'])==np.sum(emph_all['fp_30_100'])+np.sum(emph_all['fp_100_300'])\n",
    "assert np.sum(emph_all['fn_all'])==np.sum(emph_all['fn_30_100'])+np.sum(emph_all['fn_100_300'])\n",
    "assert np.sum(noemph_all['fp_all'])==np.sum(noemph_all['fp_30_100'])+np.sum(noemph_all['fp_100_300'])\n",
    "assert np.sum(noemph_all['fn_all'])==np.sum(noemph_all['fn_30_100'])+np.sum(noemph_all['fn_100_300'])\n",
    "\n",
    "print(\"FP AI emph\",np.sum(emph_all['fp_all']))\n",
    "print(\"FP AI noemph\",np.sum(noemph_all['fp_all']))\n",
    "print(\"FN read emph\",np.sum(emph_all['fn_all']))\n",
    "print(\"FN read noemph\",np.sum(noemph_all['fn_all']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "b0b8627d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Volume subgroup 30-100mm3\n",
      "For unequal sample size Mann-Whitney U test is used\n",
      "Non-emphysema Reader vs AI: 0.008976111737279201\n",
      "Emphysema Reader vs AI: 0.34982468520917553\n",
      "Emphysema vs Non-emphysema for Reader: 0.2049544529370757\n",
      "Emphysema vs Non-emphysema for AI: 0.04031313689234626\n"
     ]
    }
   ],
   "source": [
    "print(\"Volume subgroup 30-100mm3\")\n",
    "# print('Paired T-test')\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.ttest_rel(noemph_all['fn_30_100'], noemph_all['fp_30_100']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.ttest_rel(emph_all['fn_30_100'], emph_all['fp_30_100']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "# print('T-test of independent samples')\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.ttest_ind(noemph_all['fn_30_100'], noemph_all['fp_30_100']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.ttest_ind(emph_all['fn_30_100'], emph_all['fp_30_100']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "# print(\"Below only possible is independent samples t-test. Paired t-test does not make sense here.\")\n",
    "# print(\"Emphysema vs non-emphysema for reader\",stats.ttest_ind(noemph_all['fn_30_100'], emph_all['fn_30_100']).pvalue)\n",
    "# print(\"Emphysema vs non-emphysema for AI\",stats.ttest_ind(noemph_all['fp_30_100'], emph_all['fp_30_100']).pvalue)\n",
    "# print('\\n')\n",
    "\n",
    "# print(\"Below Wilcoxon-Signed Rank Test is used\")\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.wilcoxon(noemph_all['fn_30_100'], noemph_all['fp_30_100']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.wilcoxon(emph_all['fn_30_100'], emph_all['fp_30_100']).pvalue)\n",
    "# print('\\n')\n",
    "\n",
    "# print(\"For unequal sample size Mann-Whitney U test is used\")\n",
    "# print(\"Emphysema vs non-emphysema for reader\",stats.mannwhitneyu(noemph_all['fn_30_100'], emph_all['fn_30_100']).pvalue)\n",
    "# print(\"Emphysema vs non-emphysema for AI\",stats.mannwhitneyu(noemph_all['fp_30_100'], emph_all['fp_30_100']).pvalue)\n",
    "\n",
    "print(\"For unequal sample size Mann-Whitney U test is used\")\n",
    "print(\"Non-emphysema Reader vs AI:\",stats.mannwhitneyu(noemph_all['fn_30_100'], noemph_all['fp_30_100']).pvalue)\n",
    "print(\"Emphysema Reader vs AI:\",stats.mannwhitneyu(emph_all['fn_30_100'], emph_all['fp_30_100']).pvalue)\n",
    "\n",
    "print(\"Emphysema vs Non-emphysema for Reader:\",stats.mannwhitneyu(noemph_all['fn_30_100'], emph_all['fn_30_100']).pvalue)\n",
    "print(\"Emphysema vs Non-emphysema for AI:\",stats.mannwhitneyu(noemph_all['fp_30_100'], emph_all['fp_30_100']).pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "44b1c2e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Volume subgroup 100-300mm3\n",
      "For unequal sample size Mann-Whitney U test is used\n",
      "Non-emphysema Reader vs AI: 0.009245879014973505\n",
      "Emphysema Reader vs AI: 0.013486511498683289\n",
      "Emphysema vs Non-emphysema for Reader: 0.9753599246778553\n",
      "Emphysema vs Non-emphysema for AI: 0.30435836349567624\n"
     ]
    }
   ],
   "source": [
    "print(\"Volume subgroup 100-300mm3\")\n",
    "# print('Paired T-test')\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.ttest_rel(noemph_all['fn_100_300'], noemph_all['fp_100_300']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.ttest_rel(emph_all['fn_100_300'], emph_all['fp_100_300']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "# print('T-test of independent samples')\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.ttest_ind(noemph_all['fn_100_300'], noemph_all['fp_100_300']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.ttest_ind(emph_all['fn_100_300'], emph_all['fp_100_300']).pvalue) \n",
    "# print('\\n')\n",
    "\n",
    "# print(\"Below only possible is independent samples t-test. Paired t-test does not make sense here.\")\n",
    "# print(\"Emphysema vs non-emphysema for reader\",stats.ttest_ind(noemph_all['fn_100_300'], emph_all['fn_100_300']).pvalue)\n",
    "# print(\"Emphysema vs non-emphysema for AI\",stats.ttest_ind(noemph_all['fp_100_300'], emph_all['fp_100_300']).pvalue)\n",
    "# print('\\n')\n",
    "\n",
    "# print(\"Below Wilcoxon-Signed Rank Test is used\")\n",
    "# print(\"Non-emphysema Reader vs AI:\",stats.wilcoxon(noemph_all['fn_100_300'], noemph_all['fp_100_300']).pvalue)\n",
    "# print(\"Emphysema Reader vs AI:\",stats.wilcoxon(emph_all['fn_100_300'], emph_all['fp_100_300']).pvalue)\n",
    "# print('\\n')\n",
    "\n",
    "# print(\"For unequal sample size Mann-Whitney U test is used\")\n",
    "# print(\"Emphysema vs non-emphysema for reader\",stats.mannwhitneyu(noemph_all['fn_100_300'], emph_all['fn_100_300']).pvalue)\n",
    "# print(\"Emphysema vs non-emphysema for AI\",stats.mannwhitneyu(noemph_all['fp_100_300'], emph_all['fp_100_300']).pvalue)\n",
    "\n",
    "print(\"For unequal sample size Mann-Whitney U test is used\")\n",
    "print(\"Non-emphysema Reader vs AI:\",stats.mannwhitneyu(noemph_all['fn_100_300'], noemph_all['fp_100_300']).pvalue)\n",
    "print(\"Emphysema Reader vs AI:\",stats.mannwhitneyu(emph_all['fn_100_300'], emph_all['fp_100_300']).pvalue)\n",
    "\n",
    "print(\"Emphysema vs Non-emphysema for Reader:\",stats.mannwhitneyu(noemph_all['fn_100_300'], emph_all['fn_100_300']).pvalue)\n",
    "print(\"Emphysema vs Non-emphysema for AI:\",stats.mannwhitneyu(noemph_all['fp_100_300'], emph_all['fp_100_300']).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "be5f1864",
   "metadata": {},
   "source": [
    "Load dictionaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "8219c016",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load ids of FNs\n",
    "\n",
    "with open('dict_FN_wrong_emph_ids.pickle', 'rb') as f:\n",
    "    dict_FN_wrong_emph_ids = pickle.load(f)\n",
    "\n",
    "with open('dict_FN_correct_emph_ids.pickle', 'rb') as f:\n",
    "    dict_FN_correct_emph_ids = pickle.load(f)\n",
    "\n",
    "with open('dict_FN_wrong_noemph_ids.pickle', 'rb') as f:\n",
    "    dict_FN_wrong_noemph_ids = pickle.load(f)\n",
    "    \n",
    "with open('dict_FN_correct_noemph_ids.pickle', 'rb') as f:\n",
    "    dict_FN_correct_noemph_ids = pickle.load(f)\n",
    "    \n",
    "    \n",
    "#Same for their vols\n",
    "\n",
    "with open('dict_FN_wrong_emph_vols.pickle', 'rb') as f:\n",
    "    dict_FN_wrong_emph_vols = pickle.load(f)\n",
    "\n",
    "with open('dict_FN_correct_emph_vols.pickle', 'rb') as f:\n",
    "    dict_FN_correct_emph_vols = pickle.load(f)\n",
    "\n",
    "with open('dict_FN_wrong_noemph_vols.pickle', 'rb') as f:\n",
    "    dict_FN_wrong_noemph_vols = pickle.load(f)\n",
    "    \n",
    "with open('dict_FN_correct_noemph_vols.pickle', 'rb') as f:\n",
    "    dict_FN_correct_noemph_vols = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a80fa5ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Similarly for lymph nodes and nodules only and of their ids and volumes\n",
    "\n",
    "with open('lymph_FN_correct_emph.pickle','rb') as f:\n",
    "    lymph_FN_correct_emph=pickle.load(f)    \n",
    "\n",
    "with open('lymph_FN_correct_noemph.pickle','rb') as f:\n",
    "    lymph_FN_correct_noemph=pickle.load(f) \n",
    "    \n",
    "with open('nod_FN_correct_emph.pickle','rb') as f:\n",
    "    nod_FN_correct_emph=pickle.load(f)    \n",
    "\n",
    "with open('nod_FN_correct_noemph.pickle','rb') as f:\n",
    "    nod_FN_correct_noemph=pickle.load(f) \n",
    "    \n",
    "    \n",
    "with open('lymph_FN_correct_emph_ids.pickle','rb') as f:\n",
    "    lymph_FN_correct_emph_ids=pickle.load(f)    \n",
    "\n",
    "with open('lymph_FN_correct_noemph_ids.pickle','rb') as f:\n",
    "    lymph_FN_correct_noemph_ids=pickle.load(f) \n",
    "    \n",
    "with open('nod_FN_correct_emph_ids.pickle','rb') as f:\n",
    "    nod_FN_correct_emph_ids=pickle.load(f)    \n",
    "\n",
    "with open('nod_FN_correct_noemph_ids.pickle','rb') as f:\n",
    "    nod_FN_correct_noemph_ids=pickle.load(f) \n",
    "    \n",
    "    \n",
    "with open('lymph_FN_correct_emph_vols.pickle','rb') as f:\n",
    "    lymph_FN_correct_emph_vols=pickle.load(f)    \n",
    "\n",
    "with open('lymph_FN_correct_noemph_vols.pickle','rb') as f:\n",
    "    lymph_FN_correct_noemph_vols=pickle.load(f) \n",
    "    \n",
    "with open('nod_FN_correct_emph_vols.pickle','rb') as f:\n",
    "    nod_FN_correct_emph_vols=pickle.load(f)    \n",
    "\n",
    "with open('nod_FN_correct_noemph_vols.pickle','rb') as f:\n",
    "    nod_FN_correct_noemph_vols=pickle.load(f)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f12dace4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Initialize zero values for non-nodules, nodules only, and lymph nodes for each volume subgroup and for each emph/non-emph - All '_300' should be 0\n",
    "\n",
    "reader_nonods_noemph_30_100=0\n",
    "reader_nonods_noemph_100_300=0\n",
    "reader_nonods_noemph_300=0\n",
    "\n",
    "reader_nonods_emph_30_100=0\n",
    "reader_nonods_emph_100_300=0\n",
    "reader_nonods_emph_300=0\n",
    "\n",
    "\n",
    "reader_only_nods_noemph_30_100=0\n",
    "reader_only_nods_noemph_100_300=0\n",
    "reader_only_nods_noemph_300=0\n",
    "\n",
    "reader_only_nods_emph_30_100=0\n",
    "reader_only_nods_emph_100_300=0\n",
    "reader_only_nods_emph_300=0\n",
    "\n",
    "\n",
    "reader_lymph_noemph_30_100=0\n",
    "reader_lymph_noemph_100_300=0\n",
    "reader_lymph_noemph_300=0\n",
    "\n",
    "reader_lymph_emph_30_100=0\n",
    "reader_lymph_emph_100_300=0\n",
    "reader_lymph_emph_300=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "d43c205e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Similarly keep track of volumes for each of those groups (if Mann-Whitney U test is used below)\n",
    "reader_nonods_noemph_30_100_vols=[]\n",
    "reader_nonods_noemph_100_300_vols=[]\n",
    "reader_nonods_noemph_300_vols=[]\n",
    "reader_nonods_emph_30_100_vols=[]\n",
    "reader_nonods_emph_100_300_vols=[]\n",
    "reader_nonods_emph_300_vols=[]\n",
    "\n",
    "reader_only_nods_noemph_30_100_vols=[]\n",
    "reader_only_nods_noemph_100_300_vols=[]\n",
    "reader_only_nods_noemph_300_vols=[]\n",
    "reader_only_nods_emph_30_100_vols=[]\n",
    "reader_only_nods_emph_100_300_vols=[]\n",
    "reader_only_nods_emph_300_vols=[]\n",
    "\n",
    "reader_lymph_noemph_30_100_vols=[]\n",
    "reader_lymph_noemph_100_300_vols=[]\n",
    "reader_lymph_noemph_300_vols=[]\n",
    "reader_lymph_emph_30_100_vols=[]\n",
    "reader_lymph_emph_100_300_vols=[]\n",
    "reader_lymph_emph_300_vols=[]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cd23ef17",
   "metadata": {},
   "source": [
    "Get numbers of reader nodules for lymph nodes only, nodules only, and non-nodule categories in emphysema/non-emphysema groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "b8efdd57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total non-nodules in emph group is 6\n",
      "Total non-nodules in noemph group is 22\n"
     ]
    }
   ],
   "source": [
    "#Similarly for non-nodule emphysema/non-emphysema groups for FNs\n",
    "\n",
    "for emph in ['emph','noemph']:\n",
    "    tot=0\n",
    "    for pat in eval('dict_FN_wrong_'+emph+'_ids'):\n",
    "        for ind,nod_id in enumerate(eval('dict_FN_wrong_'+emph+'_ids[pat]')):\n",
    "\n",
    "            if emph=='noemph':\n",
    "                vol=float(dict_FN_wrong_noemph_vols[pat][ind])\n",
    "            else:\n",
    "                try:\n",
    "                    #It might not exist there since only FP - AI nods exist in dicts like 'mod_dict_vol[pat][nod_id-1]'\n",
    "                    vol=float(dict_FN_wrong_emph_vols[pat][ind]) \n",
    "                except: #We should never get in here\n",
    "                    print(\"We shouldn't be here\")\n",
    "                    try:\n",
    "                        vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "            if vol>=30 and vol<=100:\n",
    "                exec('reader_nonods_'+emph+'_30_100=reader_nonods_'+emph+'_30_100+1')\n",
    "                exec('reader_nonods_'+emph+'_30_100_vols.append(vol)')\n",
    "                tot=tot+1\n",
    "            elif vol>100 and vol<=300:\n",
    "                exec('reader_nonods_'+emph+'_100_300=reader_nonods_'+emph+'_100_300+1')\n",
    "                exec('reader_nonods_'+emph+'_100_300_vols.append(vol)')\n",
    "                tot=tot+1\n",
    "            elif vol>300:\n",
    "                exec('reader_nonods_'+emph+'_300=reader_nonods_'+emph+'_300+1') #Should be 0\n",
    "                exec('reader_nonods_'+emph+'_300_vols.append(vol)') #Should be 0\n",
    "            else:\n",
    "                print('For participant {} volume is smaller than 30mm3',pat)\n",
    "\n",
    "    print('Total non-nodules in {} group is {}'.format(emph,tot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "99be2e8e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total lymphs in emph group is 16\n",
      "Total lymphs in noemph group is 19\n"
     ]
    }
   ],
   "source": [
    "#Similarly for lymph nodes emphysema/non-emphysema groups for FNs\n",
    "\n",
    "for emph in ['emph','noemph']:\n",
    "    tot=0\n",
    "    for pat in eval('lymph_FN_correct_'+emph+'_ids'):\n",
    "        for ind,nod_id in enumerate(eval('lymph_FN_correct_'+emph+'_ids[pat]')):\n",
    "            \n",
    "            if emph=='noemph':\n",
    "                vol=float(lymph_FN_correct_noemph_vols[pat][ind])\n",
    "            else:\n",
    "                try:\n",
    "                    vol=float(lymph_FN_correct_emph_vols[pat][ind]) \n",
    "                except: #We should never get in here\n",
    "                    print(\"We shouldn't be here\")\n",
    "                    try:\n",
    "                        vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "            if vol>=30 and vol<=100:\n",
    "                exec('reader_lymph_'+emph+'_30_100=reader_lymph_'+emph+'_30_100+1')\n",
    "                exec('reader_lymph_'+emph+'_30_100_vols.append(vol)')\n",
    "                tot=tot+1\n",
    "            elif vol>100 and vol<=300:\n",
    "                exec('reader_lymph_'+emph+'_100_300=reader_lymph_'+emph+'_100_300+1')\n",
    "                exec('reader_lymph_'+emph+'_100_300_vols.append(vol)')\n",
    "                tot=tot+1\n",
    "            elif vol>300:\n",
    "                exec('reader_lymph_'+emph+'_300=reader_lymph_'+emph+'_300+1') #Should be 0\n",
    "                exec('reader_lymph_'+emph+'_300_vols.append(vol)')\n",
    "            else:\n",
    "                print('For participant {} volume is smaller than 30mm3',pat)\n",
    "\n",
    "    print('Total lymphs in {} group is {}'.format(emph,tot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "d778a2a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total nodules only in emph group is 13\n",
      "Total nodules only in noemph group is 21\n"
     ]
    }
   ],
   "source": [
    "#Similarly for nodules only emphysema/non-emphysema groups for FNs\n",
    "for emph in ['emph','noemph']:\n",
    "    tot=0\n",
    "    for pat in eval('nod_FN_correct_'+emph+'_ids'):\n",
    "        for ind,nod_id in enumerate(eval('nod_FN_correct_'+emph+'_ids[pat]')):\n",
    "\n",
    "            if emph=='noemph':\n",
    "                vol=float(nod_FN_correct_noemph_vols[pat][ind])\n",
    "            else:\n",
    "                try:\n",
    "                    vol=float(nod_FN_correct_emph_vols[pat][ind]) \n",
    "                except: #We should never get in here\n",
    "                    print(\"We shouldn't be here\")\n",
    "                    try:\n",
    "                        vol=float(conf_dict_vol[pat][nod_id-1])\n",
    "                    except:\n",
    "                        vol=float(adv_dict_vol[pat][nod_id-1])\n",
    "\n",
    "            if vol>=30 and vol<=100:\n",
    "                exec('reader_only_nods_'+emph+'_30_100=reader_only_nods_'+emph+'_30_100+1')\n",
    "                exec('reader_only_nods_'+emph+'_30_100_vols.append(vol)')\n",
    "                tot=tot+1\n",
    "            elif vol>100 and vol<=300:\n",
    "                exec('reader_only_nods_'+emph+'_100_300=reader_only_nods_'+emph+'_100_300+1')\n",
    "                exec('reader_only_nods_'+emph+'_100_300_vols.append(vol)')\n",
    "                tot=tot+1\n",
    "            elif vol>300:\n",
    "                exec('reader_only_nods_'+emph+'_300=reader_only_nods_'+emph+'_300+1') #Should be 0\n",
    "                exec('reader_only_nods_'+emph+'_300_vols.append(vol)')\n",
    "            else:\n",
    "                print('For participant {} volume is smaller than 30mm3',pat)\n",
    "\n",
    "    print('Total nodules only in {} group is {}'.format(emph,tot))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "efe43e9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Total number of nodules in each of the emphysema/non-emphysema groups is the sum of the nodules and lymph nodes in those\n",
    "\n",
    "reader_nods_noemph_30_100=reader_lymph_noemph_30_100+reader_only_nods_noemph_30_100\n",
    "reader_nods_noemph_100_300=reader_lymph_noemph_100_300+reader_only_nods_noemph_100_300\n",
    "reader_nods_noemph_300=reader_lymph_noemph_300+reader_only_nods_noemph_300\n",
    "\n",
    "reader_nods_emph_30_100=reader_lymph_emph_30_100+reader_only_nods_emph_30_100\n",
    "reader_nods_emph_100_300=reader_lymph_emph_100_300+reader_only_nods_emph_100_300\n",
    "reader_nods_emph_300=reader_lymph_emph_300+reader_only_nods_emph_300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a44d51b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "reader_only_nods_noemph_30_100_vols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "bcef7698",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reader_only_nods_noemph_30_100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "30164b0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert reader_nods_noemph_300==reader_nods_emph_300==0"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "af5d3ae1",
   "metadata": {},
   "source": [
    "## Create Tables & Statistics"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c2ba1b2f",
   "metadata": {},
   "source": [
    "##### Based on the current definition the following equations hold true:\n",
    "1. AI found nodules, reader missed = FN reader\n",
    "2. AI found non-nodules, reader missed = FP AI\n",
    "3. AI missed nodules, reader found = FN AI\n",
    "4. AI missed non-nodules, reader found = FP reader"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c07312b8",
   "metadata": {},
   "source": [
    "#### Emphysema non-nodules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "2a8ae87b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fibrosis/scar emphysema FP: 13\n",
      "Other non-nodules emphysema FP: 7\n",
      "Other non-nodules emphysema FP (lung): 5\n",
      "Other non-nodules emphysema FP (non-lung): 1\n",
      "Fibrosis/scar emphysema FN: 1\n",
      "Other non-nodules emphysema FN: 5\n",
      "Other non-nodules emphysema FN (lung): 1\n",
      "Other non-nodules emphysema FN (non-lung): 2\n"
     ]
    }
   ],
   "source": [
    "#Below are the non-nodule categories. With FP is denoted a finding that was missed by AI, whereas with FN a finding missed by the reader\n",
    "#Transform above dictionaries to numbers to be used below\n",
    "fibrosis_FP_emph=sum([len(x) for x in fibrosis_FP_emph.values()])\n",
    "other_nonodules_FP_emph=sum([len(x) for x in other_nonodules_FP_emph.values()])\n",
    "fibrosis_FN_emph=sum([len(x) for x in fibrosis_FN_emph.values()])\n",
    "other_nonodules_FN_emph=sum([len(x) for x in other_nonodules_FN_emph.values()])\n",
    "other_nonodules_FN_lung_emph=sum([len(x) for x in other_nonodules_FN_lung_emph.values()])\n",
    "other_nonodules_FN_nolung_emph=sum([len(x) for x in other_nonodules_FN_nolung_emph.values()])\n",
    "other_nonodules_FP_lung_emph=sum([len(x) for x in other_nonodules_FP_lung_emph.values()])\n",
    "other_nonodules_FP_nolung_emph=sum([len(x) for x in other_nonodules_FP_nolung_emph.values()])\n",
    "\n",
    "#Print the above\n",
    "print('Fibrosis/scar emphysema FP: '+str(fibrosis_FP_emph))\n",
    "print('Other non-nodules emphysema FP: '+str(other_nonodules_FP_emph))\n",
    "print('Other non-nodules emphysema FP (lung): '+str(other_nonodules_FP_lung_emph))\n",
    "print('Other non-nodules emphysema FP (non-lung): '+str(other_nonodules_FP_nolung_emph))\n",
    "print('Fibrosis/scar emphysema FN: '+str(fibrosis_FN_emph))\n",
    "print('Other non-nodules emphysema FN: '+str(other_nonodules_FN_emph))\n",
    "print('Other non-nodules emphysema FN (lung): '+str(other_nonodules_FN_lung_emph))\n",
    "print('Other non-nodules emphysema FN (non-lung): '+str(other_nonodules_FN_nolung_emph))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "70925dd8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Incorrectly detected by AI</th>\n",
       "      <th>Incorrectly detected by reader</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fibrosis/scar emphysema</th>\n",
       "      <td>13 (50.0%)</td>\n",
       "      <td>1 (3.8%)</td>\n",
       "      <td>14 (53.8%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules emphysema</th>\n",
       "      <td>7 (26.9%)</td>\n",
       "      <td>5 (19.2%)</td>\n",
       "      <td>12 (46.2%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>20 (76.9%)</td>\n",
       "      <td>6 (23.1%)</td>\n",
       "      <td>26 (100.0%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Incorrectly detected by AI  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "fibrosis/scar emphysema                              13 (50.0%)   \n",
       "other non-nodules emphysema                           7 (26.9%)   \n",
       "Total                                                20 (76.9%)   \n",
       "\n",
       "                                     Incorrectly detected by reader  \\\n",
       "GT by radiologists for discrepancies                                  \n",
       "fibrosis/scar emphysema                                    1 (3.8%)   \n",
       "other non-nodules emphysema                               5 (19.2%)   \n",
       "Total                                                     6 (23.1%)   \n",
       "\n",
       "                                     All findings  \n",
       "GT by radiologists for discrepancies               \n",
       "fibrosis/scar emphysema                14 (53.8%)  \n",
       "other non-nodules emphysema            12 (46.2%)  \n",
       "Total                                 26 (100.0%)  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Detailed comparison of FP categories for emphysema and non-emphysema groups (no volume subgroups)\n",
    "\n",
    "df_categories=pd.DataFrame(columns=['Incorrectly detected by AI','Incorrectly detected by reader'], #below index with the correct order as above\n",
    "                          index=['fibrosis/scar emphysema','other non-nodules emphysema'])\n",
    "\n",
    "df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['Incorrectly detected by AI']=[fibrosis_FP_emph,other_nonodules_FP_emph]\n",
    "\n",
    "df_categories['Incorrectly detected by reader']=[fibrosis_FN_emph,other_nonodules_FN_emph]\n",
    "\n",
    "df_categories['All findings']=df_categories['Incorrectly detected by AI']+df_categories['Incorrectly detected by reader'] #Sum of findings for each of emph/non-emph categories\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum() #Total FP findings for AI/reader\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum() #All findings\n",
    "\n",
    "#Add percentages next to the number of each category\n",
    "percentage_fp=np.round((df_categories['Incorrectly detected by AI']/all_findings)*100,1)  \n",
    "df_categories['Incorrectly detected by AI']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' \n",
    "                                             for index,value in enumerate(df_categories['Incorrectly detected by AI'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['Incorrectly detected by reader']/all_findings)*100,1) \n",
    "df_categories['Incorrectly detected by reader']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' \n",
    "                                                 for index,value in enumerate(df_categories['Incorrectly detected by reader'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/all_findings,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "# #Rename columns\n",
    "# df_categories.rename(columns={'FP': 'Incorrectly detected by AI', 'FN': 'Incorrectly detected by reader'}, inplace=True)\n",
    "\n",
    "df_categories\n",
    "# ‘Other’ could be bone, tissue, mucus, arthrosis, vessel, consolidation, infection, fat, or atelectasis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "56597ec0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.style.to_latex() #Just as a starting point - Need to be modified manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "b3e9fadf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_categories.to_excel('non_nodules_emphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "8eb8610d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Incorrectly detected by AI</th>\n",
       "      <th>Incorrectly detected by reader</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fibrosis/scar emphysema</th>\n",
       "      <td>13 (50.0%)</td>\n",
       "      <td>1 (3.8%)</td>\n",
       "      <td>14 (53.8%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules lung emphysema</th>\n",
       "      <td>5 (19.2%)</td>\n",
       "      <td>1 (3.8%)</td>\n",
       "      <td>6 (23.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules nolung emphysema</th>\n",
       "      <td>1 (3.8%)</td>\n",
       "      <td>2 (7.7%)</td>\n",
       "      <td>3 (11.5%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules (no description)</th>\n",
       "      <td>1 (3.8%)</td>\n",
       "      <td>2 (7.7%)</td>\n",
       "      <td>3 (11.5%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>20 (76.9%)</td>\n",
       "      <td>6 (23.1%)</td>\n",
       "      <td>26 (100.0%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Incorrectly detected by AI  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "fibrosis/scar emphysema                              13 (50.0%)   \n",
       "other non-nodules lung emphysema                      5 (19.2%)   \n",
       "other non-nodules nolung emphysema                     1 (3.8%)   \n",
       "other non-nodules (no description)                     1 (3.8%)   \n",
       "Total                                                20 (76.9%)   \n",
       "\n",
       "                                     Incorrectly detected by reader  \\\n",
       "GT by radiologists for discrepancies                                  \n",
       "fibrosis/scar emphysema                                    1 (3.8%)   \n",
       "other non-nodules lung emphysema                           1 (3.8%)   \n",
       "other non-nodules nolung emphysema                         2 (7.7%)   \n",
       "other non-nodules (no description)                         2 (7.7%)   \n",
       "Total                                                     6 (23.1%)   \n",
       "\n",
       "                                     All findings  \n",
       "GT by radiologists for discrepancies               \n",
       "fibrosis/scar emphysema                14 (53.8%)  \n",
       "other non-nodules lung emphysema        6 (23.1%)  \n",
       "other non-nodules nolung emphysema      3 (11.5%)  \n",
       "other non-nodules (no description)      3 (11.5%)  \n",
       "Total                                 26 (100.0%)  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Detailed comparison of FP categories for emphysema and non-emphysema groups (no volume subgroups)\n",
    "\n",
    "df_categories=pd.DataFrame(columns=['Incorrectly detected by AI','Incorrectly detected by reader'], #below index with the correct order as above\n",
    "                          index=['fibrosis/scar emphysema','other non-nodules lung emphysema','other non-nodules nolung emphysema','other non-nodules (no description)'])\n",
    "\n",
    "df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "rest_no_desc_fp=other_nonodules_FP_emph-other_nonodules_FP_lung_emph-other_nonodules_FP_nolung_emph\n",
    "df_categories['Incorrectly detected by AI']=[fibrosis_FP_emph,other_nonodules_FP_lung_emph, other_nonodules_FP_nolung_emph, rest_no_desc_fp]\n",
    "\n",
    "rest_no_desc_fn=other_nonodules_FN_emph-other_nonodules_FN_lung_emph-other_nonodules_FN_nolung_emph\n",
    "df_categories['Incorrectly detected by reader']=[fibrosis_FN_emph,other_nonodules_FN_lung_emph, other_nonodules_FN_nolung_emph, rest_no_desc_fn]\n",
    "\n",
    "df_categories['All findings']=df_categories['Incorrectly detected by AI']+df_categories['Incorrectly detected by reader'] #Sum of findings for each of emph/non-emph categories\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum() #Total FP findings for AI/reader\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum() #All findings\n",
    "\n",
    "#Add percentages next to the number of each category\n",
    "percentage_fp=np.round((df_categories['Incorrectly detected by AI']/all_findings)*100,1)  \n",
    "df_categories['Incorrectly detected by AI']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' \n",
    "                                             for index,value in enumerate(df_categories['Incorrectly detected by AI'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['Incorrectly detected by reader']/all_findings)*100,1) \n",
    "df_categories['Incorrectly detected by reader']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' \n",
    "                                                 for index,value in enumerate(df_categories['Incorrectly detected by reader'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/all_findings,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "df_categories\n",
    "# ‘Other’ could be bone, tissue, mucus, arthrosis, vessel, consolidation, infection, fat, or atelectasis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "ec5e2008",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('non_nodules_types_emphysema.xlsx')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6d4fc869",
   "metadata": {},
   "source": [
    "#### Non-emphysema non-nodules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "dfa8a7d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fibrosis/scar non-emphysema FP: 9\n",
      "Other non-nodules non-emphysema FP: 9\n",
      "Other non-nodules non-emphysema FP (lung): 6\n",
      "Other non-nodules non-emphysema FP (non-lung): 2\n",
      "Fibrosis/scar non-emphysema FN: 3\n",
      "Other non-nodules non-emphysema FN: 19\n",
      "Other non-nodules non-emphysema FN (lung): 4\n",
      "Other non-nodules non-emphysema FN (non-lung): 11\n"
     ]
    }
   ],
   "source": [
    "#Same as above for non-emphysema group\n",
    "fibrosis_FP_noemph=sum([len(x) for x in fibrosis_FP_noemph.values()])\n",
    "other_nonodules_FP_noemph=sum([len(x) for x in other_nonodules_FP_noemph.values()])\n",
    "other_nonodules_FP_lung_noemph=sum([len(x) for x in other_nonodules_FP_lung_noemph.values()])\n",
    "other_nonodules_FP_nolung_noemph=sum([len(x) for x in other_nonodules_FP_nolung_noemph.values()])\n",
    "fibrosis_FN_noemph=sum([len(x) for x in fibrosis_FN_noemph.values()])\n",
    "other_nonodules_FN_noemph=sum([len(x) for x in other_nonodules_FN_noemph.values()])\n",
    "other_nonodules_FN_lung_noemph=sum([len(x) for x in other_nonodules_FN_lung_noemph.values()])\n",
    "other_nonodules_FN_nolung_noemph=sum([len(x) for x in other_nonodules_FN_nolung_noemph.values()])\n",
    "\n",
    "#Print the above\n",
    "print('Fibrosis/scar non-emphysema FP: '+str(fibrosis_FP_noemph))\n",
    "print('Other non-nodules non-emphysema FP: '+str(other_nonodules_FP_noemph))\n",
    "print('Other non-nodules non-emphysema FP (lung): '+str(other_nonodules_FP_lung_noemph))\n",
    "print('Other non-nodules non-emphysema FP (non-lung): '+str(other_nonodules_FP_nolung_noemph))\n",
    "print('Fibrosis/scar non-emphysema FN: '+str(fibrosis_FN_noemph))\n",
    "print('Other non-nodules non-emphysema FN: '+str(other_nonodules_FN_noemph))\n",
    "print('Other non-nodules non-emphysema FN (lung): '+str(other_nonodules_FN_lung_noemph))\n",
    "print('Other non-nodules non-emphysema FN (non-lung): '+str(other_nonodules_FN_nolung_noemph))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "388bda82",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Incorrectly detected by AI</th>\n",
       "      <th>Incorrectly detected by reader</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fibrosis/scar non-emphysema</th>\n",
       "      <td>9 (22.5%)</td>\n",
       "      <td>3 (7.5%)</td>\n",
       "      <td>12 (30.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules non-emphysema</th>\n",
       "      <td>9 (22.5%)</td>\n",
       "      <td>19 (47.5%)</td>\n",
       "      <td>28 (70.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>18 (45.0%)</td>\n",
       "      <td>22 (55.0%)</td>\n",
       "      <td>40 (100.0%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Incorrectly detected by AI  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "fibrosis/scar non-emphysema                           9 (22.5%)   \n",
       "other non-nodules non-emphysema                       9 (22.5%)   \n",
       "Total                                                18 (45.0%)   \n",
       "\n",
       "                                     Incorrectly detected by reader  \\\n",
       "GT by radiologists for discrepancies                                  \n",
       "fibrosis/scar non-emphysema                                3 (7.5%)   \n",
       "other non-nodules non-emphysema                          19 (47.5%)   \n",
       "Total                                                    22 (55.0%)   \n",
       "\n",
       "                                     All findings  \n",
       "GT by radiologists for discrepancies               \n",
       "fibrosis/scar non-emphysema            12 (30.0%)  \n",
       "other non-nodules non-emphysema        28 (70.0%)  \n",
       "Total                                 40 (100.0%)  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Same as above for non-emphysema\n",
    "\n",
    "#Detailed comparison of FP categories for emphysema and non-emphysema groups (no volume subgroups)\n",
    "\n",
    "df_categories=pd.DataFrame(columns=['Incorrectly detected by AI','Incorrectly detected by reader'], #below index with the correct order as above\n",
    "                          index=['fibrosis/scar non-emphysema',\n",
    "                                 'other non-nodules non-emphysema'\n",
    "                                ])\n",
    "\n",
    "df_categories.index.name = 'GT by radiologists for discrepancies'#\\initial reading (emph+non-emph)'\n",
    "\n",
    "df_categories['Incorrectly detected by AI']=[fibrosis_FP_noemph,other_nonodules_FP_noemph]\n",
    "\n",
    "df_categories['Incorrectly detected by reader']=[fibrosis_FN_noemph,other_nonodules_FN_noemph]\n",
    "\n",
    "df_categories['All findings']=df_categories['Incorrectly detected by AI']+df_categories['Incorrectly detected by reader']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['Incorrectly detected by AI']/all_findings)*100,1)  \n",
    "df_categories['Incorrectly detected by AI']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' \n",
    "                                             for index,value in enumerate(df_categories['Incorrectly detected by AI'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['Incorrectly detected by reader']/all_findings)*100,1) \n",
    "df_categories['Incorrectly detected by reader']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' \n",
    "                                                 for index,value in enumerate(df_categories['Incorrectly detected by reader'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/all_findings,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "df_categories\n",
    "# ‘Other’ could be bone, tissue, mucus, arthrosis, vessel, consolidation, infection, fat, or atelectasis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "88ac5876",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_categories.to_excel('non_nodules_nonemphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "fff8e683",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Incorrectly detected by AI</th>\n",
       "      <th>Incorrectly detected by reader</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>fibrosis/scar non-emphysema</th>\n",
       "      <td>9 (22.5%)</td>\n",
       "      <td>3 (7.5%)</td>\n",
       "      <td>12 (30.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules lung non-emphysema</th>\n",
       "      <td>6 (15.0%)</td>\n",
       "      <td>4 (10.0%)</td>\n",
       "      <td>10 (25.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules nolung non-emphysema</th>\n",
       "      <td>2 (5.0%)</td>\n",
       "      <td>11 (27.5%)</td>\n",
       "      <td>13 (32.5%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other non-nodules (no description)</th>\n",
       "      <td>1 (2.5%)</td>\n",
       "      <td>4 (10.0%)</td>\n",
       "      <td>5 (12.5%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td>18 (45.0%)</td>\n",
       "      <td>22 (55.0%)</td>\n",
       "      <td>40 (100.0%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Incorrectly detected by AI  \\\n",
       "GT by radiologists for discrepancies                                \n",
       "fibrosis/scar non-emphysema                             9 (22.5%)   \n",
       "other non-nodules lung non-emphysema                    6 (15.0%)   \n",
       "other non-nodules nolung non-emphysema                   2 (5.0%)   \n",
       "other non-nodules (no description)                       1 (2.5%)   \n",
       "Total                                                  18 (45.0%)   \n",
       "\n",
       "                                       Incorrectly detected by reader  \\\n",
       "GT by radiologists for discrepancies                                    \n",
       "fibrosis/scar non-emphysema                                  3 (7.5%)   \n",
       "other non-nodules lung non-emphysema                        4 (10.0%)   \n",
       "other non-nodules nolung non-emphysema                     11 (27.5%)   \n",
       "other non-nodules (no description)                          4 (10.0%)   \n",
       "Total                                                      22 (55.0%)   \n",
       "\n",
       "                                       All findings  \n",
       "GT by radiologists for discrepancies                 \n",
       "fibrosis/scar non-emphysema              12 (30.0%)  \n",
       "other non-nodules lung non-emphysema     10 (25.0%)  \n",
       "other non-nodules nolung non-emphysema   13 (32.5%)  \n",
       "other non-nodules (no description)        5 (12.5%)  \n",
       "Total                                   40 (100.0%)  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Same as above for non-emphysema\n",
    "\n",
    "#Detailed comparison of FP categories for emphysema and non-emphysema groups (no volume subgroups)\n",
    "\n",
    "df_categories=pd.DataFrame(columns=['Incorrectly detected by AI','Incorrectly detected by reader'], #below index with the correct order as above\n",
    "                          index=['fibrosis/scar non-emphysema',\n",
    "                                 'other non-nodules lung non-emphysema','other non-nodules nolung non-emphysema','other non-nodules (no description)'\n",
    "                                ])\n",
    "\n",
    "df_categories.index.name = 'GT by radiologists for discrepancies'#\\initial reading (emph+non-emph)'\n",
    "\n",
    "rest_no_desc_fp=other_nonodules_FP_noemph-other_nonodules_FP_lung_noemph-other_nonodules_FP_nolung_noemph\n",
    "df_categories['Incorrectly detected by AI']=[fibrosis_FP_noemph,other_nonodules_FP_lung_noemph,other_nonodules_FP_nolung_noemph,rest_no_desc_fp]\n",
    "\n",
    "rest_no_desc_fn=other_nonodules_FN_noemph-other_nonodules_FN_lung_noemph-other_nonodules_FN_nolung_noemph\n",
    "df_categories['Incorrectly detected by reader']=[fibrosis_FN_noemph,other_nonodules_FN_lung_noemph,other_nonodules_FN_nolung_noemph,rest_no_desc_fn]\n",
    "\n",
    "df_categories['All findings']=df_categories['Incorrectly detected by AI']+df_categories['Incorrectly detected by reader']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['Incorrectly detected by AI']/all_findings)*100,1)  \n",
    "df_categories['Incorrectly detected by AI']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' \n",
    "                                             for index,value in enumerate(df_categories['Incorrectly detected by AI'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['Incorrectly detected by reader']/all_findings)*100,1) \n",
    "df_categories['Incorrectly detected by reader']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' \n",
    "                                                 for index,value in enumerate(df_categories['Incorrectly detected by reader'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/all_findings,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "916b5edd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('non_nodules_types_nonemphysema.xlsx')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "06ad0ec0",
   "metadata": {},
   "source": [
    "### Metrics Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "a1300ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load number of nodules and lymph nodes for each of the emphysema/non-emphysema groups - These are the TPs\n",
    "\n",
    "#Define nodule group names\n",
    "nod_groups_only=['sub_ground','pleural', 'calcified','other_all', 'atypical_triangular'] \n",
    "lymph_groups=['per_fisu','peri_bronch'] \n",
    "\n",
    "#Initialize number of TP for each of the reader and AI to 0\n",
    "\n",
    "#These can also be the sum of the volume subgroups below - Kept as is for now\n",
    "TP_nod_emph=0\n",
    "TP_nod_noemph=0\n",
    "TP_lymph_emph=0\n",
    "TP_lymph_noemph=0\n",
    "\n",
    "TP_nod_emph_30_100=0\n",
    "TP_nod_noemph_30_100=0\n",
    "TP_nod_emph_100_300=0\n",
    "TP_nod_noemph_100_300=0\n",
    "TP_nod_emph_300=0\n",
    "TP_nod_noemph_300=0\n",
    "\n",
    "TP_lymph_emph_30_100=0\n",
    "TP_lymph_noemph_30_100=0\n",
    "TP_lymph_emph_100_300=0\n",
    "TP_lymph_noemph_100_300=0\n",
    "TP_lymph_emph_300=0\n",
    "TP_lymph_noemph_300=0\n",
    "\n",
    "\n",
    "for emph in ['_emph','_noemph']: #Loop over emphysema/non-emphysema\n",
    "    \n",
    "    for nod_group in nod_groups_only: #Loop over nodule groups\n",
    "        \n",
    "        #Load variables with TP created in 'patient_selection_emphysema_experiment.ipynb' notebook\n",
    "        with open(nod_group+emph+'_nod_only'+'.pickle','rb') as f:\n",
    "            exec(nod_group+emph+'_nod_only= pickle.load(f)')\n",
    "            \n",
    "        #Same for each volume subgroup    \n",
    "        with open(nod_group+emph+'_nod_only_30_100'+'.pickle','rb') as f:\n",
    "            exec(nod_group+emph+'_nod_only_30_100= pickle.load(f)')   \n",
    "        with open(nod_group+emph+'_nod_only_100_300'+'.pickle','rb') as f:\n",
    "            exec(nod_group+emph+'_nod_only_100_300= pickle.load(f)')             \n",
    "        with open(nod_group+emph+'_nod_only_300'+'.pickle','rb') as f:\n",
    "            exec(nod_group+emph+'_nod_only_300= pickle.load(f)')    \n",
    "\n",
    "        if emph=='_emph': #Set variables depending on if we have emphysema or non-emphysema + for each volume subgroup\n",
    "            TP_nod_emph=TP_nod_emph+eval(nod_group+emph+'_nod_only')\n",
    "            \n",
    "            TP_nod_emph_30_100=TP_nod_emph_30_100+eval(nod_group+emph+'_nod_only_30_100')\n",
    "            TP_nod_emph_100_300=TP_nod_emph_100_300+eval(nod_group+emph+'_nod_only_100_300')\n",
    "            TP_nod_emph_300=TP_nod_emph_300+eval(nod_group+emph+'_nod_only_300')\n",
    "            \n",
    "        else:\n",
    "            TP_nod_noemph=TP_nod_noemph+eval(nod_group+emph+'_nod_only')\n",
    "            \n",
    "            TP_nod_noemph_30_100=TP_nod_noemph_30_100+eval(nod_group+emph+'_nod_only_30_100')   \n",
    "            TP_nod_noemph_100_300=TP_nod_noemph_100_300+eval(nod_group+emph+'_nod_only_100_300')\n",
    "            TP_nod_noemph_300=TP_nod_noemph_300+eval(nod_group+emph+'_nod_only_300')\n",
    "\n",
    "        \n",
    "    for lymph_group in lymph_groups: #Similar as above for lymph node groups\n",
    "        \n",
    "        with open(lymph_group+emph+'_lymph'+'.pickle','rb') as f:\n",
    "            exec(lymph_group+emph+'_lymph= pickle.load(f)')\n",
    "            \n",
    "        with open(lymph_group+emph+'_lymph_30_100'+'.pickle','rb') as f:\n",
    "            exec(lymph_group+emph+'_lymph_30_100= pickle.load(f)')   \n",
    "        with open(lymph_group+emph+'_lymph_100_300'+'.pickle','rb') as f:\n",
    "            exec(lymph_group+emph+'_lymph_100_300= pickle.load(f)')             \n",
    "        with open(lymph_group+emph+'_lymph_300'+'.pickle','rb') as f:\n",
    "            exec(lymph_group+emph+'_lymph_300= pickle.load(f)')    \n",
    "            \n",
    "        if emph=='_emph':\n",
    "            TP_lymph_emph=TP_lymph_emph+eval(lymph_group+emph+'_lymph')\n",
    "            \n",
    "            TP_lymph_emph_30_100=TP_lymph_emph_30_100+eval(lymph_group+emph+'_lymph_30_100')\n",
    "            TP_lymph_emph_100_300=TP_lymph_emph_100_300+eval(lymph_group+emph+'_lymph_100_300')\n",
    "            TP_lymph_emph_300=TP_lymph_emph_300+eval(lymph_group+emph+'_lymph_300')\n",
    "            \n",
    "        else:\n",
    "            TP_lymph_noemph=TP_lymph_noemph+eval(lymph_group+emph+'_lymph')\n",
    "            TP_lymph_noemph_30_100=TP_lymph_noemph_30_100+eval(lymph_group+emph+'_lymph_30_100')   \n",
    "            TP_lymph_noemph_100_300=TP_lymph_noemph_100_300+eval(lymph_group+emph+'_lymph_100_300')\n",
    "            TP_lymph_noemph_300=TP_lymph_noemph_300+eval(lymph_group+emph+'_lymph_300')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "5b441f78",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert TP_nod_emph_300==TP_nod_noemph_300==TP_lymph_emph_300==TP_lymph_noemph_300==0"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "38f520cc",
   "metadata": {},
   "source": [
    "#### Below definition of TP depends on reader/AI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "d646edbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get total number of nodules (nodules+lymph nodes) for the whole emphysema/non-emphysema groups and for volume subgroups\n",
    "#300+ volumes kept here since all these values should be 0 - If not, then delete them\n",
    "TP_emph=TP_nod_emph+TP_lymph_emph\n",
    "TP_noemph=TP_nod_noemph+TP_lymph_noemph\n",
    "\n",
    "TP_emph_30_100=TP_nod_emph_30_100+TP_lymph_emph_30_100\n",
    "TP_emph_100_300=TP_nod_emph_100_300+TP_lymph_emph_100_300\n",
    "TP_emph_300=TP_nod_emph_300+TP_lymph_emph_300\n",
    "TP_noemph_30_100=TP_nod_noemph_30_100+TP_lymph_noemph_30_100\n",
    "TP_noemph_100_300=TP_nod_noemph_100_300+TP_lymph_noemph_100_300\n",
    "TP_noemph_300=TP_nod_noemph_300+TP_lymph_noemph_300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "109fa9f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert TP_emph==TP_emph_30_100+TP_emph_100_300\n",
    "assert TP_noemph==TP_noemph_30_100+TP_noemph_100_300"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ccbf9927",
   "metadata": {},
   "source": [
    "#### Confidence Interval Calculations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "d081c609",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Code below taken from https://gist.github.com/maidens/29939b3383a5e57935491303cf0d8e0b\n",
    "#For F1 score there was a suggestion on https://github.com/sousanunes/confidence_intervals/blob/master/propagation_confidence_interval.py\n",
    "#This will not used since it assumes normal distribution\n",
    "\n",
    "def _proportion_confidence_interval(r, n, z): \n",
    "    \"\"\"Compute confidence interval for a proportion.\n",
    "    https://real-statistics.com/binomial-and-related-distributions/proportion-distribution/proportion-parameter-confidence-interval/\n",
    "    Follows notation described on pages 46--47 of [1]. \n",
    "    \n",
    "    References\n",
    "    ----------\n",
    "    [1] R. G. Newcombe and D. G. Altman, Proportions and their differences, in Statisics\n",
    "    with Confidence: Confidence intervals and statisctical guidelines, 2nd Ed., D. G. Altman, \n",
    "    D. Machin, T. N. Bryant and M. J. Gardner (Eds.), pp. 45-57, BMJ Books, 2000. \n",
    "\n",
    "    Based on the book, r is the observed number of subjects with some feature in a sample of size n. z is a percentile from the norm distribution.\n",
    "    The formula in the link of the code is the same as in https://real-statistics.com/binomial-and-related-distributions/proportion-distribution/proportion-parameter-confidence-interval/\n",
    "    There is no continuity correction here. This is used in http://stats.org.uk/statistical-inference/Newcombe1998.pdf\n",
    "    The actual implementation used continuity correction. This is recommended for small sample sizes:  \n",
    "    https://towardsdatascience.com/five-confidence-intervals-for-proportions-that-you-should-know-about-7ff5484c024f\n",
    "    \"\"\"\n",
    "    \n",
    "    A = 2*r + z**2\n",
    "    # B = z*np.sqrt(z**2 + 4*r*(1 - r/n))\n",
    "    B_low=1+z*np.sqrt(z**2 + 4*r*(1 - r/n) + (((4*r)-(2*n)-1)/n))\n",
    "    # if B_low<0:\n",
    "    #     B_low=0\n",
    "    \n",
    "    B_high=1+z*np.sqrt(z**2 + 4*r*(1 - r/n) - (((4*r)-(2*n)+1)/n))\n",
    "    # if B_high>1:\n",
    "    #     B_high=1\n",
    "\n",
    "    C = 2*(n + z**2)\n",
    "    return ((A-B_low)/C, (A+B_high)/C)\n",
    "\n",
    "\n",
    "def sensitivity_and_specificity_with_confidence_intervals(TP, FP, FN, TN, alpha=0.95):\n",
    "    \"\"\"Compute confidence intervals for sensitivity and specificity using Wilson's method. \n",
    "    Based on https://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval this calculation is without continuity correction.\n",
    "    For more information about that check on https://www.statskingdom.com/doc_confidence_interval.html\n",
    "    Based on the first link, of the possible approximations, Wilson score interval methods (with or without continuity correction) \n",
    "    have been shown to be the most accurate and the most robust, though some prefer the Agresti–Coull approach for larger sample sizes\n",
    "    Another link for that is https://statisticaloddsandends.wordpress.com/2019/06/09/wilson-score-and-agresti-coull-intervals-for-binomial-proportions/\n",
    "    \n",
    "    This method does not rely on a normal approximation and results in accurate confidence intervals even for small sample sizes.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    TP : int\n",
    "        Number of true positives\n",
    "    FP : int \n",
    "        Number of false positives\n",
    "    FN : int\n",
    "        Number of false negatives\n",
    "    TN : int\n",
    "        Number of true negatives\n",
    "    alpha : float, optional\n",
    "        Desired confidence. Defaults to 0.95, which yields a 95% confidence interval. \n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    sensitivity_confidence_interval : Tuple (float, float)\n",
    "        Lower and upper bounds on the alpha confidence interval for sensitivity\n",
    "    PPV_confidence_interval : Tuple (float, float)\n",
    "        Lower and upper bounds on the alpha confidence interval for PPV\n",
    "    F1_confidence_interval : Tuple (float, float)\n",
    "        Lower and upper bounds on the alpha confidence interval for F1 score\n",
    "        \n",
    "    References\n",
    "    ----------\n",
    "    [1] R. G. Newcombe and D. G. Altman, Proportions and their differences, in Statisics\n",
    "    with Confidence: Confidence intervals and statisctical guidelines, 2nd Ed., D. G. Altman, \n",
    "    D. Machin, T. N. Bryant and M. J. Gardner (Eds.), pp. 45-57, BMJ Books, 2000. \n",
    "    [2] E. B. Wilson, Probable inference, the law of succession, and statistical inference,\n",
    "    J Am Stat Assoc 22:209-12, 1927. \n",
    "    \"\"\"\n",
    "    \n",
    "    z = -ndtri((1.0-alpha)/2)\n",
    "    \n",
    "    # Compute sensitivity using method described in [1] \n",
    "    sensitivity_confidence_interval = _proportion_confidence_interval(TP, TP + FN, z)\n",
    "\n",
    "     # Compute PPV\n",
    "    PPV_confidence_interval = _proportion_confidence_interval(TP, TP + FP, z)\n",
    "    \n",
    "    #Compute F1score\n",
    "    F1_confidence_interval = _proportion_confidence_interval(2*TP, 2*TP + (FP+FN), z) #if n=TP+FP+FN used we get nan errors - sample size of proportion should be with 2*TP\n",
    "    # Check also based on https://stats.stackexchange.com/questions/363382/confidence-interval-of-precision-recall-and-f1-score\n",
    "    #It is not a binomial outcome (eg. like accuracy which is num of correct over num of predicted) and so, we probably\n",
    "    #can't apply any number of binomial conf intervals as stated in https://stats.stackexchange.com/questions/563582/calculate-confidence-intervals-on-accuracy-metrics\n",
    "    #Also if data not normally distributed we cannot use simple formulas like those in https://aegis4048.github.io/comprehensive_confidence_intervals_for_python_developers\n",
    "\n",
    "    return sensitivity_confidence_interval, PPV_confidence_interval, F1_confidence_interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "2c9f717d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sensitivity(TP,FN): #same as recall\n",
    "    return TP/(TP+FN)\n",
    "\n",
    "def PPV(TP,FP): #Same as precision\n",
    "    return TP/(TP+FP)\n",
    "\n",
    "def F1score(TP,FP,FN):\n",
    "    return (2*TP)/(2*TP+(FP+FN))\n",
    "\n",
    "#Metrics with TN in their definition can't be used"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f15c4453",
   "metadata": {},
   "source": [
    "##### Example of CI calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "5bea469a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.948718, PPV: 0.649123, F1 score: 0.770833\n",
      "alpha = 0.500000 CI for sensitivity: (0.9040941124792334, 0.9775414611018507)\n",
      "alpha = 0.500000 CI for PPV: (0.5965219235920692, 0.6988783015915245)\n",
      "alpha = 0.500000 CI for F1 score: (0.7352529801795313, 0.8034004651312406)\n"
     ]
    }
   ],
   "source": [
    "for a in [0.5]: #Can also set other values of a to check the CI\n",
    "\n",
    "    sensitivity_confidence_interval, PPV_confidence_interval, F1_confidence_interval\\\n",
    "    = sensitivity_and_specificity_with_confidence_intervals(37, 20, 2, 0, alpha=a) #Here TP, FP, FN, TN were set based on an example below - just for demonstration\n",
    "\n",
    "    print(\"Sensitivity: %f, PPV: %f, F1 score: %f\" %(sensitivity(37,2), PPV(37,20),F1score(37,20,2)))\n",
    "    print(\"alpha = %f CI for sensitivity:\"%a, sensitivity_confidence_interval)\n",
    "    print(\"alpha = %f CI for PPV:\"%a, PPV_confidence_interval)\n",
    "    print(\"alpha = %f CI for F1 score:\"%a, F1_confidence_interval)    \n",
    "    #Confidence intervals of proportions were calculated using the Wilson method (without continuity correction). "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b4c03e9b",
   "metadata": {},
   "source": [
    "From the intervals above we can conclude that we won't get the same results if we use normal approximations (z=1.96 and mean between lower and upper bound of CI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "a527dccd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get total number of nodules/non-nodules that were detected only by AI/human reader for each of the emph/non-emph groups\n",
    "\n",
    "#nodules+Lymph nodes included in the right part of the equations below - if only nodules comments below should be activated\n",
    "FP_nods_emph=ai_nods_emph_30_100+ai_nods_emph_100_300+ai_nods_emph_300 #-(ai_lymph_emph_30_100+ai_lymph_emph_100_300+ai_lymph_emph_300)\n",
    "FP_nods_noemph=ai_nods_noemph_30_100+ai_nods_noemph_100_300+ai_nods_noemph_300 #-(ai_lymph_noemph_30_100+ai_lymph_noemph_100_300+ai_lymph_noemph_300)\n",
    "\n",
    "FP_nonods_emph=ai_nonods_emph_30_100+ai_nonods_emph_100_300+ai_nonods_emph_300\n",
    "FP_nonods_noemph=ai_nonods_noemph_30_100+ai_nonods_noemph_100_300+ai_nonods_noemph_300\n",
    "\n",
    "FN_nods_emph=reader_nods_emph_30_100+reader_nods_emph_100_300+reader_nods_emph_300 #-(reader_lymph_emph_30_100+reader_lymph_emph_100_300+reader_lymph_emph_300)\n",
    "FN_nods_noemph=reader_nods_noemph_30_100+reader_nods_noemph_100_300+reader_nods_noemph_300 #-(reader_lymph_noemph_30_100+reader_lymph_noemph_100_300+reader_lymph_noemph_300)\n",
    "\n",
    "FN_nonods_emph=reader_nonods_emph_30_100+reader_nonods_emph_100_300+reader_nonods_emph_300\n",
    "FN_nonods_noemph=reader_nonods_noemph_30_100+reader_nonods_noemph_100_300+reader_nonods_noemph_300\n",
    "\n",
    "#Similar only for lymph nodes\n",
    "lymph_reader_emph=reader_lymph_emph_30_100+reader_lymph_emph_100_300+reader_lymph_emph_300\n",
    "lymph_reader_noemph=reader_lymph_noemph_30_100+reader_lymph_noemph_100_300+reader_lymph_noemph_300\n",
    "lymph_AI_emph=ai_lymph_emph_30_100+ai_lymph_emph_100_300+ai_lymph_emph_300\n",
    "lymph_AI_noemph=ai_lymph_noemph_30_100+ai_lymph_noemph_100_300+ai_lymph_noemph_300"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "39a1442a",
   "metadata": {},
   "source": [
    "Explanation below assumes that GT is whatever is detected only! For TN, this might be incorrect! We assumed that TP (in REDCap) will always be nodules, even though sometimes this might not be correct.\n",
    "\n",
    "To calculate metrics for AI we consider the following (demonstrated for emphysema - same for non-emphysema cases):\n",
    "\n",
    "1. TP_AI=TP_both+FP_nods_emph (nodules found as nodules) \n",
    "2. FP_AI=FP_nonods_emph (non-nodules found as nodules)\n",
    "3. FN_AI=FN_nods_emph (nodules missed by AI)\n",
    "\n",
    "Similarly, for reader metrics:\n",
    "\n",
    "1. TP_read=TP_both+FN_nods_emph\n",
    "2. FP_read=FN_nonods_emph\n",
    "3. FN_read=FP_nods_emph\n",
    "\n",
    "'AI found and reader found' can be seen from TP in REDCap\n",
    "\n",
    "'AI missed and reader missed' does not exist - assumes that consensus found extra nodules while they just reviewed discrepancies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "09f6499c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual number of nodules among discrepancies is 73\n",
      "From those 39 were detected by the AI only and 34 from reader only\n",
      "\n",
      "\n",
      "Actual number of lymph nodes among discrepancies is 45\n",
      "From those 10 were detected by the AI only and 35 from reader only\n"
     ]
    }
   ],
   "source": [
    "AI_found_lymph=lymph_AI_emph+lymph_AI_noemph\n",
    "read_found_lymph=lymph_reader_emph+lymph_reader_noemph\n",
    "\n",
    "reader_found_only=FN_nods_emph+FN_nods_noemph - read_found_lymph\n",
    "AI_found_only=FP_nods_emph+FP_nods_noemph-AI_found_lymph\n",
    "print(\"Actual number of nodules among discrepancies is\",reader_found_only+AI_found_only)\n",
    "print(\"From those {} were detected by the AI only and {} from reader only\".format(AI_found_only,reader_found_only))\n",
    "print(\"\\n\")\n",
    "print(\"Actual number of lymph nodes among discrepancies is\",read_found_lymph+AI_found_lymph)\n",
    "print(\"From those {} were detected by the AI only and {} from reader only\".format(AI_found_lymph,read_found_lymph))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "65934db3",
   "metadata": {},
   "source": [
    "#### Table only for nodules (lymph nodes not included in calculations - considered as non-existent) in emphysema cases. Statistical tests based on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "21fe4f6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emphysema numbers\n",
      "TP_AI 53\n",
      "FP_AI 20\n",
      "FN_AI 13\n",
      "TP_read 47\n",
      "FP_read 6\n",
      "FN_read 19\n",
      "TP_both 34\n"
     ]
    }
   ],
   "source": [
    "TP_AI=TP_emph+FP_nods_emph-(TP_lymph_emph) -lymph_AI_emph #'FP_nods' include lymph and that's why we subtract 'lymph_AI_emph'\n",
    "FP_AI=FP_nonods_emph\n",
    "FN_AI=TP_read_only=FN_nods_emph-lymph_reader_emph #nodules detected only by the reader, excluding lymph nodes\n",
    "\n",
    "TP_read=TP_emph+FN_nods_emph-(TP_lymph_emph) - lymph_reader_emph\n",
    "FP_read=FN_nonods_emph\n",
    "FN_read=TP_AI_only=FP_nods_emph-lymph_AI_emph #nodules detected only by AI, excluding lymph nodes\n",
    "\n",
    "TP_both=TP_emph-(TP_lymph_emph) #Common nodules detected by both AI and reader\n",
    "\n",
    "#Print the above\n",
    "print(\"Emphysema numbers\")\n",
    "print(\"TP_AI\",TP_AI)\n",
    "print(\"FP_AI\",FP_AI)\n",
    "print(\"FN_AI\",FN_AI)\n",
    "print(\"TP_read\",TP_read)\n",
    "print(\"FP_read\",FP_read)\n",
    "print(\"FN_read\",FN_read)\n",
    "print(\"TP_both\",TP_both)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "5087dd87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules detected</th>\n",
       "      <th>non-nodules detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, emphysema</th>\n",
       "      <td>0.8 (0.68, 0.89)</td>\n",
       "      <td>0.73 (0.61, 0.82)</td>\n",
       "      <td>0.76 (0.68, 0.83)</td>\n",
       "      <td>53 (61.6%)</td>\n",
       "      <td>20 (23.3%)</td>\n",
       "      <td>13 (15.1%)</td>\n",
       "      <td>86 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, emphysema</th>\n",
       "      <td>0.71 (0.59, 0.81)</td>\n",
       "      <td>0.89 (0.76, 0.95)</td>\n",
       "      <td>0.79 (0.7, 0.86)</td>\n",
       "      <td>47 (65.3%)</td>\n",
       "      <td>6 (8.3%)</td>\n",
       "      <td>19 (26.4%)</td>\n",
       "      <td>72 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>100</td>\n",
       "      <td>26</td>\n",
       "      <td>32</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  sensitivity (95% CI)       PPV (95% CI)  F1 score (95% CI)  \\\n",
       "AI, emphysema         0.8 (0.68, 0.89)  0.73 (0.61, 0.82)  0.76 (0.68, 0.83)   \n",
       "reader, emphysema    0.71 (0.59, 0.81)  0.89 (0.76, 0.95)   0.79 (0.7, 0.86)   \n",
       "Total                                                                          \n",
       "\n",
       "                  nodules detected non-nodules detected nodules missed  \\\n",
       "AI, emphysema           53 (61.6%)           20 (23.3%)     13 (15.1%)   \n",
       "reader, emphysema       47 (65.3%)             6 (8.3%)     19 (26.4%)   \n",
       "Total                          100                   26             32   \n",
       "\n",
       "                  All findings  \n",
       "AI, emphysema        86 (100%)  \n",
       "reader, emphysema    72 (100%)  \n",
       "Total                      158  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Two tables: one for emphysema and one for non-emphysema, having also percentages.\n",
    "#Assessing detection performance for emph/non-emph groups - For nodules only, we treat lymph nodes as non-existent\n",
    "\n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules detected','non-nodules detected','nodules missed'],\n",
    "                        index=['AI, emphysema', 'reader, emphysema'] )\n",
    "\n",
    "#AI nodules only - emphysema\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI,FP_AI),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI,FP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,3]=TP_AI\n",
    "df_all_new.iloc[0,4]=FP_AI\n",
    "df_all_new.iloc[0,5]=FN_AI\n",
    "\n",
    "#Calculate CIs for sensitivity, PPV, and F1score\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI, FP_AI, FN_AI, 0, alpha=0.95)\n",
    "\n",
    "#Round CIs to 2 digits\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "#Reader nodules only - emphysema\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_read,FN_read),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_read,FP_read),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_read,FP_read,FN_read),2)\n",
    "df_all_new.iloc[1,3]=TP_read\n",
    "df_all_new.iloc[1,4]=FP_read\n",
    "df_all_new.iloc[1,5]=FN_read\n",
    "\n",
    "sensitivity_confidence_interval_read, PPV_confidence_interval_read, F1_confidence_interval_read\\\n",
    "    = sensitivity_and_specificity_with_confidence_intervals(TP_read, FP_read, FN_read, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_read=[np.round(x,2) for x in sensitivity_confidence_interval_read]\n",
    "ci_ppv_read=[np.round(x,2) for x in PPV_confidence_interval_read]\n",
    "ci_f1_read=[np.round(x,2) for x in F1_confidence_interval_read]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_read))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_read))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_read))\n",
    "\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules detected']+df_all_new['non-nodules detected']+df_all_new['nodules missed']\n",
    "df_all_new.loc['Total']= df_all_new.sum()\n",
    "df_all_new.loc['Total'].iloc[0:3]=''\n",
    "\n",
    "all_findings=df_all_new.iloc[:-1,3:-1].sum().sum()\n",
    "\n",
    "for i in range(2): #Add percentages to df\n",
    "    row_all=np.sum(df_all_new.iloc[i][3:6].values) #get all values for a given row\n",
    "\n",
    "    percentage_tp=np.round((df_all_new.iloc[i][3]/row_all)*100,1) #% of TP\n",
    "    df_all_new['nodules detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "    percentage_fp=np.round((df_all_new.iloc[i][4]/row_all)*100,1) #% of FP\n",
    "    df_all_new['non-nodules detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "    percentage_fn=np.round((df_all_new.iloc[i][5]/row_all)*100,1) #% of FN\n",
    "    df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "    df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' (100%)'\n",
    "\n",
    "df_all_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "5cba60c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_only_emphysema.xlsx')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3e854ff3",
   "metadata": {},
   "source": [
    "#### McNemar' test\n",
    "\n",
    "- If we want it with continuity correction we should use 'exact=False, correction=False'. We can compare when it's not applied to see if these values are on different sides of the traditional 0.05 cutoff. If they are, we would have to check the 'exact=True' method to decide which to keep (no corrections at all).  Taken from https://cran.r-project.org/web/packages/exact2x2/vignettes/exactMcNemar.pdf\n",
    "- Continuity corrections no longer used based on https://stats.stackexchange.com/questions/6448/continuity-correction-for-pearson-and-mcnemars-chi-square-test but statistician suggested it due to small sample size\n",
    "- McNemar's test is used when we want to know whether there is a statistically significant difference in the proportion of nodules detected by AI and reader (paraphrased from https://www.geeksforgeeks.org/how-to-perform-mcnemars-test-in-python/).\n",
    "- Other useful sources: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8182550/ (paper used it for similar topic), https://stats.stackexchange.com/questions/358101/statistical-significance-p-value-for-comparing-two-classifiers-with-respect-to"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "fc30f81f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "McNemar's test (nodules only), AI_vs_Reader with continuity correction (not exact) p value is 0.3767591178115821\n",
      "For FP findings, with continuity correction (not exact) p value is 0.010787449254670376\n"
     ]
    }
   ],
   "source": [
    "#McNemar's test to compare Reader vs AI (using consensus panel)\n",
    "#Below format is: [[Both AI found and reader found, reader missed and AI found], [Reader found and AI missed, 0]]\n",
    "\n",
    "#For nodules\n",
    "data=[[TP_both, FN_read],\n",
    "        [FN_AI,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"McNemar's test (nodules only), AI_vs_Reader with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue) \n",
    "\n",
    "\n",
    "#For FPs\n",
    "data=[[0, FP_AI], \n",
    "        [FP_read, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For FP findings, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "74b471da",
   "metadata": {},
   "source": [
    "#### Cohen's Kappa\n",
    "\n",
    "- According to https://en.wikipedia.org/wiki/Fleiss%27_kappa, we must use Fleiss kappa when assessing the agreement between three or more raters or the intra-rater reliability (for one appraiser versus themself). To calculate this, the fleiss_kappa() function from the statsmodels library can be used. Cohen's kappa can be used for two readers and this is what we use below (https://www.statology.org/cohens-kappa-python/).We should only have 0 or 1 labels since otherwise it is considered as a multiclass problem\n",
    "- Other useful sources https://www.statology.org/cohens-kappa-statistic/, https://vitalflux.com/cohen-kappa-score-python-example-machine-learning/\n",
    "- Based on the last one, in the contigency table we have reader 1 (actual results) horizontally and reader 2 (predictions) vertically. For this to be true, reasonable to assume reader 1 is GT by radiologists and reader 2 either reader or AI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "ea2422ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #AI vs GT, TP_both included\n",
    "# # Table looks like below:\n",
    "# #              GT\n",
    "# #             Yes                       No\n",
    "# # AI   Yes    TP_both+TP_AI_only      FP_AI\n",
    "# #      No     FN_AI                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_AI_only)],[0 for x in range(FP_AI) ],[1 for x in range(FN_AI) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)], [1 for x in range(TP_AI_only)],[1 for x in range(FP_AI) ],[0 for x in range(FN_AI) ]]\n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"AI vs consensus (for nodules only), kappa is \",cohen_kappa_score(rater_GT, rater_AI))\n",
    "\n",
    "# #Reader vs GT, TP_both included\n",
    "# # Table looks like below:\n",
    "# #                    GT\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes    TP_both+TP_read_only      FP_read\n",
    "# #           No     FN_read                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[0 for x in range(FP_read) ],[1 for x in range(FN_read) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[1 for x in range(FP_read) ],[0 for x in range(FN_read) ]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs consensus (for non-nodules only), kappa is \",cohen_kappa_score(rater_GT, rater_read))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "bdda5563",
   "metadata": {},
   "source": [
    "#### Nodule only table with metrics for non-emphysema - Statistical tests based on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "5d9bf97c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-emphysema numbers\n",
      "TP_AI 77\n",
      "FP_AI 18\n",
      "FN_AI 21\n",
      "TP_read 78\n",
      "FP_read 22\n",
      "FN_read 20\n",
      "TP_both 57\n"
     ]
    }
   ],
   "source": [
    "TP_AI= TP_noemph+FP_nods_noemph -(TP_lymph_noemph) -lymph_AI_noemph #'FP_nods' include lymph and that's why we subtract 'lymph_AI_noemph'\n",
    "FP_AI=FP_nonods_noemph\n",
    "FN_AI=TP_read_only=FN_nods_noemph-lymph_reader_noemph #nodules detected only by the reader, excluding lymph nodes\n",
    "\n",
    "TP_read=TP_noemph+FN_nods_noemph -(TP_lymph_noemph) - lymph_reader_noemph\n",
    "FP_read=FN_nonods_noemph\n",
    "FN_read=TP_AI_only=FP_nods_noemph-lymph_AI_noemph #nodules detected only by AI, excluding lymph nodes\n",
    "\n",
    "TP_both=TP_noemph-(TP_lymph_noemph) #Common nodules detected by both AI and reader\n",
    "\n",
    "#Print the above\n",
    "print(\"Non-emphysema numbers\")\n",
    "print(\"TP_AI\",TP_AI)\n",
    "print(\"FP_AI\",FP_AI)\n",
    "print(\"FN_AI\",FN_AI)\n",
    "print(\"TP_read\",TP_read)\n",
    "print(\"FP_read\",FP_read)\n",
    "print(\"FN_read\",FN_read)\n",
    "print(\"TP_both\",TP_both)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "0d732795",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules detected</th>\n",
       "      <th>non-nodules detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, nonemphysema</th>\n",
       "      <td>0.79 (0.69, 0.86)</td>\n",
       "      <td>0.81 (0.71, 0.88)</td>\n",
       "      <td>0.8 (0.73, 0.85)</td>\n",
       "      <td>77 (66.4%)</td>\n",
       "      <td>18 (15.5%)</td>\n",
       "      <td>21 (18.1%)</td>\n",
       "      <td>116 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, nonemphysema</th>\n",
       "      <td>0.8 (0.7, 0.87)</td>\n",
       "      <td>0.78 (0.68, 0.85)</td>\n",
       "      <td>0.79 (0.72, 0.84)</td>\n",
       "      <td>78 (65.0%)</td>\n",
       "      <td>22 (18.3%)</td>\n",
       "      <td>20 (16.7%)</td>\n",
       "      <td>120 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>155</td>\n",
       "      <td>40</td>\n",
       "      <td>41</td>\n",
       "      <td>236</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     sensitivity (95% CI)       PPV (95% CI)  \\\n",
       "AI, nonemphysema        0.79 (0.69, 0.86)  0.81 (0.71, 0.88)   \n",
       "reader, nonemphysema      0.8 (0.7, 0.87)  0.78 (0.68, 0.85)   \n",
       "Total                                                          \n",
       "\n",
       "                      F1 score (95% CI) nodules detected non-nodules detected  \\\n",
       "AI, nonemphysema       0.8 (0.73, 0.85)       77 (66.4%)           18 (15.5%)   \n",
       "reader, nonemphysema  0.79 (0.72, 0.84)       78 (65.0%)           22 (18.3%)   \n",
       "Total                                                155                   40   \n",
       "\n",
       "                     nodules missed All findings  \n",
       "AI, nonemphysema         21 (18.1%)   116 (100%)  \n",
       "reader, nonemphysema     20 (16.7%)   120 (100%)  \n",
       "Total                            41          236  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Second part of table split for non-emphysema\n",
    "    \n",
    "#Assessing detection performance - For nodules only, we treat lymph nodes as non-existent - for emph/non-emph groups\n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules detected','non-nodules detected','nodules missed'],\n",
    "                        index=['AI, nonemphysema', 'reader, nonemphysema'])\n",
    "\n",
    "#AI nodules only - no emphysema\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI,FP_AI),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI,FP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,3]=TP_AI\n",
    "df_all_new.iloc[0,4]=FP_AI\n",
    "df_all_new.iloc[0,5]=FN_AI\n",
    "\n",
    "#Calculate CIs for sensitivity, PPV, and F1score\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI, FP_AI, FN_AI, 0, alpha=0.95)\n",
    "\n",
    "#Round CIs to 2 digits\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "#Reader nodules only - no emphysema\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_read,FN_read),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_read,FP_read),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_read,FP_read,FN_read),2)\n",
    "df_all_new.iloc[1,3]=TP_read\n",
    "df_all_new.iloc[1,4]=FP_read\n",
    "df_all_new.iloc[1,5]=FN_read\n",
    "\n",
    "sensitivity_confidence_interval_read, PPV_confidence_interval_read, F1_confidence_interval_read\\\n",
    "    = sensitivity_and_specificity_with_confidence_intervals(TP_read, FP_read, FN_read, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_read=[np.round(x,2) for x in sensitivity_confidence_interval_read]\n",
    "ci_ppv_read=[np.round(x,2) for x in PPV_confidence_interval_read]\n",
    "ci_f1_read=[np.round(x,2) for x in F1_confidence_interval_read]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_read))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_read))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_read))\n",
    "\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules detected']+df_all_new['non-nodules detected']+df_all_new['nodules missed']\n",
    "df_all_new.loc['Total']= df_all_new.sum()\n",
    "df_all_new.loc['Total'].iloc[0:3]=''\n",
    "\n",
    "all_findings=df_all_new.iloc[:-1,3:-1].sum().sum()\n",
    "\n",
    "for i in range(2): #Add percentages to df\n",
    "    row_all=np.sum(df_all_new.iloc[i][3:6].values)\n",
    "\n",
    "    percentage_fp=np.round((df_all_new.iloc[i][4]/row_all)*100,1) \n",
    "    df_all_new['non-nodules detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "    percentage_tp=np.round((df_all_new.iloc[i][3]/row_all)*100,1) \n",
    "    df_all_new['nodules detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "    percentage_fn=np.round((df_all_new.iloc[i][5]/row_all)*100,1) \n",
    "    df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "    df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' (100%)'\n",
    "\n",
    "df_all_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "e7314d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_only_nonemphysema.xlsx')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0de21988",
   "metadata": {},
   "source": [
    "##### McNemar's test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "e55b28ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For nodules only (AI vs reader) with continuity correction (not exact) p value is 1.0\n",
      "For FP findings, with continuity correction (not exact) p value is 0.6352562959972483\n"
     ]
    }
   ],
   "source": [
    "#For nodules\n",
    "data=[[TP_both, FN_read],\n",
    "        [FN_AI,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules only (AI vs reader) with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "\n",
    "\n",
    "#For FPs\n",
    "data=[[0, FP_AI], \n",
    "        [FP_read, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For FP findings, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "70f091c4",
   "metadata": {},
   "source": [
    "##### Cohen's Kappa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "1a43b8fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #AI vs GT, TP_both included - \n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_AI_only)],[0 for x in range(FP_AI) ],[1 for x in range(FN_AI) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)], [1 for x in range(TP_AI_only)],[1 for x in range(FP_AI) ],[0 for x in range(FN_AI) ]]\n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"AI vs consensus (for nodules only), kappa is \",cohen_kappa_score(rater_GT, rater_AI))\n",
    "\n",
    "# #Reader vs GT, TP_both included\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[0 for x in range(FP_read) ],[1 for x in range(FN_read) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[1 for x in range(FP_read) ],[0 for x in range(FN_read) ]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"read vs consensus (for non-nodules only), kappa is \",cohen_kappa_score(rater_GT, rater_read))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9b661eaa",
   "metadata": {},
   "source": [
    "#### Comparison of volume subgroups for emphysema - Statistical tests based on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "44bbeb26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emphysema numbers\n",
      "TP_AI_100 30\n",
      "FP_AI_100 8\n",
      "FN_AI_100 10\n",
      "TP_read_100 29\n",
      "FP_read_100 5\n",
      "FN_read_100 11\n",
      "TP_AI_100_300 23\n",
      "FP_AI_100_300 12\n",
      "FN_AI_100_300 3\n",
      "TP_read_100_300 18\n",
      "FP_read_100_300 1\n",
      "FN_read_100_300 8\n",
      "TP_both_100 19\n",
      "TP_both_100_300 15\n"
     ]
    }
   ],
   "source": [
    "TP_AI_100=TP_emph_30_100+ai_nods_emph_30_100-(TP_lymph_emph_30_100) -ai_lymph_emph_30_100  \n",
    "FP_AI_100=ai_nonods_emph_30_100\n",
    "FN_AI_100=reader_nods_emph_30_100-reader_lymph_emph_30_100 #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100=TP_emph_30_100+reader_nods_emph_30_100-(TP_lymph_emph_30_100) - reader_lymph_emph_30_100\n",
    "FP_read_100=reader_nonods_emph_30_100\n",
    "FN_read_100=ai_nods_emph_30_100-ai_lymph_emph_30_100 #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_AI_100_300=TP_emph_100_300+ai_nods_emph_100_300-(TP_lymph_emph_100_300) -(ai_lymph_emph_100_300)\n",
    "FP_AI_100_300=ai_nonods_emph_100_300\n",
    "FN_AI_100_300=reader_nods_emph_100_300-(reader_lymph_emph_100_300) #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100_300=TP_emph_100_300+reader_nods_emph_100_300-(TP_lymph_emph_100_300) - (reader_lymph_emph_100_300)\n",
    "FP_read_100_300=reader_nonods_emph_100_300\n",
    "FN_read_100_300=ai_nods_emph_100_300-(ai_lymph_emph_100_300) #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_both_100=TP_emph_30_100-(TP_lymph_emph_30_100) \n",
    "TP_both_100_300=TP_emph_100_300-(TP_lymph_emph_100_300)\n",
    "\n",
    "#Print the above\n",
    "print(\"Emphysema numbers\")\n",
    "print(\"TP_AI_100\",TP_AI_100)\n",
    "print(\"FP_AI_100\",FP_AI_100)\n",
    "print(\"FN_AI_100\",FN_AI_100)\n",
    "print(\"TP_read_100\",TP_read_100)\n",
    "print(\"FP_read_100\",FP_read_100)\n",
    "print(\"FN_read_100\",FN_read_100)\n",
    "print(\"TP_AI_100_300\",TP_AI_100_300)\n",
    "print(\"FP_AI_100_300\",FP_AI_100_300)\n",
    "print(\"FN_AI_100_300\",FN_AI_100_300)\n",
    "print(\"TP_read_100_300\",TP_read_100_300)\n",
    "print(\"FP_read_100_300\",FP_read_100_300)\n",
    "print(\"FN_read_100_300\",FN_read_100_300)\n",
    "print(\"TP_both_100\",TP_both_100)\n",
    "print(\"TP_both_100_300\",TP_both_100_300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "c8cd19ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules correctly detected</th>\n",
       "      <th>non-nodules incorrectly detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, emphysema 30-100mm3</th>\n",
       "      <td>0.75 (0.58, 0.87)</td>\n",
       "      <td>0.79 (0.62, 0.9)</td>\n",
       "      <td>0.77 (0.66, 0.85)</td>\n",
       "      <td>30 (34.9%)</td>\n",
       "      <td>8 (9.3%)</td>\n",
       "      <td>10 (11.6%)</td>\n",
       "      <td>48 (55.8%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AI, emphysema 100-300mm3</th>\n",
       "      <td>0.88 (0.69, 0.97)</td>\n",
       "      <td>0.66 (0.48, 0.8)</td>\n",
       "      <td>0.75 (0.62, 0.85)</td>\n",
       "      <td>23 (26.7%)</td>\n",
       "      <td>12 (14.0%)</td>\n",
       "      <td>3 (3.5%)</td>\n",
       "      <td>38 (44.2%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>86 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, emphysema 30-100mm3</th>\n",
       "      <td>0.72 (0.56, 0.85)</td>\n",
       "      <td>0.85 (0.68, 0.94)</td>\n",
       "      <td>0.78 (0.67, 0.87)</td>\n",
       "      <td>29 (40.3%)</td>\n",
       "      <td>5 (6.9%)</td>\n",
       "      <td>11 (15.3%)</td>\n",
       "      <td>45 (62.5%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, emphysema 100-300mm3</th>\n",
       "      <td>0.69 (0.48, 0.85)</td>\n",
       "      <td>0.95 (0.72, 1.0)</td>\n",
       "      <td>0.8 (0.65, 0.9)</td>\n",
       "      <td>18 (25.0%)</td>\n",
       "      <td>1 (1.4%)</td>\n",
       "      <td>8 (11.1%)</td>\n",
       "      <td>27 (37.5%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>72 (100%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     sensitivity (95% CI)       PPV (95% CI)  \\\n",
       "GT by radiologists for discrepancies                                           \n",
       "AI, emphysema 30-100mm3                 0.75 (0.58, 0.87)   0.79 (0.62, 0.9)   \n",
       "AI, emphysema 100-300mm3                0.88 (0.69, 0.97)   0.66 (0.48, 0.8)   \n",
       "                                                                               \n",
       "reader, emphysema 30-100mm3             0.72 (0.56, 0.85)  0.85 (0.68, 0.94)   \n",
       "reader, emphysema 100-300mm3            0.69 (0.48, 0.85)   0.95 (0.72, 1.0)   \n",
       "                                                                               \n",
       "\n",
       "                                      F1 score (95% CI)  \\\n",
       "GT by radiologists for discrepancies                      \n",
       "AI, emphysema 30-100mm3               0.77 (0.66, 0.85)   \n",
       "AI, emphysema 100-300mm3              0.75 (0.62, 0.85)   \n",
       "                                                          \n",
       "reader, emphysema 30-100mm3           0.78 (0.67, 0.87)   \n",
       "reader, emphysema 100-300mm3            0.8 (0.65, 0.9)   \n",
       "                                                          \n",
       "\n",
       "                                     nodules correctly detected  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "AI, emphysema 30-100mm3                              30 (34.9%)   \n",
       "AI, emphysema 100-300mm3                             23 (26.7%)   \n",
       "                                                                  \n",
       "reader, emphysema 30-100mm3                          29 (40.3%)   \n",
       "reader, emphysema 100-300mm3                         18 (25.0%)   \n",
       "                                                                  \n",
       "\n",
       "                                     non-nodules incorrectly detected  \\\n",
       "GT by radiologists for discrepancies                                    \n",
       "AI, emphysema 30-100mm3                                      8 (9.3%)   \n",
       "AI, emphysema 100-300mm3                                   12 (14.0%)   \n",
       "                                                                        \n",
       "reader, emphysema 30-100mm3                                  5 (6.9%)   \n",
       "reader, emphysema 100-300mm3                                 1 (1.4%)   \n",
       "                                                                        \n",
       "\n",
       "                                     nodules missed All findings  \n",
       "GT by radiologists for discrepancies                              \n",
       "AI, emphysema 30-100mm3                  10 (11.6%)   48 (55.8%)  \n",
       "AI, emphysema 100-300mm3                   3 (3.5%)   38 (44.2%)  \n",
       "                                                       86 (100%)  \n",
       "reader, emphysema 30-100mm3              11 (15.3%)   45 (62.5%)  \n",
       "reader, emphysema 100-300mm3              8 (11.1%)   27 (37.5%)  \n",
       "                                                       72 (100%)  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#For emphysema only comparison between reader and AI for volume subgroups\n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules correctly detected','non-nodules incorrectly detected','nodules missed'], \n",
    "                        index=['AI, emphysema 30-100mm3', 'AI, emphysema 100-300mm3','',\n",
    "                               'reader, emphysema 30-100mm3', 'reader, emphysema 100-300mm3',''\n",
    "                              ])\n",
    "\n",
    "df_all_new.index.name = 'GT by radiologists for discrepancies' \n",
    "\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI_100,FP_AI_100),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI_100,FP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,3]=TP_AI_100\n",
    "df_all_new.iloc[0,4]=FP_AI_100\n",
    "df_all_new.iloc[0,5]=FN_AI_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100, FP_AI_100, FN_AI_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "df_all_new.iloc[3,0]=np.round(sensitivity(TP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,1]=np.round(PPV(TP_read_100,FP_read_100),2)\n",
    "df_all_new.iloc[3,2]=np.round(F1score(TP_read_100,FP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,3]=TP_read_100\n",
    "df_all_new.iloc[3,4]=FP_read_100\n",
    "df_all_new.iloc[3,5]=FN_read_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100, FP_read_100, FN_read_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[3]=str(df_all_new['sensitivity (95% CI)'].iloc[3])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[3]=str(df_all_new['PPV (95% CI)'].iloc[3])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[3]=str(df_all_new['F1 score (95% CI)'].iloc[3])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_AI_100_300,FP_AI_100_300),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_AI_100_300,FP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,3]=TP_AI_100_300\n",
    "df_all_new.iloc[1,4]=FP_AI_100_300\n",
    "df_all_new.iloc[1,5]=FN_AI_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100_300, FP_AI_100_300, FN_AI_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "df_all_new.iloc[4,0]=np.round(sensitivity(TP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,1]=np.round(PPV(TP_read_100_300,FP_read_100_300),2)\n",
    "df_all_new.iloc[4,2]=np.round(F1score(TP_read_100_300,FP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,3]=TP_read_100_300\n",
    "df_all_new.iloc[4,4]=FP_read_100_300\n",
    "df_all_new.iloc[4,5]=FN_read_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100_300, FP_read_100_300, FN_read_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[4]=str(df_all_new['sensitivity (95% CI)'].iloc[4])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[4]=str(df_all_new['PPV (95% CI)'].iloc[4])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[4]=str(df_all_new['F1 score (95% CI)'].iloc[4])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "df_all_new.iloc[2,0]=0\n",
    "df_all_new.iloc[2,1]=0\n",
    "df_all_new.iloc[2,2]=0\n",
    "df_all_new.iloc[2,3]=0\n",
    "df_all_new.iloc[2,4]=0\n",
    "df_all_new.iloc[2,5]=0\n",
    "\n",
    "AI_all=np.sum(df_all_new.iloc[0:2,3:].values)\n",
    "reader_all=np.sum(df_all_new.iloc[3:5,3:].values)\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules correctly detected']+df_all_new['non-nodules incorrectly detected']+df_all_new['nodules missed']\n",
    "\n",
    "df_all_new.iloc[5,0]=''\n",
    "df_all_new.iloc[5,1]=''\n",
    "df_all_new.iloc[5,2]=''\n",
    "df_all_new.iloc[5,3]=''\n",
    "df_all_new.iloc[5,4]=''\n",
    "df_all_new.iloc[5,5]=''\n",
    "df_all_new.iloc[5,6]=np.sum(df_all_new['All findings'].iloc[3:5])\n",
    "\n",
    "df_all_new.iloc[2,0]=''\n",
    "df_all_new.iloc[2,1]=''\n",
    "df_all_new.iloc[2,2]=''\n",
    "df_all_new.iloc[2,3]=''\n",
    "df_all_new.iloc[2,4]=''\n",
    "df_all_new.iloc[2,5]=''\n",
    "df_all_new.iloc[2,6]=np.sum(df_all_new['All findings'].iloc[0:2])\n",
    "\n",
    "# print(df_all_new['sensitivity (95% CI)'])\n",
    "for i in range(5):\n",
    "    if i!=2:\n",
    "        if i<2:\n",
    "            sum_all=AI_all\n",
    "        elif i>2:\n",
    "            sum_all=reader_all\n",
    "            \n",
    "        percentage_tp=np.round((df_all_new.iloc[i][3]/sum_all)*100,1) \n",
    "        df_all_new['nodules correctly detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "\n",
    "        percentage_fp=np.round((df_all_new.iloc[i][4]/sum_all)*100,1) \n",
    "        df_all_new['non-nodules incorrectly detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "\n",
    "        percentage_fn=np.round((df_all_new.iloc[i][5]/sum_all)*100,1) \n",
    "        df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "        df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' ('+str(np.round(100*df_all_new.iloc[i][6]/sum_all,1))+'%)'\n",
    "\n",
    "    \n",
    "df_all_new['All findings'].iloc[2]=str(df_all_new.iloc[2][6])+' (100%)'\n",
    "df_all_new['All findings'].iloc[5]=str(df_all_new.iloc[5][6])+' (100%)'\n",
    "\n",
    "df_all_new #Detection performance comparison for nodules and lymph nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "c050ebc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_only_volumes_emphysema_all.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "17bcefd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For nodules only (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is 1.0\n",
      "For FP findings of 30-100mm3, with continuity correction (not exact) p value is 0.5790997419539188\n",
      "\n",
      "\n",
      "For nodules only (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is 0.22779999398822554\n",
      "For FP findings of 100-300mm3, with continuity correction (not exact) p value is 0.0055456673152440615\n"
     ]
    }
   ],
   "source": [
    "data=[[TP_both_100, FN_read_100],\n",
    "        [FN_AI_100,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules only (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "#For FPs\n",
    "data=[[0, FP_AI_100], \n",
    "        [FP_read_100, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For FP findings of 30-100mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "data=[[TP_both_100_300,FN_read_100_300], \n",
    "        [FN_AI_100_300, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules only (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "#For FPs\n",
    "data=[[0, FP_AI_100_300], \n",
    "        [FP_read_100_300, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For FP findings of 100-300mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4cf74b14",
   "metadata": {},
   "source": [
    "#### Comparison of volume subgroups for no emphysema - Statistical tests based on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "9520fd44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-emphysema numbers\n",
      "TP_AI_100 58\n",
      "FP_AI_100 5\n",
      "FN_AI_100 19\n",
      "TP_read_100 62\n",
      "FP_read_100 20\n",
      "FN_read_100 15\n",
      "TP_AI_100_300 19\n",
      "FP_AI_100_300 13\n",
      "FN_AI_100_300 2\n",
      "TP_read_100_300 16\n",
      "FP_read_100_300 2\n",
      "FN_read_100_300 5\n",
      "TP_both_100 43\n",
      "TP_both_100_300 14\n"
     ]
    }
   ],
   "source": [
    "TP_AI_100=TP_noemph_30_100+ai_nods_noemph_30_100-(TP_lymph_noemph_30_100) -ai_lymph_noemph_30_100\n",
    "FP_AI_100=ai_nonods_noemph_30_100\n",
    "FN_AI_100=reader_nods_noemph_30_100-reader_lymph_noemph_30_100 #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100=TP_noemph_30_100+reader_nods_noemph_30_100-(TP_lymph_noemph_30_100) - reader_lymph_noemph_30_100\n",
    "FP_read_100=reader_nonods_noemph_30_100\n",
    "FN_read_100=ai_nods_noemph_30_100-ai_lymph_noemph_30_100 #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_AI_100_300=TP_noemph_100_300+ai_nods_noemph_100_300-(TP_lymph_noemph_100_300) -(ai_lymph_noemph_100_300)\n",
    "FP_AI_100_300=ai_nonods_noemph_100_300\n",
    "FN_AI_100_300=reader_nods_noemph_100_300-(reader_lymph_noemph_100_300) #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100_300=TP_noemph_100_300+reader_nods_noemph_100_300-(TP_lymph_noemph_100_300) - (reader_lymph_noemph_100_300)\n",
    "FP_read_100_300=reader_nonods_noemph_100_300\n",
    "FN_read_100_300=ai_nods_noemph_100_300-(ai_lymph_noemph_100_300) #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_both_100=TP_noemph_30_100-(TP_lymph_noemph_30_100)\n",
    "TP_both_100_300=TP_noemph_100_300-(TP_lymph_noemph_100_300)\n",
    "\n",
    "#Print the above\n",
    "print(\"Non-emphysema numbers\")\n",
    "print(\"TP_AI_100\",TP_AI_100)\n",
    "print(\"FP_AI_100\",FP_AI_100)\n",
    "print(\"FN_AI_100\",FN_AI_100)\n",
    "print(\"TP_read_100\",TP_read_100)\n",
    "print(\"FP_read_100\",FP_read_100)\n",
    "print(\"FN_read_100\",FN_read_100)\n",
    "print(\"TP_AI_100_300\",TP_AI_100_300)\n",
    "print(\"FP_AI_100_300\",FP_AI_100_300)\n",
    "print(\"FN_AI_100_300\",FN_AI_100_300)\n",
    "print(\"TP_read_100_300\",TP_read_100_300)\n",
    "print(\"FP_read_100_300\",FP_read_100_300)\n",
    "print(\"FN_read_100_300\",FN_read_100_300)\n",
    "print(\"TP_both_100\",TP_both_100)\n",
    "print(\"TP_both_100_300\",TP_both_100_300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "bc15101f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules correctly detected</th>\n",
       "      <th>non-nodules incorrectly detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, nonemphysema 30-100mm3</th>\n",
       "      <td>0.75 (0.64, 0.84)</td>\n",
       "      <td>0.92 (0.82, 0.97)</td>\n",
       "      <td>0.83 (0.75, 0.88)</td>\n",
       "      <td>58 (50.0%)</td>\n",
       "      <td>5 (4.3%)</td>\n",
       "      <td>19 (16.4%)</td>\n",
       "      <td>82 (70.7%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AI, nonemphysema 100-300mm3</th>\n",
       "      <td>0.9 (0.68, 0.98)</td>\n",
       "      <td>0.59 (0.41, 0.76)</td>\n",
       "      <td>0.72 (0.57, 0.83)</td>\n",
       "      <td>19 (16.4%)</td>\n",
       "      <td>13 (11.2%)</td>\n",
       "      <td>2 (1.7%)</td>\n",
       "      <td>34 (29.3%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>116 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, nonemphysema 30-100mm3</th>\n",
       "      <td>0.81 (0.7, 0.88)</td>\n",
       "      <td>0.76 (0.65, 0.84)</td>\n",
       "      <td>0.78 (0.71, 0.84)</td>\n",
       "      <td>62 (51.7%)</td>\n",
       "      <td>20 (16.7%)</td>\n",
       "      <td>15 (12.5%)</td>\n",
       "      <td>97 (80.8%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, nonemphysema 100-300mm3</th>\n",
       "      <td>0.76 (0.52, 0.91)</td>\n",
       "      <td>0.89 (0.64, 0.98)</td>\n",
       "      <td>0.82 (0.66, 0.92)</td>\n",
       "      <td>16 (13.3%)</td>\n",
       "      <td>2 (1.7%)</td>\n",
       "      <td>5 (4.2%)</td>\n",
       "      <td>23 (19.2%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>120 (100%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     sensitivity (95% CI)       PPV (95% CI)  \\\n",
       "GT by radiologists for discrepancies                                           \n",
       "AI, nonemphysema 30-100mm3              0.75 (0.64, 0.84)  0.92 (0.82, 0.97)   \n",
       "AI, nonemphysema 100-300mm3              0.9 (0.68, 0.98)  0.59 (0.41, 0.76)   \n",
       "                                                                               \n",
       "reader, nonemphysema 30-100mm3           0.81 (0.7, 0.88)  0.76 (0.65, 0.84)   \n",
       "reader, nonemphysema 100-300mm3         0.76 (0.52, 0.91)  0.89 (0.64, 0.98)   \n",
       "                                                                               \n",
       "\n",
       "                                      F1 score (95% CI)  \\\n",
       "GT by radiologists for discrepancies                      \n",
       "AI, nonemphysema 30-100mm3            0.83 (0.75, 0.88)   \n",
       "AI, nonemphysema 100-300mm3           0.72 (0.57, 0.83)   \n",
       "                                                          \n",
       "reader, nonemphysema 30-100mm3        0.78 (0.71, 0.84)   \n",
       "reader, nonemphysema 100-300mm3       0.82 (0.66, 0.92)   \n",
       "                                                          \n",
       "\n",
       "                                     nodules correctly detected  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "AI, nonemphysema 30-100mm3                           58 (50.0%)   \n",
       "AI, nonemphysema 100-300mm3                          19 (16.4%)   \n",
       "                                                                  \n",
       "reader, nonemphysema 30-100mm3                       62 (51.7%)   \n",
       "reader, nonemphysema 100-300mm3                      16 (13.3%)   \n",
       "                                                                  \n",
       "\n",
       "                                     non-nodules incorrectly detected  \\\n",
       "GT by radiologists for discrepancies                                    \n",
       "AI, nonemphysema 30-100mm3                                   5 (4.3%)   \n",
       "AI, nonemphysema 100-300mm3                                13 (11.2%)   \n",
       "                                                                        \n",
       "reader, nonemphysema 30-100mm3                             20 (16.7%)   \n",
       "reader, nonemphysema 100-300mm3                              2 (1.7%)   \n",
       "                                                                        \n",
       "\n",
       "                                     nodules missed All findings  \n",
       "GT by radiologists for discrepancies                              \n",
       "AI, nonemphysema 30-100mm3               19 (16.4%)   82 (70.7%)  \n",
       "AI, nonemphysema 100-300mm3                2 (1.7%)   34 (29.3%)  \n",
       "                                                      116 (100%)  \n",
       "reader, nonemphysema 30-100mm3           15 (12.5%)   97 (80.8%)  \n",
       "reader, nonemphysema 100-300mm3            5 (4.2%)   23 (19.2%)  \n",
       "                                                      120 (100%)  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#For emphysema only comparison between reader and AI for volume subgroups\n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules correctly detected','non-nodules incorrectly detected','nodules missed'], \n",
    "                        index=['AI, nonemphysema 30-100mm3', 'AI, nonemphysema 100-300mm3','',\n",
    "                               'reader, nonemphysema 30-100mm3', 'reader, nonemphysema 100-300mm3',''\n",
    "                              ])\n",
    "\n",
    "df_all_new.index.name = 'GT by radiologists for discrepancies' \n",
    "\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI_100,FP_AI_100),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI_100,FP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,3]=TP_AI_100\n",
    "df_all_new.iloc[0,4]=FP_AI_100\n",
    "df_all_new.iloc[0,5]=FN_AI_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100, FP_AI_100, FN_AI_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "df_all_new.iloc[3,0]=np.round(sensitivity(TP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,1]=np.round(PPV(TP_read_100,FP_read_100),2)\n",
    "df_all_new.iloc[3,2]=np.round(F1score(TP_read_100,FP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,3]=TP_read_100\n",
    "df_all_new.iloc[3,4]=FP_read_100\n",
    "df_all_new.iloc[3,5]=FN_read_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100, FP_read_100, FN_read_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[3]=str(df_all_new['sensitivity (95% CI)'].iloc[3])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[3]=str(df_all_new['PPV (95% CI)'].iloc[3])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[3]=str(df_all_new['F1 score (95% CI)'].iloc[3])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_AI_100_300,FP_AI_100_300),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_AI_100_300,FP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,3]=TP_AI_100_300\n",
    "df_all_new.iloc[1,4]=FP_AI_100_300\n",
    "df_all_new.iloc[1,5]=FN_AI_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100_300, FP_AI_100_300, FN_AI_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "df_all_new.iloc[4,0]=np.round(sensitivity(TP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,1]=np.round(PPV(TP_read_100_300,FP_read_100_300),2)\n",
    "df_all_new.iloc[4,2]=np.round(F1score(TP_read_100_300,FP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,3]=TP_read_100_300\n",
    "df_all_new.iloc[4,4]=FP_read_100_300\n",
    "df_all_new.iloc[4,5]=FN_read_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100_300, FP_read_100_300, FN_read_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[4]=str(df_all_new['sensitivity (95% CI)'].iloc[4])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[4]=str(df_all_new['PPV (95% CI)'].iloc[4])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[4]=str(df_all_new['F1 score (95% CI)'].iloc[4])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "df_all_new.iloc[2,0]=0\n",
    "df_all_new.iloc[2,1]=0\n",
    "df_all_new.iloc[2,2]=0\n",
    "df_all_new.iloc[2,3]=0\n",
    "df_all_new.iloc[2,4]=0\n",
    "df_all_new.iloc[2,5]=0\n",
    "\n",
    "AI_all=np.sum(df_all_new.iloc[0:2,3:].values)\n",
    "reader_all=np.sum(df_all_new.iloc[3:5,3:].values)\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules correctly detected']+df_all_new['non-nodules incorrectly detected']+df_all_new['nodules missed']\n",
    "\n",
    "df_all_new.iloc[5,0]=''\n",
    "df_all_new.iloc[5,1]=''\n",
    "df_all_new.iloc[5,2]=''\n",
    "df_all_new.iloc[5,3]=''\n",
    "df_all_new.iloc[5,4]=''\n",
    "df_all_new.iloc[5,5]=''\n",
    "df_all_new.iloc[5,6]=np.sum(df_all_new['All findings'].iloc[3:5])\n",
    "\n",
    "df_all_new.iloc[2,0]=''\n",
    "df_all_new.iloc[2,1]=''\n",
    "df_all_new.iloc[2,2]=''\n",
    "df_all_new.iloc[2,3]=''\n",
    "df_all_new.iloc[2,4]=''\n",
    "df_all_new.iloc[2,5]=''\n",
    "df_all_new.iloc[2,6]=np.sum(df_all_new['All findings'].iloc[0:2])\n",
    "\n",
    "# print(df_all_new['sensitivity (95% CI)'])\n",
    "for i in range(5):\n",
    "    if i!=2:\n",
    "        if i<2:\n",
    "            sum_all=AI_all\n",
    "        elif i>2:\n",
    "            sum_all=reader_all\n",
    "            \n",
    "        percentage_tp=np.round((df_all_new.iloc[i][3]/sum_all)*100,1) \n",
    "        df_all_new['nodules correctly detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "\n",
    "        percentage_fp=np.round((df_all_new.iloc[i][4]/sum_all)*100,1) \n",
    "        df_all_new['non-nodules incorrectly detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "\n",
    "        percentage_fn=np.round((df_all_new.iloc[i][5]/sum_all)*100,1) \n",
    "        df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "        df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' ('+str(np.round(100*df_all_new.iloc[i][6]/sum_all,1))+'%)'\n",
    "\n",
    "    \n",
    "df_all_new['All findings'].iloc[2]=str(df_all_new.iloc[2][6])+' (100%)'\n",
    "df_all_new['All findings'].iloc[5]=str(df_all_new.iloc[5][6])+' (100%)'\n",
    "\n",
    "df_all_new #Detection performance comparison for nodules and lymph nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "0a823852",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_only_volumes_nonemphysema_all.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "52338c22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For nodules only (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is 0.6069054272179508\n",
      "For FP findings of 30-100mm3, with continuity correction (not exact) p value is 0.005110260660855866\n",
      "\n",
      "\n",
      "For nodules only (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is 0.4496917979688908\n",
      "For FP findings of 100-300mm3, with continuity correction (not exact) p value is 0.009823274507519235\n"
     ]
    }
   ],
   "source": [
    "data=[[TP_both_100, FN_read_100],\n",
    "        [FN_AI_100,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules only (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "#For FPs\n",
    "data=[[0, FP_AI_100], \n",
    "        [FP_read_100, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For FP findings of 30-100mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "print(\"\\n\")\n",
    "\n",
    "\n",
    "\n",
    "data=[[TP_both_100_300,FN_read_100_300 ], \n",
    "        [FN_AI_100_300, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules only (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "#For FPs\n",
    "data=[[0, FP_AI_100_300], \n",
    "        [FP_read_100_300, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For FP findings of 100-300mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ed5a3442",
   "metadata": {},
   "source": [
    "Analysis based on volume for subcategories not possible since we only have volume subgroups for TPs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4b2e122c",
   "metadata": {},
   "source": [
    "Nodule types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "ae0cb9d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "# #Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "# df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "#                           index=['pleural nodules',\n",
    "#                                  'calcified nodules',\n",
    "#                                  'subsolid & ground glass nodules',\n",
    "#                                  'other nodules',\n",
    "#                                  'atypical PFNs',\n",
    "#                                  'typical PFNs & periphysural lymph nodes',\n",
    "#                                  'bronchiovascular lymph nodes'\n",
    "#                                 ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "\n",
    "# df_categories['FP']=[sum([len(x) for x in pleural_FP_emph.values()])+sum([len(x) for x in pleural_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in calcif_FP_emph.values()])+sum([len(x) for x in calcif_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in sub_ground_FP_emph.values()])+sum([len(x) for x in sub_ground_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in other_nodules_FP_emph.values()])+sum([len(x) for x in other_nodules_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in atyp_FP_emph.values()])+sum([len(x) for x in atyp_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in per_FP_emph.values()])+sum([len(x) for x in per_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in bronchioperi_FP_emph.values()])+sum([len(x) for x in bronchioperi_FP_noemph.values()])\n",
    "# ]\n",
    "\n",
    "\n",
    "# df_categories['FN']=[sum([len(x) for x in pleural_FN_emph.values()])+sum([len(x) for x in pleural_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in calcif_FN_emph.values()])+sum([len(x) for x in calcif_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in sub_ground_FN_emph.values()])+sum([len(x) for x in sub_ground_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in other_nodules_FN_emph.values()])+sum([len(x) for x in other_nodules_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in atyp_FN_emph.values()])+sum([len(x) for x in atyp_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in per_FN_emph.values()])+sum([len(x) for x in per_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in bronchioperi_FN_emph.values()])+sum([len(x) for x in bronchioperi_FN_noemph.values()])\n",
    "# ]\n",
    "\n",
    "\n",
    "# df_categories['TP']=[pleural_emph_nod_only+pleural_noemph_nod_only,\n",
    "#                      calcified_emph_nod_only+calcified_noemph_nod_only,\n",
    "#                      sub_ground_emph_nod_only+sub_ground_noemph_nod_only,\n",
    "#                      other_all_emph_nod_only+other_all_noemph_nod_only,\n",
    "#                      atypical_triangular_emph_lymph+atypical_triangular_noemph_lymph,\n",
    "#                      per_fisu_emph_lymph+per_fisu_noemph_lymph,\n",
    "#                      peri_bronch_emph_lymph+peri_bronch_noemph_lymph\n",
    "# ]\n",
    "\n",
    "# df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "# df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "# total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "# all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "# percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "# df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "# percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "# df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "# percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "# df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "# df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "# #Rename columns\n",
    "# df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "# df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "74499c25",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "#Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "                          index=['pleural nodules',\n",
    "                                 'calcified nodules',\n",
    "                                 'subsolid & ground glass nodules',\n",
    "                                 'other nodules'\n",
    "                                ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['FP']=[sum([len(x) for x in pleural_FP_emph.values()])+sum([len(x) for x in pleural_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in calcif_FP_emph.values()])+sum([len(x) for x in calcif_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in sub_ground_FP_emph.values()])+sum([len(x) for x in sub_ground_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in other_nodules_FP_emph.values()])+sum([len(x) for x in other_nodules_FP_noemph.values()])]\n",
    "\n",
    "df_categories['FN']=[sum([len(x) for x in pleural_FN_emph.values()])+sum([len(x) for x in pleural_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in calcif_FN_emph.values()])+sum([len(x) for x in calcif_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in sub_ground_FN_emph.values()])+sum([len(x) for x in sub_ground_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in other_nodules_FN_emph.values()])+sum([len(x) for x in other_nodules_FN_noemph.values()])]\n",
    "\n",
    "df_categories['TP']=[pleural_emph_nod_only+pleural_noemph_nod_only,\n",
    "                     calcified_emph_nod_only+calcified_noemph_nod_only,\n",
    "                     sub_ground_emph_nod_only+sub_ground_noemph_nod_only,\n",
    "                     other_all_emph_nod_only+other_all_noemph_nod_only]\n",
    "\n",
    "df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "#Rename columns\n",
    "df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "e8e237ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_categories.to_excel('nodule_types_all.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a0fede7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Emphysema only\n",
    "#Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "#Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "                          index=['pleural nodules',\n",
    "                                 'calcified nodules',\n",
    "                                 'subsolid & ground glass nodules',\n",
    "                                 'other nodules'\n",
    "                                ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['FP']=[sum([len(x) for x in pleural_FP_emph.values()]),\n",
    "                     sum([len(x) for x in calcif_FP_emph.values()]),\n",
    "                     sum([len(x) for x in sub_ground_FP_emph.values()]),\n",
    "                     sum([len(x) for x in other_nodules_FP_emph.values()])]\n",
    "\n",
    "df_categories['FN']=[sum([len(x) for x in pleural_FN_emph.values()]),\n",
    "                     sum([len(x) for x in calcif_FN_emph.values()]),\n",
    "                     sum([len(x) for x in sub_ground_FN_emph.values()]),\n",
    "                     sum([len(x) for x in other_nodules_FN_emph.values()])]\n",
    "\n",
    "df_categories['TP']=[pleural_emph_nod_only,\n",
    "                     calcified_emph_nod_only,\n",
    "                     sub_ground_emph_nod_only,\n",
    "                     other_all_emph_nod_only]\n",
    "\n",
    "df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "#Rename columns\n",
    "df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "29bcfb0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('nodule_types_emph.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "401eef2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Non-emphysema\n",
    "#Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "#Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "                          index=['pleural nodules',\n",
    "                                 'calcified nodules',\n",
    "                                 'subsolid & ground glass nodules',\n",
    "                                 'other nodules'\n",
    "                                ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['FP']=[sum([len(x) for x in pleural_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in calcif_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in sub_ground_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in other_nodules_FP_noemph.values()])]\n",
    "\n",
    "df_categories['FN']=[sum([len(x) for x in pleural_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in calcif_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in sub_ground_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in other_nodules_FN_noemph.values()])]\n",
    "\n",
    "df_categories['TP']=[pleural_noemph_nod_only,\n",
    "                     calcified_noemph_nod_only,\n",
    "                     sub_ground_noemph_nod_only,\n",
    "                     other_all_noemph_nod_only]\n",
    "\n",
    "df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "#Rename columns\n",
    "df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "f7509aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('nodule_types_noemph.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "510a702c",
   "metadata": {},
   "source": [
    "Same as above without TP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "615bcd44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Emphysema only\n",
    "# #Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "# #Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "# df_categories=pd.DataFrame(columns=['FP','FN'], #below index with the correct order as above\n",
    "#                           index=['pleural nodules',\n",
    "#                                  'calcified nodules',\n",
    "#                                  'subsolid & ground glass nodules',\n",
    "#                                  'other nodules'\n",
    "#                                 ])\n",
    "\n",
    "# # df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "# df_categories['FP']=[sum([len(x) for x in pleural_FP_emph.values()]),\n",
    "#                      sum([len(x) for x in calcif_FP_emph.values()]),\n",
    "#                      sum([len(x) for x in sub_ground_FP_emph.values()]),\n",
    "#                      sum([len(x) for x in other_nodules_FP_emph.values()])]\n",
    "\n",
    "# df_categories['FN']=[sum([len(x) for x in pleural_FN_emph.values()]),\n",
    "#                      sum([len(x) for x in calcif_FN_emph.values()]),\n",
    "#                      sum([len(x) for x in sub_ground_FN_emph.values()]),\n",
    "#                      sum([len(x) for x in other_nodules_FN_emph.values()])]\n",
    "\n",
    "# all_findings_fp=df_categories['FP'].sum()\n",
    "# all_findings_fn=df_categories['FN'].sum()\n",
    "\n",
    "# percentage_fp=np.round((df_categories['FP']/all_findings_fp)*100,1) \n",
    "# df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "# percentage_fn=np.round((df_categories['FN']/all_findings_fn)*100,1) #sum(df_categories['emphysema'])\n",
    "# df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "\n",
    "# df_categories.loc['Total']= [str(all_findings_fp)+ ' (100%)',str(all_findings_fn)+' (100%)']\n",
    "\n",
    "# #Rename columns\n",
    "# df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found'}, inplace=True)\n",
    "\n",
    "# df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "1c80b059",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('nodule_types_emph_notp.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "553516ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Non-emphysema\n",
    "# #Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "# #Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "# df_categories=pd.DataFrame(columns=['FP','FN'], #below index with the correct order as above\n",
    "#                           index=['pleural nodules',\n",
    "#                                  'calcified nodules',\n",
    "#                                  'subsolid & ground glass nodules',\n",
    "#                                  'other nodules'\n",
    "#                                 ])\n",
    "\n",
    "# # df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "# df_categories['FP']=[sum([len(x) for x in pleural_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in calcif_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in sub_ground_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in other_nodules_FP_noemph.values()])]\n",
    "\n",
    "# df_categories['FN']=[sum([len(x) for x in pleural_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in calcif_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in sub_ground_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in other_nodules_FN_noemph.values()])]\n",
    "\n",
    "\n",
    "# all_findings_fp=df_categories['FP'].sum()\n",
    "# all_findings_fn=df_categories['FN'].sum()\n",
    "\n",
    "# percentage_fp=np.round((df_categories['FP']/all_findings_fp)*100,1) \n",
    "# df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "# percentage_fn=np.round((df_categories['FN']/all_findings_fn)*100,1) #sum(df_categories['emphysema'])\n",
    "# df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "\n",
    "# df_categories.loc['Total']= [str(all_findings_fp)+ ' (100%)',str(all_findings_fn)+' (100%)']\n",
    "\n",
    "# #Rename columns\n",
    "# df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found'}, inplace=True) #, 'TP':'Both found'\n",
    "\n",
    "# df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "897fadd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('nodule_types_noemph_notp.xlsx')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "73aea955",
   "metadata": {},
   "source": [
    "### Lymph node types (atypical_triangular_emph_lymph transformed to atypical_triangular_emph_nod_only etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "be3f03ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "#Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "                          index=['atypical PFNs',\n",
    "                                 'typical PFNs & periphysural lymph nodes',\n",
    "                                 'bronchiovascular lymph nodes'\n",
    "                                ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['FP']=[sum([len(x) for x in atyp_FP_emph.values()])+sum([len(x) for x in atyp_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in per_FP_emph.values()])+sum([len(x) for x in per_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in bronchioperi_FP_emph.values()])+sum([len(x) for x in bronchioperi_FP_noemph.values()])]\n",
    "\n",
    "df_categories['FN']=[sum([len(x) for x in atyp_FN_emph.values()])+sum([len(x) for x in atyp_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in per_FN_emph.values()])+sum([len(x) for x in per_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in bronchioperi_FN_emph.values()])+sum([len(x) for x in bronchioperi_FN_noemph.values()])]\n",
    "\n",
    "df_categories['TP']=[atypical_triangular_emph_nod_only+atypical_triangular_noemph_nod_only,\n",
    "                     per_fisu_emph_lymph+per_fisu_noemph_lymph,\n",
    "                     peri_bronch_emph_lymph+peri_bronch_noemph_lymph]\n",
    "\n",
    "df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "#Rename columns\n",
    "df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "9024fc0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_categories.to_excel('lymph_types_all.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "668c77b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Emphysema\n",
    "#Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "#Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "                          index=['atypical PFNs',\n",
    "                                 'typical PFNs & periphysural lymph nodes',\n",
    "                                 'bronchiovascular lymph nodes'\n",
    "                                ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['FP']=[sum([len(x) for x in atyp_FP_emph.values()]),\n",
    "                     sum([len(x) for x in per_FP_emph.values()]),\n",
    "                     sum([len(x) for x in bronchioperi_FP_emph.values()])]\n",
    "\n",
    "df_categories['FN']=[sum([len(x) for x in atyp_FN_emph.values()]),\n",
    "                     sum([len(x) for x in per_FN_emph.values()]),\n",
    "                     sum([len(x) for x in bronchioperi_FN_emph.values()])]\n",
    "\n",
    "df_categories['TP']=[atypical_triangular_emph_nod_only,\n",
    "                     per_fisu_emph_lymph,\n",
    "                     peri_bronch_emph_lymph]\n",
    "\n",
    "df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "#Rename columns\n",
    "df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "97e63f9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('lymph_types_emph.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ad53b53e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Non-emphysema\n",
    "#Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "#Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "df_categories=pd.DataFrame(columns=['TP','FP','FN'], #below index with the correct order as above\n",
    "                          index=['atypical PFNs',\n",
    "                                 'typical PFNs & periphysural lymph nodes',\n",
    "                                 'bronchiovascular lymph nodes'\n",
    "                                ])\n",
    "\n",
    "# df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "df_categories['FP']=[sum([len(x) for x in atyp_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in per_FP_noemph.values()]),\n",
    "                     sum([len(x) for x in bronchioperi_FP_noemph.values()])]\n",
    "\n",
    "df_categories['FN']=[sum([len(x) for x in atyp_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in per_FN_noemph.values()]),\n",
    "                     sum([len(x) for x in bronchioperi_FN_noemph.values()])]\n",
    "\n",
    "df_categories['TP']=[atypical_triangular_noemph_nod_only,\n",
    "                    per_fisu_noemph_lymph,\n",
    "                     peri_bronch_noemph_lymph]\n",
    "\n",
    "df_categories['All findings']=df_categories['FP']+df_categories['FN']+df_categories['TP']\n",
    "\n",
    "df_categories.loc['Total']= df_categories.sum()\n",
    "\n",
    "total_num_discrepancies_with_tp=df_categories.iloc[:-1,:-1].sum().sum() #To be used in next cells for percentages\n",
    "\n",
    "all_findings=df_categories.iloc[:-1,:-1].sum().sum()\n",
    "\n",
    "percentage_fp=np.round((df_categories['FP']/total_num_discrepancies_with_tp)*100,1) \n",
    "df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['FN']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "percentage_fn=np.round((df_categories['TP']/total_num_discrepancies_with_tp)*100,1) #sum(df_categories['emphysema'])\n",
    "df_categories['TP']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['TP'].items())]\n",
    "\n",
    "df_categories['All findings']=[str(val)+' ('+str(np.round(100*val/total_num_discrepancies_with_tp,1))+'%)' for val in df_categories['All findings'].values]\n",
    "\n",
    "#Rename columns\n",
    "df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found', 'TP':'Both found'}, inplace=True)\n",
    "\n",
    "df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "71d86c37",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('lymph_types_noemph.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0696773",
   "metadata": {},
   "source": [
    "Same as above without TP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "ab66f123",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "# #Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "# df_categories=pd.DataFrame(columns=['FP','FN'], #below index with the correct order as above\n",
    "#                           index=['atypical PFNs',\n",
    "#                                  'typical PFNs & periphysural lymph nodes',\n",
    "#                                  'bronchiovascular lymph nodes'\n",
    "#                                 ])\n",
    "\n",
    "# # df_categories.index.name = 'GT by radiologists for discrepancies'\n",
    "\n",
    "# df_categories['FP']=[sum([len(x) for x in atyp_FP_emph.values()])+sum([len(x) for x in atyp_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in per_FP_emph.values()])+sum([len(x) for x in per_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in bronchioperi_FP_emph.values()])+sum([len(x) for x in bronchioperi_FP_noemph.values()])]\n",
    "\n",
    "# df_categories['FN']=[sum([len(x) for x in atyp_FN_emph.values()])+sum([len(x) for x in atyp_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in per_FN_emph.values()])+sum([len(x) for x in per_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in bronchioperi_FN_emph.values()])+sum([len(x) for x in bronchioperi_FN_noemph.values()])]\n",
    "\n",
    "\n",
    "# all_findings_fp=df_categories['FP'].sum()\n",
    "# all_findings_fn=df_categories['FN'].sum()\n",
    "\n",
    "# percentage_fp=np.round((df_categories['FP']/all_findings_fp)*100,1) \n",
    "# df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "# percentage_fn=np.round((df_categories['FN']/all_findings_fn)*100,1) #sum(df_categories['emphysema'])\n",
    "# df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "# df_categories.loc['Total']= [str(all_findings_fp)+ ' (100%)',str(all_findings_fn)+' (100%)']\n",
    "\n",
    "# #Rename columns\n",
    "# df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found'}, inplace=True)\n",
    "\n",
    "# df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "0b74feea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('lymph_types_emph_notp.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "c3a08fd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Non-emphysema\n",
    "# #Further analysis for nodule/lymph node subcategories - Not kept for now\n",
    "# #Detailed analysis of what detected or not from both AI and reader for each category in nodules & lymph nodes \n",
    "\n",
    "# df_categories=pd.DataFrame(columns=['FP','FN'], #below index with the correct order as above\n",
    "#                           index=['atypical PFNs',\n",
    "#                                  'typical PFNs & periphysural lymph nodes',\n",
    "#                                  'bronchiovascular lymph nodes'\n",
    "#                                 ])\n",
    "\n",
    "\n",
    "# df_categories['FP']=[sum([len(x) for x in atyp_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in per_FP_noemph.values()]),\n",
    "#                      sum([len(x) for x in bronchioperi_FP_noemph.values()])]\n",
    "\n",
    "# df_categories['FN']=[sum([len(x) for x in atyp_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in per_FN_noemph.values()]),\n",
    "#                      sum([len(x) for x in bronchioperi_FN_noemph.values()])]\n",
    "\n",
    "\n",
    "# all_findings_fp=df_categories['FP'].sum()\n",
    "# all_findings_fn=df_categories['FN'].sum()\n",
    "\n",
    "\n",
    "# percentage_fp=np.round((df_categories['FP']/all_findings_fp)*100,1) \n",
    "# df_categories['FP']=[str(value[1])+' ('+str(percentage_fp[index])+'%)' for index,value in enumerate(df_categories['FP'].items())]\n",
    "\n",
    "# percentage_fn=np.round((df_categories['FN']/all_findings_fn)*100,1) #sum(df_categories['emphysema'])\n",
    "# df_categories['FN']=[str(value[1])+' ('+str(percentage_fn[index])+'%)' for index,value in enumerate(df_categories['FN'].items())]\n",
    "\n",
    "# df_categories.loc['Total']= [str(all_findings_fp)+ ' (100%)',str(all_findings_fn)+' (100%)']\n",
    "\n",
    "# #Rename columns\n",
    "# df_categories.rename(columns={'FP': 'AI found, reader missed', 'FN': 'AI missed, reader found'}, inplace=True)\n",
    "\n",
    "# df_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "b7fc9b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_categories.to_excel('lymph_types_noemph_notp.xlsx')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c2bdf1d7",
   "metadata": {},
   "source": [
    "### Similar analysis as the one performed above (emphysema/non-emphysema, volume subgroups) but including lymph nodes this time"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "928c3dfd",
   "metadata": {},
   "source": [
    "Emphysema (nodules and lymph nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "d7cf075a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emphysema numbers\n",
      "TP_AI:  62\n",
      "FP_AI:  20\n",
      "FN_AI:  29\n",
      "TP_read:  69\n",
      "FP_read:  6\n",
      "FN_read:  22\n",
      "TP_both:  40\n"
     ]
    }
   ],
   "source": [
    "TP_AI= TP_AI_emph=TP_emph+FP_nods_emph\n",
    "FP_AI=FP_AI_emph=FP_nonods_emph\n",
    "FN_AI=FN_AI_emph=TP_read_only=FN_nods_emph #nodules and lymph nodes detected only by the reader\n",
    "\n",
    "TP_read=TP_read_emph=TP_emph+FN_nods_emph\n",
    "FP_read=FP_read_emph=FN_nonods_emph\n",
    "FN_read=FN_read_emph=TP_AI_only=FP_nods_emph #nodules and lymph nodes detected only by AI\n",
    "\n",
    "TP_both=TP_emph #Common nodules and lymph nodules detected by both AI and reader\n",
    "\n",
    "#Print the above\n",
    "print(\"Emphysema numbers\")\n",
    "print('TP_AI: ',TP_AI)\n",
    "print('FP_AI: ',FP_AI)\n",
    "print('FN_AI: ',FN_AI)\n",
    "print('TP_read: ',TP_read)\n",
    "print('FP_read: ',FP_read)\n",
    "print('FN_read: ',FN_read)\n",
    "print('TP_both: ',TP_both)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "83580731",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules detected</th>\n",
       "      <th>non-nodules detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, emphysema</th>\n",
       "      <td>0.68 (0.57, 0.77)</td>\n",
       "      <td>0.76 (0.65, 0.84)</td>\n",
       "      <td>0.72 (0.64, 0.78)</td>\n",
       "      <td>62 (55.9%)</td>\n",
       "      <td>20 (18.0%)</td>\n",
       "      <td>29 (26.1%)</td>\n",
       "      <td>111 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, emphysema</th>\n",
       "      <td>0.76 (0.66, 0.84)</td>\n",
       "      <td>0.92 (0.83, 0.97)</td>\n",
       "      <td>0.83 (0.76, 0.88)</td>\n",
       "      <td>69 (71.1%)</td>\n",
       "      <td>6 (6.2%)</td>\n",
       "      <td>22 (22.7%)</td>\n",
       "      <td>97 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>131</td>\n",
       "      <td>26</td>\n",
       "      <td>51</td>\n",
       "      <td>208</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  sensitivity (95% CI)       PPV (95% CI)  F1 score (95% CI)  \\\n",
       "AI, emphysema        0.68 (0.57, 0.77)  0.76 (0.65, 0.84)  0.72 (0.64, 0.78)   \n",
       "reader, emphysema    0.76 (0.66, 0.84)  0.92 (0.83, 0.97)  0.83 (0.76, 0.88)   \n",
       "Total                                                                          \n",
       "\n",
       "                  nodules detected non-nodules detected nodules missed  \\\n",
       "AI, emphysema           62 (55.9%)           20 (18.0%)     29 (26.1%)   \n",
       "reader, emphysema       69 (71.1%)             6 (6.2%)     22 (22.7%)   \n",
       "Total                          131                   26             51   \n",
       "\n",
       "                  All findings  \n",
       "AI, emphysema       111 (100%)  \n",
       "reader, emphysema    97 (100%)  \n",
       "Total                      208  "
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Table split in two tables: one for emphysema only and one for non-emphysema having also percentages.\n",
    "\n",
    "#Assessing detection performance - For nodules + lymph nodes \n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules detected','non-nodules detected','nodules missed'],\n",
    "                        index=['AI, emphysema', 'reader, emphysema'])\n",
    "\n",
    "#AI nodules only emph\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI,FP_AI),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI,FP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,3]=TP_AI\n",
    "df_all_new.iloc[0,4]=FP_AI\n",
    "df_all_new.iloc[0,5]=FN_AI\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI, FP_AI, FN_AI, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "#reader nodules only emph\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_read,FN_read),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_read,FP_read),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_read,FP_read,FN_read),2)\n",
    "df_all_new.iloc[1,3]=TP_read\n",
    "df_all_new.iloc[1,4]=FP_read\n",
    "df_all_new.iloc[1,5]=FN_read\n",
    "\n",
    "sensitivity_confidence_interval_read, PPV_confidence_interval_read, F1_confidence_interval_read\\\n",
    "    = sensitivity_and_specificity_with_confidence_intervals(TP_read, FP_read, FN_read, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_read=[np.round(x,2) for x in sensitivity_confidence_interval_read]\n",
    "ci_ppv_read=[np.round(x,2) for x in PPV_confidence_interval_read]\n",
    "ci_f1_read=[np.round(x,2) for x in F1_confidence_interval_read]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_read))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_read))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_read))\n",
    "\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules detected']+df_all_new['non-nodules detected']+df_all_new['nodules missed']\n",
    "df_all_new.loc['Total']= df_all_new.sum()\n",
    "df_all_new.loc['Total'].iloc[0:3]=''\n",
    "\n",
    "\n",
    "all_findings=df_all_new.iloc[:-1,3:-1].sum().sum()\n",
    "\n",
    "for i in range(2):\n",
    "    row_all=np.sum(df_all_new.iloc[i][3:6].values)\n",
    "\n",
    "    percentage_tp=np.round((df_all_new.iloc[i][3]/row_all)*100,1) \n",
    "    df_all_new['nodules detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "    percentage_fp=np.round((df_all_new.iloc[i][4]/row_all)*100,1) \n",
    "    df_all_new['non-nodules detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "    percentage_fn=np.round((df_all_new.iloc[i][5]/row_all)*100,1) \n",
    "    df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "    df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' (100%)'\n",
    "\n",
    "df_all_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "cd2b30c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_lymph_emphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "23e9d9bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "McNemar's test (nodules and lymphs), AI_vs_Reader with continuity correction (not exact) p value is 0.40081416938293424\n"
     ]
    }
   ],
   "source": [
    "#McNemar's test to compare Reader vs AI (using consensus panel)\n",
    "#Below format is: [[Both AI found and reader found, reader missed and AI found], [Reader found and AI missed, 0]]\n",
    "\n",
    "#For nodules\n",
    "data=[[TP_both, FN_read],\n",
    "        [FN_AI,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"McNemar's test (nodules and lymphs), AI_vs_Reader with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue) \n",
    "\n",
    "\n",
    "# #For FPs\n",
    "# data=[[0, FP_AI], \n",
    "#         [FP_read, 0]]\n",
    "# # print(data)\n",
    "\n",
    "# # McNemar's Test without continuity correction\n",
    "# print(\"For FP findings, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "ba71ed1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #AI vs GT, TP_both included\n",
    "# # Table looks like below:\n",
    "# #              GT\n",
    "# #             Yes                       No\n",
    "# # AI   Yes    TP_both+TP_AI_only      FP_AI\n",
    "# #      No     FN_AI                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_AI_only)],[0 for x in range(FP_AI) ],[1 for x in range(FN_AI) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)], [1 for x in range(TP_AI_only)],[1 for x in range(FP_AI) ],[0 for x in range(FN_AI) ]]\n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"AI vs consensus (for nodules and lymphs), kappa is \",cohen_kappa_score(rater_GT, rater_AI))\n",
    "\n",
    "# #Reader vs GT, TP_both included\n",
    "# # Table looks like below:\n",
    "# #                    GT\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes    TP_both+TP_read_only      FP_read\n",
    "# #           No     FN_read                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[0 for x in range(FP_read) ],[1 for x in range(FN_read) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[1 for x in range(FP_read) ],[0 for x in range(FN_read) ]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs consensus (for non-nodules only), kappa is \",cohen_kappa_score(rater_GT, rater_read))\n",
    "\n",
    "\n",
    "\n",
    "# #Due to small numbers of TP, FP and FN, we cannot calculate the kappa for consensus vs AI (or reader). Better for AI vs reader.\n",
    "\n",
    "# #Reader vs AI, TP_both included\n",
    "\n",
    "# # Table looks like below (for nodule and lymph nodes (not FP)):\n",
    "# #                    AI\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes    TP_both                  FN_AI\n",
    "# #           No     FN_read                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(FN_read) ],[0 for x in range(FN_AI)]] \n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[0 for x in range(FN_read) ],[1 for x in range(FN_AI)]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs AI (for nodules/lymph nodes only), kappa is \",cohen_kappa_score(rater_AI, rater_read))\n",
    "\n",
    "\n",
    "# # Table looks like below (for FP):\n",
    "# #                    AI\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes        0                     FP_read\n",
    "# #           No     FP_AI                       0\n",
    "\n",
    "# list_of_lists=[[0 for x in range(FP_read) ],[1 for x in range(FP_AI)]] \n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(FP_read) ],[0 for x in range(FP_AI)]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs AI (for non-nodules only), kappa is \",cohen_kappa_score(rater_AI, rater_read))\n",
    "\n",
    "# #For both nodules/lymphs and non-nodules:\n",
    "# list_of_lists=[[0 for x in range(FP_read) ],[1 for x in range(FP_AI)],[1 for x in range(TP_both)],[1 for x in range(FN_read) ],[0 for x in range(FN_AI)]] \n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(FP_read) ],[0 for x in range(FP_AI)],[1 for x in range(TP_both)],[0 for x in range(FN_read) ],[1 for x in range(FN_AI)]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs AI (for both nods/lymphs and non-nodules), kappa is \",cohen_kappa_score(rater_AI, rater_read))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1292bcb5",
   "metadata": {},
   "source": [
    "Non-emphysema (nodules and lymph nodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "f6286202",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-emphysema numbers\n",
      "TP_AI:  96\n",
      "FP_AI:  18\n",
      "FN_AI:  40\n",
      "TP_read:  109\n",
      "FP_read:  22\n",
      "FN_read:  27\n",
      "TP_both:  69\n"
     ]
    }
   ],
   "source": [
    "TP_AI=TP_AI_noemph=TP_noemph+FP_nods_noemph\n",
    "FP_AI=FP_AI_noemph=FP_nonods_noemph\n",
    "FN_AI=FN_AI_noemph=TP_read_only=FN_nods_noemph #nodules and lymph nodes detected only by the reader\n",
    "\n",
    "TP_read=TP_read_noemph=TP_noemph+FN_nods_noemph\n",
    "FP_read=FP_read_noemph=FN_nonods_noemph\n",
    "FN_read=FN_read_noemph=TP_AI_only=FP_nods_noemph #nodules and lymph nodes detected only by AI\n",
    "\n",
    "TP_both=TP_noemph #Common nodules and lymph nodules detected by both AI and reader\n",
    "\n",
    "#Print the above\n",
    "print(\"Non-emphysema numbers\")\n",
    "print('TP_AI: ',TP_AI)\n",
    "print('FP_AI: ',FP_AI)\n",
    "print('FN_AI: ',FN_AI)\n",
    "print('TP_read: ',TP_read)\n",
    "print('FP_read: ',FP_read)\n",
    "print('FN_read: ',FN_read)\n",
    "print('TP_both: ',TP_both)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "79345ae3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules detected</th>\n",
       "      <th>non-nodules detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, nonemphysema</th>\n",
       "      <td>0.71 (0.62, 0.78)</td>\n",
       "      <td>0.84 (0.76, 0.9)</td>\n",
       "      <td>0.77 (0.71, 0.82)</td>\n",
       "      <td>96 (62.3%)</td>\n",
       "      <td>18 (11.7%)</td>\n",
       "      <td>40 (26.0%)</td>\n",
       "      <td>154 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, nonemphysema</th>\n",
       "      <td>0.8 (0.72, 0.86)</td>\n",
       "      <td>0.83 (0.75, 0.89)</td>\n",
       "      <td>0.82 (0.76, 0.86)</td>\n",
       "      <td>109 (69.0%)</td>\n",
       "      <td>22 (13.9%)</td>\n",
       "      <td>27 (17.1%)</td>\n",
       "      <td>158 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Total</th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>205</td>\n",
       "      <td>40</td>\n",
       "      <td>67</td>\n",
       "      <td>312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     sensitivity (95% CI)       PPV (95% CI)  \\\n",
       "AI, nonemphysema        0.71 (0.62, 0.78)   0.84 (0.76, 0.9)   \n",
       "reader, nonemphysema     0.8 (0.72, 0.86)  0.83 (0.75, 0.89)   \n",
       "Total                                                          \n",
       "\n",
       "                      F1 score (95% CI) nodules detected non-nodules detected  \\\n",
       "AI, nonemphysema      0.77 (0.71, 0.82)       96 (62.3%)           18 (11.7%)   \n",
       "reader, nonemphysema  0.82 (0.76, 0.86)      109 (69.0%)           22 (13.9%)   \n",
       "Total                                                205                   40   \n",
       "\n",
       "                     nodules missed All findings  \n",
       "AI, nonemphysema         40 (26.0%)   154 (100%)  \n",
       "reader, nonemphysema     27 (17.1%)   158 (100%)  \n",
       "Total                            67          312  "
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Assessing detection performance - For nodules and lymph nodes \n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules detected','non-nodules detected','nodules missed'],\n",
    "                        index=['AI, nonemphysema', 'reader, nonemphysema' ])\n",
    "\n",
    "\n",
    "#AI nodules only nonemph\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI,FP_AI),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI,FP_AI,FN_AI),2)\n",
    "df_all_new.iloc[0,3]=TP_AI\n",
    "df_all_new.iloc[0,4]=FP_AI\n",
    "df_all_new.iloc[0,5]=FN_AI\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI, FP_AI, FN_AI, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "#reader nodules only nonemph\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_read,FN_read),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_read,FP_read),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_read,FP_read,FN_read),2)\n",
    "df_all_new.iloc[1,3]=TP_read\n",
    "df_all_new.iloc[1,4]=FP_read\n",
    "df_all_new.iloc[1,5]=FN_read\n",
    "\n",
    "sensitivity_confidence_interval_read, PPV_confidence_interval_read, F1_confidence_interval_read\\\n",
    "    = sensitivity_and_specificity_with_confidence_intervals(TP_read, FP_read, FN_read, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_read=[np.round(x,2) for x in sensitivity_confidence_interval_read]\n",
    "ci_ppv_read=[np.round(x,2) for x in PPV_confidence_interval_read]\n",
    "ci_f1_read=[np.round(x,2) for x in F1_confidence_interval_read]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_read))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_read))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_read))\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules detected']+df_all_new['non-nodules detected']+df_all_new['nodules missed']\n",
    "df_all_new.loc['Total']= df_all_new.sum()\n",
    "df_all_new.loc['Total'].iloc[0:3]=''\n",
    "\n",
    "all_findings=df_all_new.iloc[:-1,3:-1].sum().sum()\n",
    "\n",
    "for i in range(2):\n",
    "    row_all=np.sum(df_all_new.iloc[i][3:6].values)\n",
    "    percentage_fp=np.round((df_all_new.iloc[i][4]/row_all)*100,1) \n",
    "    df_all_new['non-nodules detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "    percentage_tp=np.round((df_all_new.iloc[i][3]/row_all)*100,1) \n",
    "    df_all_new['nodules detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "    percentage_fn=np.round((df_all_new.iloc[i][5]/row_all)*100,1) \n",
    "    df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "    df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' (100%)'\n",
    "\n",
    "df_all_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "24cb0b0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_lymph_nonemphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "21aa7a51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "McNemar's test (nodules and lymphs), AI_vs_Reader with continuity correction (not exact) p value is 0.14263920633056812\n"
     ]
    }
   ],
   "source": [
    "#McNemar's test to compare Reader vs AI (using consensus panel)\n",
    "#Below format is: [[Both AI found and reader found, reader missed and AI found], [Reader found and AI missed, 0]]\n",
    "\n",
    "#For nodules\n",
    "data=[[TP_both, FN_read],\n",
    "        [FN_AI,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"McNemar's test (nodules and lymphs), AI_vs_Reader with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue) \n",
    "\n",
    "\n",
    "# #For FPs\n",
    "# data=[[0, FP_AI], \n",
    "#         [FP_read, 0]]\n",
    "# # print(data)\n",
    "\n",
    "# # McNemar's Test without continuity correction\n",
    "# print(\"For FP findings, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "2f87de85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #AI vs GT, TP_both included\n",
    "# # Table looks like below:\n",
    "# #              GT\n",
    "# #             Yes                       No\n",
    "# # AI   Yes    TP_both+TP_AI_only      FP_AI\n",
    "# #      No     FN_AI                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_AI_only)],[0 for x in range(FP_AI) ],[1 for x in range(FN_AI) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)], [1 for x in range(TP_AI_only)],[1 for x in range(FP_AI) ],[0 for x in range(FN_AI) ]]\n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"AI vs consensus (for nodules and lymphs), kappa is \",cohen_kappa_score(rater_GT, rater_AI))\n",
    "\n",
    "# #Reader vs GT, TP_both included\n",
    "# # Table looks like below:\n",
    "# #                    GT\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes    TP_both+TP_read_only      FP_read\n",
    "# #           No     FN_read                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[0 for x in range(FP_read) ],[1 for x in range(FN_read) ] ] \n",
    "# rater_GT=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(TP_read_only)],[1 for x in range(FP_read) ],[0 for x in range(FN_read) ]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs consensus (for non-nodules only), kappa is \",cohen_kappa_score(rater_GT, rater_read))\n",
    "\n",
    "# #Due to small numbers of TP, FP and FN, we cannot calculate the kappa for consensus vs AI (or reader). Better for AI vs reader.\n",
    "\n",
    "# #Reader vs AI, TP_both included\n",
    "\n",
    "# # Table looks like below (for nodule and lymph nodes (not FP)):\n",
    "# #                    AI\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes    TP_both                  FN_AI\n",
    "# #           No     FN_read                   0\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[1 for x in range(FN_read) ],[0 for x in range(FN_AI)]] \n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(TP_both)],[0 for x in range(FN_read) ],[1 for x in range(FN_AI)]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs AI (for nodules/lymph nodes only), kappa is \",cohen_kappa_score(rater_AI, rater_read))\n",
    "\n",
    "\n",
    "# # Table looks like below (for FP):\n",
    "# #                    AI\n",
    "# #                   Yes                       No\n",
    "# # Reader   Yes        0                     FP_read\n",
    "# #           No     FP_AI                       0\n",
    "\n",
    "# list_of_lists=[[0 for x in range(FP_read) ],[1 for x in range(FP_AI)]] \n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(FP_read) ],[0 for x in range(FP_AI)]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs AI (for non-nodules only), kappa is \",cohen_kappa_score(rater_AI, rater_read))\n",
    "\n",
    "\n",
    "# #For both nodules/lymphs and non-nodules:\n",
    "# list_of_lists=[[0 for x in range(FP_read) ],[1 for x in range(FP_AI)],[1 for x in range(TP_both)],[1 for x in range(FN_read) ],[0 for x in range(FN_AI)]] \n",
    "# rater_AI=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# list_of_lists=[[1 for x in range(FP_read) ],[0 for x in range(FP_AI)],[1 for x in range(TP_both)],[0 for x in range(FN_read) ],[1 for x in range(FN_AI)]]\n",
    "# rater_read=[item for sublist in list_of_lists for item in sublist]\n",
    "\n",
    "# print(\"Reader vs AI (for both nods/lymphs and non-nodules), kappa is \",cohen_kappa_score(rater_AI, rater_read))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "11047c87",
   "metadata": {},
   "source": [
    "Note: We should not perform comparisons between emphysema/non-emphysema groups using McNemar's test - It should only be used in the same group of participants!"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3f47400d",
   "metadata": {},
   "source": [
    "#### Same analysis for volume subgroups (nodules and lymph nodes)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "946955fc",
   "metadata": {},
   "source": [
    "Emphysema volume subgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "d38a146f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Emphysema numbers\n",
      "TP_AI_100:  37\n",
      "FP_AI_100:  8\n",
      "FN_AI_100:  26\n",
      "TP_read_100:  50\n",
      "FP_read_100:  5\n",
      "FN_read_100:  13\n",
      "TP_AI_100_300:  25\n",
      "FP_AI_100_300:  12\n",
      "FN_AI_100_300:  3\n",
      "TP_read_100_300:  19\n",
      "FP_read_100_300:  1\n",
      "FN_read_100_300:  9\n",
      "TP_both_100:  24\n",
      "TP_both_100_300:  16\n"
     ]
    }
   ],
   "source": [
    "TP_AI_100=TP_emph_30_100+ai_nods_emph_30_100 \n",
    "FP_AI_100=ai_nonods_emph_30_100\n",
    "FN_AI_100=reader_nods_emph_30_100 #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100=TP_emph_30_100+reader_nods_emph_30_100\n",
    "FP_read_100=reader_nonods_emph_30_100\n",
    "FN_read_100=ai_nods_emph_30_100 #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_AI_100_300=TP_emph_100_300+ai_nods_emph_100_300\n",
    "FP_AI_100_300=ai_nonods_emph_100_300\n",
    "FN_AI_100_300=reader_nods_emph_100_300 #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100_300=TP_emph_100_300+reader_nods_emph_100_300\n",
    "FP_read_100_300=reader_nonods_emph_100_300\n",
    "FN_read_100_300=ai_nods_emph_100_300 #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_both_100=TP_emph_30_100\n",
    "TP_both_100_300=TP_emph_100_300\n",
    "\n",
    "#Print the above\n",
    "print(\"Emphysema numbers\")\n",
    "print('TP_AI_100: ',TP_AI_100)\n",
    "print('FP_AI_100: ',FP_AI_100)\n",
    "print('FN_AI_100: ',FN_AI_100)\n",
    "print('TP_read_100: ',TP_read_100)\n",
    "print('FP_read_100: ',FP_read_100)\n",
    "print('FN_read_100: ',FN_read_100)\n",
    "print('TP_AI_100_300: ',TP_AI_100_300)\n",
    "print('FP_AI_100_300: ',FP_AI_100_300)\n",
    "print('FN_AI_100_300: ',FN_AI_100_300)\n",
    "print('TP_read_100_300: ',TP_read_100_300)\n",
    "print('FP_read_100_300: ',FP_read_100_300)\n",
    "print('FN_read_100_300: ',FN_read_100_300)\n",
    "print('TP_both_100: ',TP_both_100)\n",
    "print('TP_both_100_300: ',TP_both_100_300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "1d197759",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reader</th>\n",
       "      <th>AI</th>\n",
       "      <th>Consensus</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Emphysema cases</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Non-nodules</th>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP 30-100mm3</th>\n",
       "      <td>50</td>\n",
       "      <td>37</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP 100-300mm3</th>\n",
       "      <td>19</td>\n",
       "      <td>25</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Reader  AI  Consensus\n",
       "Emphysema cases                       \n",
       "Non-nodules           6  20         26\n",
       "TP 30-100mm3         50  37         39\n",
       "TP 100-300mm3        19  25         12"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Table with Reader, AI and consensus findings for nodules and lymph nodes for emphysema\n",
    "df_all_new=pd.DataFrame(columns=['Reader','AI','Consensus'],\n",
    "                        index=['Non-nodules','TP 30-100mm3','TP 100-300mm3'])\n",
    "\n",
    "df_all_new.index.name = 'Emphysema cases'\n",
    "\n",
    "df_all_new['Reader']=[FP_read_100+FP_read_100_300,TP_both_100+FN_AI_100,TP_both_100_300+FN_AI_100_300]\n",
    "df_all_new['AI']=[FP_AI_100+FP_AI_100_300,TP_both_100+FN_read_100,TP_both_100_300+FN_read_100_300]\n",
    "df_all_new['Consensus']=[FP_AI_100+FP_AI_100_300+FP_read_100+FP_read_100_300,FN_AI_100+FN_read_100, FN_AI_100_300+FN_read_100_300]\n",
    "\n",
    "df_all_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "5c4a756f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('non_nodules_and_TP_emphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "d62361e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules correctly detected</th>\n",
       "      <th>non-nodules incorrectly detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, emphysema 30-100mm3</th>\n",
       "      <td>0.59 (0.46, 0.71)</td>\n",
       "      <td>0.82 (0.67, 0.91)</td>\n",
       "      <td>0.69 (0.59, 0.77)</td>\n",
       "      <td>37 (33.3%)</td>\n",
       "      <td>8 (7.2%)</td>\n",
       "      <td>26 (23.4%)</td>\n",
       "      <td>71 (64.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AI, emphysema 100-300mm3</th>\n",
       "      <td>0.89 (0.71, 0.97)</td>\n",
       "      <td>0.68 (0.5, 0.81)</td>\n",
       "      <td>0.77 (0.65, 0.86)</td>\n",
       "      <td>25 (22.5%)</td>\n",
       "      <td>12 (10.8%)</td>\n",
       "      <td>3 (2.7%)</td>\n",
       "      <td>40 (36.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>111 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, emphysema 30-100mm3</th>\n",
       "      <td>0.79 (0.67, 0.88)</td>\n",
       "      <td>0.91 (0.79, 0.97)</td>\n",
       "      <td>0.85 (0.77, 0.9)</td>\n",
       "      <td>50 (51.5%)</td>\n",
       "      <td>5 (5.2%)</td>\n",
       "      <td>13 (13.4%)</td>\n",
       "      <td>68 (70.1%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, emphysema 100-300mm3</th>\n",
       "      <td>0.68 (0.48, 0.83)</td>\n",
       "      <td>0.95 (0.73, 1.0)</td>\n",
       "      <td>0.79 (0.65, 0.89)</td>\n",
       "      <td>19 (19.6%)</td>\n",
       "      <td>1 (1.0%)</td>\n",
       "      <td>9 (9.3%)</td>\n",
       "      <td>29 (29.9%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>97 (100%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     sensitivity (95% CI)       PPV (95% CI)  \\\n",
       "GT by radiologists for discrepancies                                           \n",
       "AI, emphysema 30-100mm3                 0.59 (0.46, 0.71)  0.82 (0.67, 0.91)   \n",
       "AI, emphysema 100-300mm3                0.89 (0.71, 0.97)   0.68 (0.5, 0.81)   \n",
       "                                                                               \n",
       "reader, emphysema 30-100mm3             0.79 (0.67, 0.88)  0.91 (0.79, 0.97)   \n",
       "reader, emphysema 100-300mm3            0.68 (0.48, 0.83)   0.95 (0.73, 1.0)   \n",
       "                                                                               \n",
       "\n",
       "                                      F1 score (95% CI)  \\\n",
       "GT by radiologists for discrepancies                      \n",
       "AI, emphysema 30-100mm3               0.69 (0.59, 0.77)   \n",
       "AI, emphysema 100-300mm3              0.77 (0.65, 0.86)   \n",
       "                                                          \n",
       "reader, emphysema 30-100mm3            0.85 (0.77, 0.9)   \n",
       "reader, emphysema 100-300mm3          0.79 (0.65, 0.89)   \n",
       "                                                          \n",
       "\n",
       "                                     nodules correctly detected  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "AI, emphysema 30-100mm3                              37 (33.3%)   \n",
       "AI, emphysema 100-300mm3                             25 (22.5%)   \n",
       "                                                                  \n",
       "reader, emphysema 30-100mm3                          50 (51.5%)   \n",
       "reader, emphysema 100-300mm3                         19 (19.6%)   \n",
       "                                                                  \n",
       "\n",
       "                                     non-nodules incorrectly detected  \\\n",
       "GT by radiologists for discrepancies                                    \n",
       "AI, emphysema 30-100mm3                                      8 (7.2%)   \n",
       "AI, emphysema 100-300mm3                                   12 (10.8%)   \n",
       "                                                                        \n",
       "reader, emphysema 30-100mm3                                  5 (5.2%)   \n",
       "reader, emphysema 100-300mm3                                 1 (1.0%)   \n",
       "                                                                        \n",
       "\n",
       "                                     nodules missed All findings  \n",
       "GT by radiologists for discrepancies                              \n",
       "AI, emphysema 30-100mm3                  26 (23.4%)   71 (64.0%)  \n",
       "AI, emphysema 100-300mm3                   3 (2.7%)   40 (36.0%)  \n",
       "                                                      111 (100%)  \n",
       "reader, emphysema 30-100mm3              13 (13.4%)   68 (70.1%)  \n",
       "reader, emphysema 100-300mm3               9 (9.3%)   29 (29.9%)  \n",
       "                                                       97 (100%)  "
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#For emphysema only comparison between reader and AI for volume subgroups\n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules correctly detected','non-nodules incorrectly detected','nodules missed'], \n",
    "                        index=['AI, emphysema 30-100mm3', 'AI, emphysema 100-300mm3','',\n",
    "                               'reader, emphysema 30-100mm3', 'reader, emphysema 100-300mm3',''\n",
    "                              ])\n",
    "\n",
    "df_all_new.index.name = 'GT by radiologists for discrepancies' \n",
    "\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI_100,FP_AI_100),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI_100,FP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,3]=TP_AI_100\n",
    "df_all_new.iloc[0,4]=FP_AI_100\n",
    "df_all_new.iloc[0,5]=FN_AI_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100, FP_AI_100, FN_AI_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "df_all_new.iloc[3,0]=np.round(sensitivity(TP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,1]=np.round(PPV(TP_read_100,FP_read_100),2)\n",
    "df_all_new.iloc[3,2]=np.round(F1score(TP_read_100,FP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,3]=TP_read_100\n",
    "df_all_new.iloc[3,4]=FP_read_100\n",
    "df_all_new.iloc[3,5]=FN_read_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100, FP_read_100, FN_read_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[3]=str(df_all_new['sensitivity (95% CI)'].iloc[3])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[3]=str(df_all_new['PPV (95% CI)'].iloc[3])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[3]=str(df_all_new['F1 score (95% CI)'].iloc[3])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_AI_100_300,FP_AI_100_300),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_AI_100_300,FP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,3]=TP_AI_100_300\n",
    "df_all_new.iloc[1,4]=FP_AI_100_300\n",
    "df_all_new.iloc[1,5]=FN_AI_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100_300, FP_AI_100_300, FN_AI_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "\n",
    "df_all_new.iloc[4,0]=np.round(sensitivity(TP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,1]=np.round(PPV(TP_read_100_300,FP_read_100_300),2)\n",
    "df_all_new.iloc[4,2]=np.round(F1score(TP_read_100_300,FP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,3]=TP_read_100_300\n",
    "df_all_new.iloc[4,4]=FP_read_100_300\n",
    "df_all_new.iloc[4,5]=FN_read_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100_300, FP_read_100_300, FN_read_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[4]=str(df_all_new['sensitivity (95% CI)'].iloc[4])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[4]=str(df_all_new['PPV (95% CI)'].iloc[4])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[4]=str(df_all_new['F1 score (95% CI)'].iloc[4])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "\n",
    "df_all_new.iloc[2,0]=0\n",
    "df_all_new.iloc[2,1]=0\n",
    "df_all_new.iloc[2,2]=0\n",
    "df_all_new.iloc[2,3]=0\n",
    "df_all_new.iloc[2,4]=0\n",
    "df_all_new.iloc[2,5]=0\n",
    "\n",
    "AI_all=np.sum(df_all_new.iloc[0:2,3:].values)\n",
    "reader_all=np.sum(df_all_new.iloc[3:5,3:].values)\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules correctly detected']+df_all_new['non-nodules incorrectly detected']+df_all_new['nodules missed']\n",
    "\n",
    "df_all_new.iloc[5,0]=''\n",
    "df_all_new.iloc[5,1]=''\n",
    "df_all_new.iloc[5,2]=''\n",
    "df_all_new.iloc[5,3]=''\n",
    "df_all_new.iloc[5,4]=''\n",
    "df_all_new.iloc[5,5]=''\n",
    "df_all_new.iloc[5,6]=np.sum(df_all_new['All findings'].iloc[3:5])\n",
    "\n",
    "df_all_new.iloc[2,0]=''\n",
    "df_all_new.iloc[2,1]=''\n",
    "df_all_new.iloc[2,2]=''\n",
    "df_all_new.iloc[2,3]=''\n",
    "df_all_new.iloc[2,4]=''\n",
    "df_all_new.iloc[2,5]=''\n",
    "df_all_new.iloc[2,6]=np.sum(df_all_new['All findings'].iloc[0:2])\n",
    "\n",
    "# print(df_all_new['sensitivity (95% CI)'])\n",
    "for i in range(5):\n",
    "    if i!=2:\n",
    "        if i<2:\n",
    "            sum_all=AI_all\n",
    "        elif i>2:\n",
    "            sum_all=reader_all\n",
    "            \n",
    "        percentage_tp=np.round((df_all_new.iloc[i][3]/sum_all)*100,1) \n",
    "        df_all_new['nodules correctly detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "\n",
    "        percentage_fp=np.round((df_all_new.iloc[i][4]/sum_all)*100,1) \n",
    "        df_all_new['non-nodules incorrectly detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "\n",
    "        percentage_fn=np.round((df_all_new.iloc[i][5]/sum_all)*100,1) \n",
    "        df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "        df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' ('+str(np.round(100*df_all_new.iloc[i][6]/sum_all,1))+'%)'\n",
    "\n",
    "    \n",
    "df_all_new['All findings'].iloc[2]=str(df_all_new.iloc[2][6])+' (100%)'\n",
    "df_all_new['All findings'].iloc[5]=str(df_all_new.iloc[5][6])+' (100%)'\n",
    "\n",
    "df_all_new #Detection performance comparison for nodules and lymph nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "acca778a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_lymph_volumes_emphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "eca9019d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For nodules and lymph nodes (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is 0.05466393589167511\n",
      "For nodules and lymph nodes (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is 0.14891467317876161\n"
     ]
    }
   ],
   "source": [
    "data=[[TP_both_100, FN_read_100],\n",
    "        [FN_AI_100,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules and lymph nodes (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "# #For FPs\n",
    "# data=[[0, FP_AI_100], \n",
    "#         [FP_read_100, 0]]\n",
    "# # print(data)\n",
    "\n",
    "# # McNemar's Test without continuity correction\n",
    "# print(\"For FP findings of 30-100mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "\n",
    "\n",
    "data=[[TP_both_100_300,FN_read_100_300 ], \n",
    "        [FN_AI_100_300, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules and lymph nodes (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "# #For FPs\n",
    "# data=[[0, FP_AI_100_300], \n",
    "#         [FP_read_100_300, 0]]\n",
    "# # print(data)\n",
    "\n",
    "# # McNemar's Test without continuity correction\n",
    "# print(\"For FP findings of 100-300mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4bcbc046",
   "metadata": {},
   "source": [
    "Nonemphysema volume subgroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "9baf96f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Non-emphysema numbers\n",
      "TP_AI_100:  70\n",
      "FP_AI_100:  5\n",
      "FN_AI_100:  38\n",
      "TP_read_100:  87\n",
      "FP_read_100:  20\n",
      "FN_read_100:  21\n",
      "TP_AI_100_300:  26\n",
      "FP_AI_100_300:  13\n",
      "FN_AI_100_300:  2\n",
      "TP_read_100_300:  22\n",
      "FP_read_100_300:  2\n",
      "FN_read_100_300:  6\n",
      "TP_both_100:  49\n",
      "TP_both_100_300:  20\n"
     ]
    }
   ],
   "source": [
    "TP_AI_100=TP_noemph_30_100+ai_nods_noemph_30_100 \n",
    "FP_AI_100=ai_nonods_noemph_30_100\n",
    "FN_AI_100=reader_nods_noemph_30_100 #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100=TP_noemph_30_100+reader_nods_noemph_30_100\n",
    "FP_read_100=reader_nonods_noemph_30_100\n",
    "FN_read_100=ai_nods_noemph_30_100 #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_AI_100_300=TP_noemph_100_300+ai_nods_noemph_100_300\n",
    "FP_AI_100_300=ai_nonods_noemph_100_300\n",
    "FN_AI_100_300=reader_nods_noemph_100_300 #nodules of reader excluding lymph nodes\n",
    "\n",
    "TP_read_100_300=TP_noemph_100_300+reader_nods_noemph_100_300\n",
    "FP_read_100_300=reader_nonods_noemph_100_300\n",
    "FN_read_100_300=ai_nods_noemph_100_300 #nodules of AI excluding lymph nodes\n",
    "\n",
    "TP_both_100=TP_noemph_30_100\n",
    "TP_both_100_300=TP_noemph_100_300\n",
    "\n",
    "#Print the above\n",
    "print(\"Non-emphysema numbers\")\n",
    "print('TP_AI_100: ',TP_AI_100)\n",
    "print('FP_AI_100: ',FP_AI_100)\n",
    "print('FN_AI_100: ',FN_AI_100)\n",
    "print('TP_read_100: ',TP_read_100)\n",
    "print('FP_read_100: ',FP_read_100)\n",
    "print('FN_read_100: ',FN_read_100)\n",
    "print('TP_AI_100_300: ',TP_AI_100_300)\n",
    "print('FP_AI_100_300: ',FP_AI_100_300)\n",
    "print('FN_AI_100_300: ',FN_AI_100_300)\n",
    "print('TP_read_100_300: ',TP_read_100_300)\n",
    "print('FP_read_100_300: ',FP_read_100_300)\n",
    "print('FN_read_100_300: ',FN_read_100_300)\n",
    "print('TP_both_100: ',TP_both_100)\n",
    "print('TP_both_100_300: ',TP_both_100_300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "3ba09661",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reader</th>\n",
       "      <th>AI</th>\n",
       "      <th>Consensus</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Non-Emphysema cases</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Non-nodules</th>\n",
       "      <td>22</td>\n",
       "      <td>18</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP 30-100mm3</th>\n",
       "      <td>87</td>\n",
       "      <td>70</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TP 100-300mm3</th>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Reader  AI  Consensus\n",
       "Non-Emphysema cases                       \n",
       "Non-nodules              22  18         40\n",
       "TP 30-100mm3             87  70         59\n",
       "TP 100-300mm3            22  26          8"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Table with Reader, AI and consensus findings for nodules and lymph nodes for non-emphysema\n",
    "df_all_new=pd.DataFrame(columns=['Reader','AI','Consensus'],\n",
    "                        index=['Non-nodules','TP 30-100mm3','TP 100-300mm3'])\n",
    "\n",
    "df_all_new.index.name = 'Non-Emphysema cases'\n",
    "\n",
    "df_all_new['Reader']=[FP_read_100+FP_read_100_300,TP_both_100+FN_AI_100,TP_both_100_300+FN_AI_100_300]\n",
    "df_all_new['AI']=[FP_AI_100+FP_AI_100_300,TP_both_100+FN_read_100,TP_both_100_300+FN_read_100_300]\n",
    "df_all_new['Consensus']=[FP_AI_100+FP_AI_100_300+FP_read_100+FP_read_100_300,FN_AI_100+FN_read_100, FN_AI_100_300+FN_read_100_300]\n",
    "\n",
    "df_all_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "869399ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('non_nodules_and_TP_nonemphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "0adc27ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sensitivity (95% CI)</th>\n",
       "      <th>PPV (95% CI)</th>\n",
       "      <th>F1 score (95% CI)</th>\n",
       "      <th>nodules correctly detected</th>\n",
       "      <th>non-nodules incorrectly detected</th>\n",
       "      <th>nodules missed</th>\n",
       "      <th>All findings</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GT by radiologists for discrepancies</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>AI, nonemphysema 30-100mm3</th>\n",
       "      <td>0.65 (0.55, 0.74)</td>\n",
       "      <td>0.93 (0.84, 0.98)</td>\n",
       "      <td>0.77 (0.7, 0.82)</td>\n",
       "      <td>70 (45.5%)</td>\n",
       "      <td>5 (3.2%)</td>\n",
       "      <td>38 (24.7%)</td>\n",
       "      <td>113 (73.4%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AI, nonemphysema 100-300mm3</th>\n",
       "      <td>0.93 (0.75, 0.99)</td>\n",
       "      <td>0.67 (0.5, 0.8)</td>\n",
       "      <td>0.78 (0.65, 0.87)</td>\n",
       "      <td>26 (16.9%)</td>\n",
       "      <td>13 (8.4%)</td>\n",
       "      <td>2 (1.3%)</td>\n",
       "      <td>41 (26.6%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>154 (100%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, nonemphysema 30-100mm3</th>\n",
       "      <td>0.81 (0.72, 0.87)</td>\n",
       "      <td>0.81 (0.72, 0.88)</td>\n",
       "      <td>0.81 (0.75, 0.86)</td>\n",
       "      <td>87 (55.1%)</td>\n",
       "      <td>20 (12.7%)</td>\n",
       "      <td>21 (13.3%)</td>\n",
       "      <td>128 (81.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reader, nonemphysema 100-300mm3</th>\n",
       "      <td>0.79 (0.59, 0.91)</td>\n",
       "      <td>0.92 (0.72, 0.99)</td>\n",
       "      <td>0.85 (0.71, 0.93)</td>\n",
       "      <td>22 (13.9%)</td>\n",
       "      <td>2 (1.3%)</td>\n",
       "      <td>6 (3.8%)</td>\n",
       "      <td>30 (19.0%)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>158 (100%)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     sensitivity (95% CI)       PPV (95% CI)  \\\n",
       "GT by radiologists for discrepancies                                           \n",
       "AI, nonemphysema 30-100mm3              0.65 (0.55, 0.74)  0.93 (0.84, 0.98)   \n",
       "AI, nonemphysema 100-300mm3             0.93 (0.75, 0.99)    0.67 (0.5, 0.8)   \n",
       "                                                                               \n",
       "reader, nonemphysema 30-100mm3          0.81 (0.72, 0.87)  0.81 (0.72, 0.88)   \n",
       "reader, nonemphysema 100-300mm3         0.79 (0.59, 0.91)  0.92 (0.72, 0.99)   \n",
       "                                                                               \n",
       "\n",
       "                                      F1 score (95% CI)  \\\n",
       "GT by radiologists for discrepancies                      \n",
       "AI, nonemphysema 30-100mm3             0.77 (0.7, 0.82)   \n",
       "AI, nonemphysema 100-300mm3           0.78 (0.65, 0.87)   \n",
       "                                                          \n",
       "reader, nonemphysema 30-100mm3        0.81 (0.75, 0.86)   \n",
       "reader, nonemphysema 100-300mm3       0.85 (0.71, 0.93)   \n",
       "                                                          \n",
       "\n",
       "                                     nodules correctly detected  \\\n",
       "GT by radiologists for discrepancies                              \n",
       "AI, nonemphysema 30-100mm3                           70 (45.5%)   \n",
       "AI, nonemphysema 100-300mm3                          26 (16.9%)   \n",
       "                                                                  \n",
       "reader, nonemphysema 30-100mm3                       87 (55.1%)   \n",
       "reader, nonemphysema 100-300mm3                      22 (13.9%)   \n",
       "                                                                  \n",
       "\n",
       "                                     non-nodules incorrectly detected  \\\n",
       "GT by radiologists for discrepancies                                    \n",
       "AI, nonemphysema 30-100mm3                                   5 (3.2%)   \n",
       "AI, nonemphysema 100-300mm3                                 13 (8.4%)   \n",
       "                                                                        \n",
       "reader, nonemphysema 30-100mm3                             20 (12.7%)   \n",
       "reader, nonemphysema 100-300mm3                              2 (1.3%)   \n",
       "                                                                        \n",
       "\n",
       "                                     nodules missed All findings  \n",
       "GT by radiologists for discrepancies                              \n",
       "AI, nonemphysema 30-100mm3               38 (24.7%)  113 (73.4%)  \n",
       "AI, nonemphysema 100-300mm3                2 (1.3%)   41 (26.6%)  \n",
       "                                                      154 (100%)  \n",
       "reader, nonemphysema 30-100mm3           21 (13.3%)  128 (81.0%)  \n",
       "reader, nonemphysema 100-300mm3            6 (3.8%)   30 (19.0%)  \n",
       "                                                      158 (100%)  "
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#For emphysema only comparison between reader and AI for volume subgroups\n",
    "df_all_new=pd.DataFrame(columns=['sensitivity (95% CI)','PPV (95% CI)','F1 score (95% CI)','nodules correctly detected','non-nodules incorrectly detected','nodules missed'], \n",
    "                        index=['AI, nonemphysema 30-100mm3', 'AI, nonemphysema 100-300mm3','',\n",
    "                               'reader, nonemphysema 30-100mm3', 'reader, nonemphysema 100-300mm3',''\n",
    "                              ])\n",
    "\n",
    "df_all_new.index.name = 'GT by radiologists for discrepancies' \n",
    "\n",
    "df_all_new.iloc[0,0]=np.round(sensitivity(TP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,1]=np.round(PPV(TP_AI_100,FP_AI_100),2)\n",
    "df_all_new.iloc[0,2]=np.round(F1score(TP_AI_100,FP_AI_100,FN_AI_100),2)\n",
    "df_all_new.iloc[0,3]=TP_AI_100\n",
    "df_all_new.iloc[0,4]=FP_AI_100\n",
    "df_all_new.iloc[0,5]=FN_AI_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100, FP_AI_100, FN_AI_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[0]=str(df_all_new['sensitivity (95% CI)'].iloc[0])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[0]=str(df_all_new['PPV (95% CI)'].iloc[0])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[0]=str(df_all_new['F1 score (95% CI)'].iloc[0])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "df_all_new.iloc[3,0]=np.round(sensitivity(TP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,1]=np.round(PPV(TP_read_100,FP_read_100),2)\n",
    "df_all_new.iloc[3,2]=np.round(F1score(TP_read_100,FP_read_100,FN_read_100),2)\n",
    "df_all_new.iloc[3,3]=TP_read_100\n",
    "df_all_new.iloc[3,4]=FP_read_100\n",
    "df_all_new.iloc[3,5]=FN_read_100\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100, FP_read_100, FN_read_100, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[3]=str(df_all_new['sensitivity (95% CI)'].iloc[3])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[3]=str(df_all_new['PPV (95% CI)'].iloc[3])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[3]=str(df_all_new['F1 score (95% CI)'].iloc[3])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "\n",
    "df_all_new.iloc[1,0]=np.round(sensitivity(TP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,1]=np.round(PPV(TP_AI_100_300,FP_AI_100_300),2)\n",
    "df_all_new.iloc[1,2]=np.round(F1score(TP_AI_100_300,FP_AI_100_300,FN_AI_100_300),2)\n",
    "df_all_new.iloc[1,3]=TP_AI_100_300\n",
    "df_all_new.iloc[1,4]=FP_AI_100_300\n",
    "df_all_new.iloc[1,5]=FN_AI_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_AI_100_300, FP_AI_100_300, FN_AI_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[1]=str(df_all_new['sensitivity (95% CI)'].iloc[1])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[1]=str(df_all_new['PPV (95% CI)'].iloc[1])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[1]=str(df_all_new['F1 score (95% CI)'].iloc[1])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "\n",
    "df_all_new.iloc[4,0]=np.round(sensitivity(TP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,1]=np.round(PPV(TP_read_100_300,FP_read_100_300),2)\n",
    "df_all_new.iloc[4,2]=np.round(F1score(TP_read_100_300,FP_read_100_300,FN_read_100_300),2)\n",
    "df_all_new.iloc[4,3]=TP_read_100_300\n",
    "df_all_new.iloc[4,4]=FP_read_100_300\n",
    "df_all_new.iloc[4,5]=FN_read_100_300\n",
    "\n",
    "sensitivity_confidence_interval_AI, PPV_confidence_interval_AI, F1_confidence_interval_AI \\\n",
    "= sensitivity_and_specificity_with_confidence_intervals(TP_read_100_300, FP_read_100_300, FN_read_100_300, 0, alpha=0.95)\n",
    "\n",
    "ci_sens_ai=[np.round(x,2) for x in sensitivity_confidence_interval_AI]\n",
    "ci_ppv_ai=[np.round(x,2) for x in PPV_confidence_interval_AI]\n",
    "ci_f1_ai=[np.round(x,2) for x in F1_confidence_interval_AI]\n",
    "\n",
    "df_all_new['sensitivity (95% CI)'].iloc[4]=str(df_all_new['sensitivity (95% CI)'].iloc[4])+' '+str(tuple(ci_sens_ai))\n",
    "df_all_new['PPV (95% CI)'].iloc[4]=str(df_all_new['PPV (95% CI)'].iloc[4])+' '+str(tuple(ci_ppv_ai))\n",
    "df_all_new['F1 score (95% CI)'].iloc[4]=str(df_all_new['F1 score (95% CI)'].iloc[4])+' '+str(tuple(ci_f1_ai))\n",
    "\n",
    "\n",
    "\n",
    "df_all_new.iloc[2,0]=0\n",
    "df_all_new.iloc[2,1]=0\n",
    "df_all_new.iloc[2,2]=0\n",
    "df_all_new.iloc[2,3]=0\n",
    "df_all_new.iloc[2,4]=0\n",
    "df_all_new.iloc[2,5]=0\n",
    "\n",
    "AI_all=np.sum(df_all_new.iloc[0:2,3:].values)\n",
    "reader_all=np.sum(df_all_new.iloc[3:5,3:].values)\n",
    "\n",
    "df_all_new['All findings']=df_all_new['nodules correctly detected']+df_all_new['non-nodules incorrectly detected']+df_all_new['nodules missed']\n",
    "\n",
    "df_all_new.iloc[5,0]=''\n",
    "df_all_new.iloc[5,1]=''\n",
    "df_all_new.iloc[5,2]=''\n",
    "df_all_new.iloc[5,3]=''\n",
    "df_all_new.iloc[5,4]=''\n",
    "df_all_new.iloc[5,5]=''\n",
    "df_all_new.iloc[5,6]=np.sum(df_all_new['All findings'].iloc[3:5])\n",
    "\n",
    "df_all_new.iloc[2,0]=''\n",
    "df_all_new.iloc[2,1]=''\n",
    "df_all_new.iloc[2,2]=''\n",
    "df_all_new.iloc[2,3]=''\n",
    "df_all_new.iloc[2,4]=''\n",
    "df_all_new.iloc[2,5]=''\n",
    "df_all_new.iloc[2,6]=np.sum(df_all_new['All findings'].iloc[0:2])\n",
    "\n",
    "# print(df_all_new['sensitivity (95% CI)'])\n",
    "for i in range(5):\n",
    "    if i!=2:\n",
    "        if i<2:\n",
    "            sum_all=AI_all\n",
    "        elif i>2:\n",
    "            sum_all=reader_all\n",
    "            \n",
    "        percentage_tp=np.round((df_all_new.iloc[i][3]/sum_all)*100,1) \n",
    "        df_all_new['nodules correctly detected'].iloc[i]=str(df_all_new.iloc[i][3])+' ('+str(percentage_tp)+'%)'\n",
    "\n",
    "        percentage_fp=np.round((df_all_new.iloc[i][4]/sum_all)*100,1) \n",
    "        df_all_new['non-nodules incorrectly detected'].iloc[i]=str(df_all_new.iloc[i][4])+' ('+str(percentage_fp)+'%)'\n",
    "\n",
    "        percentage_fn=np.round((df_all_new.iloc[i][5]/sum_all)*100,1) \n",
    "        df_all_new['nodules missed'].iloc[i]=str(df_all_new.iloc[i][5])+' ('+str(percentage_fn)+'%)'\n",
    "\n",
    "        df_all_new['All findings'].iloc[i]=str(df_all_new.iloc[i][6])+' ('+str(np.round(100*df_all_new.iloc[i][6]/sum_all,1))+'%)'\n",
    "\n",
    "    \n",
    "df_all_new['All findings'].iloc[2]=str(df_all_new.iloc[2][6])+' (100%)'\n",
    "df_all_new['All findings'].iloc[5]=str(df_all_new.iloc[5][6])+' (100%)'\n",
    "\n",
    "df_all_new #Detection performance comparison for nodules and lymph nodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "12eb4780",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all_new.to_excel('nodules_lymph_volumes_nonemphysema.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "d4678b1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For nodules and lymph nodes (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is 0.03724916580683402\n",
      "For nodules and lymph nodes (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is 0.2888443663464818\n"
     ]
    }
   ],
   "source": [
    "data=[[TP_both_100, FN_read_100],\n",
    "        [FN_AI_100,0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules and lymph nodes (AI vs reader) of 30-100mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "# #For FPs\n",
    "# data=[[0, FP_AI_100], \n",
    "#         [FP_read_100, 0]]\n",
    "# # print(data)\n",
    "\n",
    "# # McNemar's Test without continuity correction\n",
    "# print(\"For FP findings of 30-100mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "# print(\"\\n\")\n",
    "\n",
    "\n",
    "\n",
    "data=[[TP_both_100_300,FN_read_100_300 ], \n",
    "        [FN_AI_100_300, 0]]\n",
    "# print(data)\n",
    "\n",
    "# McNemar's Test without continuity correction\n",
    "print(\"For nodules and lymph nodes (AI vs reader) of 100-300mm3 with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)\n",
    "\n",
    "# #For FPs\n",
    "# data=[[0, FP_AI_100_300], \n",
    "#         [FP_read_100_300, 0]]\n",
    "# # print(data)\n",
    "\n",
    "# # McNemar's Test without continuity correction\n",
    "# print(\"For FP findings of 100-300mm3, with continuity correction (not exact) p value is\",mcnemar(data, exact=False,correction=True).pvalue)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "716b6ab0",
   "metadata": {},
   "source": [
    "#### Mann-Whitney U test to check for differences in volumes between low/high BMI within each volume subgroup (not used)\n",
    "\n",
    "This is an unpaired test meaning that we consider each nodule as separate from others. It can be used with unequal sample sizes as well.\n",
    "Bland-Altman is not a good choice since it can be performed on nodules detected by both AI and reader to assess for agreement in the volume measurement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "99a325a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Perform the Mann-Whitney U test for nodules only\n",
    "# # stats.mannwhitneyu(group1, group2, alternative='two-sided')\n",
    "\n",
    "# #Compare all non-emphysema reader vs AI volumes\n",
    "# print('non emphysema reader vs AI volumes p value is', stats.mannwhitneyu(reader_only_nods_noemph_30_100_vols+reader_only_nods_noemph_100_300_vols+reader_only_nods_noemph_300_vols,\n",
    "#                    ai_only_nods_noemph_30_100_vols+ai_only_nods_noemph_100_300_vols+ai_only_nods_noemph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema reader vs AI volumes\n",
    "# print('emphysema reader vs AI volumes p value is',stats.mannwhitneyu(reader_only_nods_emph_30_100_vols+reader_only_nods_emph_100_300_vols+reader_only_nods_emph_300_vols,\n",
    "#                    ai_only_nods_emph_30_100_vols+ai_only_nods_emph_100_300_vols+ai_only_nods_emph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema vs non-emphysema volumes for nodules, for reader only\n",
    "# print('emphysema vs non-emphysema for reader p value is',stats.mannwhitneyu(reader_only_nods_emph_30_100_vols+reader_only_nods_emph_100_300_vols+reader_only_nods_emph_300_vols,\n",
    "#                    reader_only_nods_noemph_30_100_vols+reader_only_nods_noemph_100_300_vols+reader_only_nods_noemph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema vs non-emphysema volumes for nodules, for AI only\n",
    "# print('emphysema vs non-emphysema for AI p value is',stats.mannwhitneyu(ai_only_nods_noemph_30_100_vols+ai_only_nods_noemph_100_300_vols+ai_only_nods_noemph_300_vols,\n",
    "#                    ai_only_nods_emph_30_100_vols+ai_only_nods_emph_100_300_vols+ai_only_nods_emph_300_vols).pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "3b75ba3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Similarly as above for non-nodule findings\n",
    "\n",
    "# #Compare all non-emphysema reader vs AI volumes for non-nodule findings\n",
    "# print('non-emphysema reader vs AI volumes for non-nodule findings p value is',stats.mannwhitneyu(reader_nonods_noemph_30_100_vols+reader_nonods_noemph_100_300_vols+reader_nonods_noemph_300_vols,\n",
    "#                    ai_nonods_noemph_30_100_vols+ai_nonods_noemph_100_300_vols+ai_nonods_noemph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema reader vs AI volumes for non-nodule findings\n",
    "# print('emphysema reader vs AI volumes for non-nodule findings p value is',stats.mannwhitneyu(reader_nonods_emph_30_100_vols+reader_nonods_emph_100_300_vols+reader_nonods_emph_300_vols,\n",
    "#                    ai_nonods_emph_30_100_vols+ai_nonods_emph_100_300_vols+ai_nonods_emph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema vs non-emphysema volumes for reader only, for non-nodule findings\n",
    "# print('emphysema vs non-emphysema volumes for reader only, for non-nodule findings p value is',stats.mannwhitneyu(reader_nonods_emph_30_100_vols+reader_nonods_emph_100_300_vols+reader_nonods_emph_300_vols,\n",
    "#                    reader_nonods_noemph_30_100_vols+reader_nonods_noemph_100_300_vols+reader_nonods_noemph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema vs non-emphysema volumes for AI only, for non-nodule findings\n",
    "# print('emphysema vs non-emphysema volumes for AI only, for non-nodule findings p value is',stats.mannwhitneyu(ai_nonods_noemph_30_100_vols+ai_nonods_noemph_100_300_vols+ai_nonods_noemph_300_vols,\n",
    "#                    ai_nonods_emph_30_100_vols+ai_nonods_emph_100_300_vols+ai_nonods_emph_300_vols).pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "d428aa81",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Similarly as above for nodules and lymph nodes\n",
    "\n",
    "# #Compare all non-emphysema reader vs AI volumes for nodules and lymph nodes\n",
    "# print('non-emphysema reader vs AI volumes for nodules and lymph nodes p value is',stats.mannwhitneyu(reader_only_nods_noemph_30_100_vols+reader_only_nods_noemph_100_300_vols+reader_only_nods_noemph_300_vols+\n",
    "#                    reader_lymph_noemph_30_100_vols+ reader_lymph_noemph_100_300_vols+ reader_lymph_noemph_300_vols,\n",
    "#                    ai_only_nods_noemph_30_100_vols+ai_only_nods_noemph_100_300_vols+ai_only_nods_noemph_300_vols+\n",
    "#                   ai_lymph_noemph_30_100_vols+ ai_lymph_noemph_100_300_vols+ ai_lymph_noemph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema reader vs AI volumes for nodules and lymph nodes\n",
    "# print('emphysema reader vs AI volumes for nodules and lymph nodes p value is',stats.mannwhitneyu(reader_only_nods_emph_30_100_vols+reader_only_nods_emph_100_300_vols+reader_only_nods_emph_300_vols+\n",
    "#                    reader_lymph_emph_30_100_vols+ reader_lymph_emph_100_300_vols+ reader_lymph_emph_300_vols,\n",
    "#                    ai_only_nods_emph_30_100_vols+ai_only_nods_emph_100_300_vols+ai_only_nods_emph_300_vols+\n",
    "#                   ai_lymph_emph_30_100_vols+ ai_lymph_emph_100_300_vols+ ai_lymph_emph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema vs non-emphysema volumes for reader only for nodules and lymph nodes\n",
    "# print('emphysema vs non-emphysema volumes for reader only for nodules and lymph nodes p value is',stats.mannwhitneyu(reader_only_nods_emph_30_100_vols+reader_only_nods_emph_100_300_vols+reader_only_nods_emph_300_vols+\n",
    "#                    reader_lymph_emph_30_100_vols+ reader_lymph_emph_100_300_vols+ reader_lymph_emph_300_vols,\n",
    "#                    reader_only_nods_noemph_30_100_vols+reader_only_nods_noemph_100_300_vols+reader_only_nods_noemph_300_vols+\n",
    "#                   reader_lymph_noemph_30_100_vols+ reader_lymph_noemph_100_300_vols+ reader_lymph_noemph_300_vols).pvalue)\n",
    "\n",
    "# #Compare all emphysema vs non-emphysema volumes for AI only for nodules and lymph nodes\n",
    "# print('emphysema vs non-emphysema volumes for AI only for nodules and lymph nodes p value is',stats.mannwhitneyu(ai_only_nods_noemph_30_100_vols+ai_only_nods_noemph_100_300_vols+ai_only_nods_noemph_300_vols+\n",
    "#                    ai_lymph_noemph_30_100_vols+ ai_lymph_noemph_100_300_vols+ ai_lymph_noemph_300_vols,\n",
    "#                    ai_only_nods_emph_30_100_vols+ai_only_nods_emph_100_300_vols+ai_only_nods_emph_300_vols+\n",
    "#                   ai_lymph_emph_30_100_vols+ ai_lymph_emph_100_300_vols+ ai_lymph_emph_300_vols).pvalue)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}