1_Generate HAIM-MIMIC-MM from downloaded MIMIC-IV and MIMIC-CXR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code to Generate the HAIM-MIMIC-MM multimodal dataset in picke file-format from MIMIC-IV and MIMIC-CXR\n",
    "\n",
    "### Project Info\n",
    " ->Copyright 2020 (Last Update: June 07, 2022)\n",
    " \n",
    " -> Authors: \n",
    "        Luis R Soenksen (<soenksen@mit.edu>),\n",
    "        Yu Ma (<midsumer@mit.edu>),\n",
    "        Cynthia Zeng (<czeng12@mit.edu>),\n",
    "        Ignacio Fuentes (<ifuentes@mit.edu>),\n",
    "        Leonard David Jean Boussioux (<leobix@mit.edu>),\n",
    "        Agni Orfanoudaki (<agniorf@mit.edu>),\n",
    "        Holly Mika Wiberg (<hwiberg@mit.edu>),\n",
    "        Michael Lingzhi Li (<mlli@mit.edu>),\n",
    "        Kimberly M Villalobos Carballo (<kimvc@mit.edu>),\n",
    "        Liangyuan Na (<lyna@mit.edu>),\n",
    "        Dimitris J Bertsimas (<dbertsim@mit.edu>),\n",
    "\n",
    "```\n",
    "**Licensed under the Apache License, Version 2.0**\n",
    "You may not use this file except in compliance with the License. You may obtain a copy of the License at\n",
    "https://www.apache.org/licenses/LICENSE-2.0\n",
    "Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Requires \n",
    "```\n",
    " -> At least 20Gb of available RAM\n",
    " -> Downloaded version of MIMIC-IV 1.0 from credentialed access (https://physionet.org/content/mimiciv/1.0/) in folder structure [data/HAIM/physionet/files/mimiciv/1.0/]\n",
    " -> Downloaded version of MIMIC-CXR-JPG 2.0.0 from credentialed access (https://physionet.org/content/mimic-cxr-jpg/2.0.0/) in folder structure [data/HAIM/physionet/files/mimiciv/1.0/mimic-cxr-jpg/2.0.0/] \n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### -> Library Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#HAIM\n",
    "import sys\n",
    "from MIMIC_IV_HAIM_API import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Display optiona\n",
    "from IPython.display import Image # IPython display\n",
    "pd.set_option('display.max_rows', None, 'display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.set_option('float_format', '{:f}'.format)\n",
    "pd.options.mode.chained_assignment = None  # default='warn'\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### -> Initializations & Data Loading\n",
    "Resources to identify tables and variables of interest can be found in the MIMIC-IV official API (https://mimic-iv.mit.edu/docs/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define MIMIC IV Data Location\n",
    "core_mimiciv_path = 'data/HAIM/physionet/files/mimiciv/1.0/'\n",
    "\n",
    "# Define MIMIC IV Image Data Location (usually external drive)\n",
    "core_mimiciv_imgcxr_path = 'data/HAIM/physionet/files/mimiciv/1.0/mimic-cxr-jpg/2.0.0/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "## CORE\n",
    "df_admissions = dd.read_csv(core_mimiciv_path + 'core/admissions.csv', assume_missing=True, dtype={'admission_location': 'object','deathtime': 'object','edouttime': 'object','edregtime': 'object'})\n",
    "df_patients = dd.read_csv(core_mimiciv_path + 'core/patients.csv', assume_missing=True, dtype={'dod': 'object'})  \n",
    "df_transfers = dd.read_csv(core_mimiciv_path + 'core/transfers.csv', assume_missing=True, dtype={'careunit': 'object'})\n",
    "\n",
    "## HOSP\n",
    "df_d_labitems = dd.read_csv(core_mimiciv_path + 'hosp/d_labitems.csv', assume_missing=True, dtype={'loinc_code': 'object'})\n",
    "df_d_icd_procedures = dd.read_csv(core_mimiciv_path + 'hosp/d_icd_procedures.csv', assume_missing=True, dtype={'icd_code': 'object', 'icd_version': 'object'})\n",
    "df_d_icd_diagnoses = dd.read_csv(core_mimiciv_path + 'hosp/d_icd_diagnoses.csv', assume_missing=True, dtype={'icd_code': 'object', 'icd_version': 'object'})\n",
    "df_d_hcpcs = dd.read_csv(core_mimiciv_path + 'hosp/d_hcpcs.csv', assume_missing=True, dtype={'category': 'object'})\n",
    "df_diagnoses_icd = dd.read_csv(core_mimiciv_path + 'hosp/diagnoses_icd.csv', assume_missing=True, dtype={'icd_code': 'object', 'icd_version': 'object'})\n",
    "df_drgcodes = dd.read_csv(core_mimiciv_path + 'hosp/drgcodes.csv', assume_missing=True)\n",
    "df_emar = dd.read_csv(core_mimiciv_path + 'hosp/emar.csv', assume_missing=True)\n",
    "df_emar_detail = dd.read_csv(core_mimiciv_path + 'hosp/emar_detail.csv', assume_missing=True, low_memory=False, dtype={'completion_interval': 'object','dose_due': 'object','dose_given': 'object','infusion_complete': 'object','infusion_rate_adjustment': 'object','infusion_rate_unit': 'object','new_iv_bag_hung': 'object','product_description_other': 'object','reason_for_no_barcode': 'object','restart_interval': 'object','route': 'object','side': 'object','site': 'object','continued_infusion_in_other_location': 'object','infusion_rate': 'object','non_formulary_visual_verification': 'object','prior_infusion_rate': 'object','product_amount_given': 'object', 'infusion_rate_adjustment_amount': 'object'})\n",
    "df_hcpcsevents = dd.read_csv(core_mimiciv_path + 'hosp/hcpcsevents.csv', assume_missing=True, dtype={'hcpcs_cd': 'object'})\n",
    "df_labevents = dd.read_csv(core_mimiciv_path + 'hosp/labevents.csv', assume_missing=True, dtype={'storetime': 'object', 'value': 'object', 'valueuom': 'object', 'flag': 'object', 'priority': 'object', 'comments': 'object'})\n",
    "df_microbiologyevents = dd.read_csv(core_mimiciv_path + 'hosp/microbiologyevents.csv', assume_missing=True, dtype={'comments': 'object', 'quantity': 'object'})\n",
    "df_poe = dd.read_csv(core_mimiciv_path + 'hosp/poe.csv', assume_missing=True, dtype={'discontinue_of_poe_id': 'object','discontinued_by_poe_id': 'object','order_status': 'object'})\n",
    "df_poe_detail = dd.read_csv(core_mimiciv_path + 'hosp/poe_detail.csv', assume_missing=True)\n",
    "df_prescriptions = dd.read_csv(core_mimiciv_path + 'hosp/prescriptions.csv', assume_missing=True, dtype={'form_rx': 'object','gsn': 'object'})\n",
    "df_procedures_icd = dd.read_csv(core_mimiciv_path + 'hosp/procedures_icd.csv', assume_missing=True, dtype={'icd_code': 'object', 'icd_version': 'object'})\n",
    "df_services = dd.read_csv(core_mimiciv_path + 'hosp/services.csv', assume_missing=True, dtype={'prev_service': 'object'})\n",
    "\n",
    "## ICU\n",
    "df_d_items = dd.read_csv(core_mimiciv_path + 'icu/d_items.csv', assume_missing=True)\n",
    "df_procedureevents = dd.read_csv(core_mimiciv_path + 'icu/procedureevents.csv', assume_missing=True, dtype={'value': 'object', 'secondaryordercategoryname': 'object', 'totalamountuom': 'object'})\n",
    "df_outputevents = dd.read_csv(core_mimiciv_path + 'icu/outputevents.csv', assume_missing=True, dtype={'value': 'object'})\n",
    "df_inputevents = dd.read_csv(core_mimiciv_path + 'icu/inputevents.csv', assume_missing=True, dtype={'value': 'object', 'secondaryordercategoryname': 'object', 'totalamountuom': 'object'})\n",
    "df_icustays = dd.read_csv(core_mimiciv_path + 'icu/icustays.csv', assume_missing=True)\n",
    "df_datetimeevents = dd.read_csv(core_mimiciv_path + 'icu/datetimeevents.csv', assume_missing=True, dtype={'value': 'object'})\n",
    "df_chartevents = dd.read_csv(core_mimiciv_path + 'icu/chartevents.csv', assume_missing=True, low_memory=False, dtype={'value': 'object', 'valueuom': 'object'})\n",
    "\n",
    "## CXR\n",
    "df_mimic_cxr_split = dd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-split.csv', assume_missing=True)\n",
    "df_mimic_cxr_chexpert = dd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-chexpert.csv', assume_missing=True)\n",
    "try:\n",
    "    df_mimic_cxr_metadata = dd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-metadata.csv', assume_missing=True, dtype={'dicom_id': 'object'}, blocksize=None)\n",
    "except:\n",
    "    df_mimic_cxr_metadata = pd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-metadata.csv', dtype={'dicom_id': 'object'})\n",
    "    df_mimic_cxr_metadata = dd.from_pandas(df_mimic_cxr_metadata, npartitions=7)\n",
    "df_mimic_cxr_negbio = dd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-negbio.csv', assume_missing=True)\n",
    "\n",
    "## NOTES\n",
    "df_noteevents = dd.from_pandas(pd.read_csv(core_mimiciv_path + 'note/noteevents.csv', dtype={'charttime': 'object', 'storetime': 'object', 'text': 'object'}), chunksize=8)\n",
    "df_dsnotes = dd.from_pandas(pd.read_csv(core_mimiciv_path + 'note/ds_icustay.csv', dtype={'charttime': 'object', 'storetime': 'object', 'text': 'object'}), chunksize=8)\n",
    "df_ecgnotes = dd.from_pandas(pd.read_csv(core_mimiciv_path + 'note/ecg_icustay.csv', dtype={'charttime': 'object', 'storetime': 'object', 'text': 'object'}), chunksize=8)\n",
    "df_echonotes = dd.from_pandas(pd.read_csv(core_mimiciv_path + 'note/echo_icustay.csv', dtype={'charttime': 'object', 'storetime': 'object', 'text': 'object'}), chunksize=8)\n",
    "df_radnotes = dd.from_pandas(pd.read_csv(core_mimiciv_path + 'note/rad_icustay.csv', dtype={'charttime': 'object', 'storetime': 'object', 'text': 'object'}), chunksize=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### -> Data Preparation\n",
    "#### Create full database in dask format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Fix data type issues to allow for merging\n",
    "\n",
    "## CORE\n",
    "df_admissions['admittime'] = dd.to_datetime(df_admissions['admittime'])\n",
    "df_admissions['dischtime'] = dd.to_datetime(df_admissions['dischtime'])\n",
    "df_admissions['deathtime'] = dd.to_datetime(df_admissions['deathtime'])\n",
    "df_admissions['edregtime'] = dd.to_datetime(df_admissions['edregtime'])\n",
    "df_admissions['edouttime'] = dd.to_datetime(df_admissions['edouttime'])\n",
    "\n",
    "df_transfers['intime'] = dd.to_datetime(df_transfers['intime'])\n",
    "df_transfers['outtime'] = dd.to_datetime(df_transfers['outtime'])\n",
    "\n",
    "\n",
    "## HOSP\n",
    "df_diagnoses_icd.icd_code = df_diagnoses_icd.icd_code.str.strip()\n",
    "df_diagnoses_icd.icd_version = df_diagnoses_icd.icd_version.str.strip()\n",
    "df_d_icd_diagnoses.icd_code = df_d_icd_diagnoses.icd_code.str.strip()\n",
    "df_d_icd_diagnoses.icd_version = df_d_icd_diagnoses.icd_version.str.strip()\n",
    "\n",
    "df_procedures_icd.icd_code = df_procedures_icd.icd_code.str.strip()\n",
    "df_procedures_icd.icd_version = df_procedures_icd.icd_version.str.strip()\n",
    "df_d_icd_procedures.icd_code = df_d_icd_procedures.icd_code.str.strip()\n",
    "df_d_icd_procedures.icd_version = df_d_icd_procedures.icd_version.str.strip()\n",
    "\n",
    "df_hcpcsevents.hcpcs_cd = df_hcpcsevents.hcpcs_cd.str.strip()\n",
    "df_d_hcpcs.code = df_d_hcpcs.code.str.strip()\n",
    "\n",
    "df_prescriptions['starttime'] = dd.to_datetime(df_prescriptions['starttime'])\n",
    "df_prescriptions['stoptime'] = dd.to_datetime(df_prescriptions['stoptime'])\n",
    "\n",
    "df_emar['charttime'] = dd.to_datetime(df_emar['charttime'])\n",
    "df_emar['scheduletime'] = dd.to_datetime(df_emar['scheduletime'])\n",
    "df_emar['storetime'] = dd.to_datetime(df_emar['storetime'])\n",
    "\n",
    "df_labevents['charttime'] = dd.to_datetime(df_labevents['charttime'])\n",
    "df_labevents['storetime'] = dd.to_datetime(df_labevents['storetime'])\n",
    "\n",
    "df_microbiologyevents['chartdate'] = dd.to_datetime(df_microbiologyevents['chartdate'])\n",
    "df_microbiologyevents['charttime'] = dd.to_datetime(df_microbiologyevents['charttime'])\n",
    "df_microbiologyevents['storedate'] = dd.to_datetime(df_microbiologyevents['storedate'])\n",
    "df_microbiologyevents['storetime'] = dd.to_datetime(df_microbiologyevents['storetime'])\n",
    "\n",
    "df_poe['ordertime'] = dd.to_datetime(df_poe['ordertime'])\n",
    "df_services['transfertime'] = dd.to_datetime(df_services['transfertime'])\n",
    "\n",
    "\n",
    "## ICU\n",
    "df_procedureevents['starttime'] = dd.to_datetime(df_procedureevents['starttime'])\n",
    "df_procedureevents['endtime'] = dd.to_datetime(df_procedureevents['endtime'])\n",
    "df_procedureevents['storetime'] = dd.to_datetime(df_procedureevents['storetime'])\n",
    "df_procedureevents['comments_date'] = dd.to_datetime(df_procedureevents['comments_date'])\n",
    "\n",
    "df_outputevents['charttime'] = dd.to_datetime(df_outputevents['charttime'])\n",
    "df_outputevents['storetime'] = dd.to_datetime(df_outputevents['storetime'])\n",
    "\n",
    "df_inputevents['starttime'] = dd.to_datetime(df_inputevents['starttime'])\n",
    "df_inputevents['endtime'] = dd.to_datetime(df_inputevents['endtime'])\n",
    "df_inputevents['storetime'] = dd.to_datetime(df_inputevents['storetime'])\n",
    "\n",
    "df_icustays['intime'] = dd.to_datetime(df_icustays['intime'])\n",
    "df_icustays['outtime'] = dd.to_datetime(df_icustays['outtime'])\n",
    "\n",
    "df_datetimeevents['charttime'] = dd.to_datetime(df_datetimeevents['charttime'])\n",
    "df_datetimeevents['storetime'] = dd.to_datetime(df_datetimeevents['storetime'])\n",
    "\n",
    "df_chartevents['charttime'] = dd.to_datetime(df_chartevents['charttime'])\n",
    "df_chartevents['storetime'] = dd.to_datetime(df_chartevents['storetime'])\n",
    "\n",
    "\n",
    "## CXR\n",
    "if (not 'cxrtime' in df_mimic_cxr_metadata.columns) or (not 'Img_Filename' in df_mimic_cxr_metadata.columns):\n",
    "    # Create CXRTime variable if it does not exist already\n",
    "    print(\"Processing CXRtime stamps\")\n",
    "    df_cxr = df_mimic_cxr_metadata.compute()\n",
    "    df_cxr['StudyDateForm'] = pd.to_datetime(df_cxr['StudyDate'], format='%Y%m%d')\n",
    "    df_cxr['StudyTimeForm'] = df_cxr.apply(lambda x : '%#010.3f' % x['StudyTime'] ,1)\n",
    "    df_cxr['StudyTimeForm'] = pd.to_datetime(df_cxr['StudyTimeForm'], format='%H%M%S.%f').dt.time\n",
    "    df_cxr['cxrtime'] = df_cxr.apply(lambda r : dt.datetime.combine(r['StudyDateForm'],r['StudyTimeForm']),1)\n",
    "    # Add paths and info to images in cxr\n",
    "    df_mimic_cxr_jpg =pd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-jpeg-txt.csv')\n",
    "    df_cxr = pd.merge(df_mimic_cxr_jpg, df_cxr, on='dicom_id')\n",
    "    # Save\n",
    "    df_cxr.to_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-metadata.csv', index=False)\n",
    "    #Read back the dataframe\n",
    "    try:\n",
    "        df_mimic_cxr_metadata = dd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-metadata.csv', assume_missing=True, dtype={'dicom_id': 'object', 'Note': 'object'}, blocksize=None)\n",
    "    except:\n",
    "        df_mimic_cxr_metadata = pd.read_csv(core_mimiciv_path + 'mimic-cxr-jpg/2.0.0/mimic-cxr-2.0.0-metadata.csv', dtype={'dicom_id': 'object', 'Note': 'object'})\n",
    "        df_mimic_cxr_metadata = dd.from_pandas(df_mimic_cxr_metadata, npartitions=7)\n",
    "df_mimic_cxr_metadata['cxrtime'] = dd.to_datetime(df_mimic_cxr_metadata['cxrtime'])\n",
    "\n",
    "\n",
    "## NOTES\n",
    "df_noteevents['chartdate'] = dd.to_datetime(df_noteevents['chartdate'])\n",
    "df_noteevents['charttime'] = dd.to_datetime(df_noteevents['charttime'])\n",
    "df_noteevents['storetime'] = dd.to_datetime(df_noteevents['storetime'])\n",
    "\n",
    "df_dsnotes['charttime'] = dd.to_datetime(df_dsnotes['charttime'])\n",
    "df_dsnotes['storetime'] = dd.to_datetime(df_dsnotes['storetime'])\n",
    "\n",
    "df_ecgnotes['charttime'] = dd.to_datetime(df_ecgnotes['charttime'])\n",
    "df_ecgnotes['storetime'] = dd.to_datetime(df_ecgnotes['storetime'])\n",
    "\n",
    "df_echonotes['charttime'] = dd.to_datetime(df_echonotes['charttime'])\n",
    "df_echonotes['storetime'] = dd.to_datetime(df_echonotes['storetime'])\n",
    "\n",
    "df_radnotes['charttime'] = dd.to_datetime(df_radnotes['charttime'])\n",
    "df_radnotes['storetime'] = dd.to_datetime(df_radnotes['storetime'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROCESSING \"CORE\" DB...\n",
      "[########################################] | 100% Completed |  2.2s\n",
      "[########################################] | 100% Completed |  0.2s\n",
      "[########################################] | 100% Completed |  3.2s\n",
      "PROCESSING \"HOSP\" DB...\n",
      "[########################################] | 100% Completed |  3.4s\n",
      "[########################################] | 100% Completed |  0.5s\n",
      "[########################################] | 100% Completed |  1min 14.5s\n",
      "[########################################] | 100% Completed |  2min 21.4s\n",
      "[########################################] | 100% Completed |  0.5s\n",
      "[########################################] | 100% Completed |  2min 23.9s\n",
      "[########################################] | 100% Completed | 13.6s\n",
      "[########################################] | 100% Completed | 57.7s\n",
      "[########################################] | 100% Completed |  1.9s\n",
      "[########################################] | 100% Completed | 32.3s\n",
      "[########################################] | 100% Completed |  0.8s\n",
      "[########################################] | 100% Completed |  0.6s\n",
      "[########################################] | 100% Completed |  0.3s\n",
      "[########################################] | 100% Completed |  0.2s\n",
      "[########################################] | 100% Completed |  0.1s\n",
      "[########################################] | 100% Completed |  0.1s\n",
      "PROCESSING \"ICU\" DB...\n",
      "[########################################] | 100% Completed |  2.8s\n",
      "[########################################] | 100% Completed |  5.0s\n",
      "[########################################] | 100% Completed | 34.5s\n",
      "[########################################] | 100% Completed |  0.2s\n",
      "[########################################] | 100% Completed |  8.1s\n",
      "[########################################] | 100% Completed |  4min 52.4s\n",
      "[########################################] | 100% Completed |  0.2s\n",
      "PROCESSING \"CXR\" DB...\n",
      "[########################################] | 100% Completed |  1.1s\n",
      "[########################################] | 100% Completed |  0.4s\n",
      "[########################################] | 100% Completed |  5.5s\n",
      "[########################################] | 100% Completed |  0.2s\n",
      "PROCESSING \"NOTES\" DB...\n",
      "[########################################] | 100% Completed | 13min 13.4s\n",
      "[########################################] | 100% Completed | 21.0s\n",
      "[########################################] | 100% Completed | 48.8s\n",
      "[########################################] | 100% Completed | 26.4s\n",
      "[########################################] | 100% Completed |  3min 23.9s\n"
     ]
    }
   ],
   "source": [
    "# -> SORT data\n",
    "## CORE\n",
    "print('PROCESSING \"CORE\" DB...')\n",
    "df_admissions = df_admissions.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_patients = df_patients.compute().sort_values(by=['subject_id'])\n",
    "df_transfers = df_transfers.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "\n",
    "\n",
    "## HOSP\n",
    "print('PROCESSING \"HOSP\" DB...')\n",
    "df_diagnoses_icd = df_diagnoses_icd.compute().sort_values(by=['subject_id'])\n",
    "df_drgcodes = df_drgcodes.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_emar = df_emar.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_emar_detail = df_emar_detail.compute().sort_values(by=['subject_id'])\n",
    "df_hcpcsevents = df_hcpcsevents.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_labevents = df_labevents.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_microbiologyevents = df_microbiologyevents.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_poe = df_poe.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_poe_detail = df_poe_detail.compute().sort_values(by=['subject_id'])\n",
    "df_prescriptions = df_prescriptions.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_procedures_icd = df_procedures_icd.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_services = df_services.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "#--> Unwrap dictionaries\n",
    "df_d_icd_diagnoses = df_d_icd_diagnoses.compute()\n",
    "df_d_icd_procedures = df_d_icd_procedures.compute()\n",
    "df_d_hcpcs = df_d_hcpcs.compute()\n",
    "df_d_labitems = df_d_labitems.compute()\n",
    "\n",
    "\n",
    "## ICU\n",
    "print('PROCESSING \"ICU\" DB...')\n",
    "df_procedureevents = df_procedureevents.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_outputevents = df_outputevents.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_inputevents = df_inputevents.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_icustays = df_icustays.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_datetimeevents = df_datetimeevents.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_chartevents = df_chartevents.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "#--> Unwrap dictionaries\n",
    "df_d_items = df_d_items.compute()\n",
    "\n",
    "\n",
    "## CXR\n",
    "print('PROCESSING \"CXR\" DB...')\n",
    "df_mimic_cxr_split = df_mimic_cxr_split.compute().sort_values(by=['subject_id'])\n",
    "df_mimic_cxr_chexpert = df_mimic_cxr_chexpert.compute().sort_values(by=['subject_id'])\n",
    "df_mimic_cxr_metadata = df_mimic_cxr_metadata.compute().sort_values(by=['subject_id'])\n",
    "df_mimic_cxr_negbio = df_mimic_cxr_negbio.compute().sort_values(by=['subject_id'])\n",
    "\n",
    "\n",
    "## NOTES\n",
    "print('PROCESSING \"NOTES\" DB...')\n",
    "df_noteevents = df_noteevents.compute().sort_values(by=['subject_id','hadm_id'])\n",
    "df_dsnotes = df_dsnotes.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_ecgnotes = df_ecgnotes.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_echonotes = df_echonotes.compute().sort_values(by=['subject_id','hadm_id','stay_id'])\n",
    "df_radnotes = df_radnotes.compute().sort_values(by=['subject_id','hadm_id','stay_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -> MASTER DICTIONARY of health items\n",
    "# Generate dictionary for chartevents, labevents and HCPCS\n",
    "df_patientevents_categorylabels_dict = pd.DataFrame(columns = ['eventtype', 'category', 'label'])\n",
    "\n",
    "# Get Chartevent items with labels & category\n",
    "df = df_d_items\n",
    "for category_idx, category in enumerate(sorted((df.category.astype(str).unique()))):\n",
    "    #print(category)\n",
    "    category_list = df[df['category']==category]\n",
    "    for item_idx, item in enumerate(sorted(category_list.label.astype(str).unique())):\n",
    "        df_patientevents_categorylabels_dict = df_patientevents_categorylabels_dict.append({'eventtype': 'chart', 'category': category, 'label': item}, ignore_index=True)\n",
    "\n",
    "# Get Lab items with labels & category\n",
    "df = df_d_labitems\n",
    "for category_idx, category in enumerate(sorted((df.category.astype(str).unique()))):\n",
    "    #print(category)\n",
    "    category_list = df[df['category']==category]\n",
    "    for item_idx, item in enumerate(sorted(category_list.label.astype(str).unique())):\n",
    "        df_patientevents_categorylabels_dict = df_patientevents_categorylabels_dict.append({'eventtype': 'lab', 'category': category, 'label': item}, ignore_index=True)\n",
    "        \n",
    "# Get HCPCS items with labels & category\n",
    "df = df_d_hcpcs\n",
    "for category_idx, category in enumerate(sorted((df.category.astype(str).unique()))):\n",
    "    #print(category)\n",
    "    category_list = df[df['category']==category]\n",
    "    for item_idx, item in enumerate(sorted(category_list.long_description.astype(str).unique())):\n",
    "        df_patientevents_categorylabels_dict = df_patientevents_categorylabels_dict.append({'eventtype': 'hcpcs', 'category': category, 'label': item}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- CORE > df_admissions\n",
      "--------------------------------\n",
      "subject_id                     float64\n",
      "hadm_id                        float64\n",
      "admittime               datetime64[ns]\n",
      "dischtime               datetime64[ns]\n",
      "deathtime               datetime64[ns]\n",
      "admission_type                  object\n",
      "admission_location              object\n",
      "discharge_location              object\n",
      "insurance                       object\n",
      "language                        object\n",
      "marital_status                  object\n",
      "ethnicity                       object\n",
      "edregtime               datetime64[ns]\n",
      "edouttime               datetime64[ns]\n",
      "hospital_expire_flag           float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- CORE > df_patients\n",
      "--------------------------------\n",
      "subject_id           float64\n",
      "gender                object\n",
      "anchor_age           float64\n",
      "anchor_year          float64\n",
      "anchor_year_group     object\n",
      "dod                   object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- CORE > df_transfers\n",
      "--------------------------------\n",
      "subject_id            float64\n",
      "hadm_id               float64\n",
      "transfer_id           float64\n",
      "eventtype              object\n",
      "careunit               object\n",
      "intime         datetime64[ns]\n",
      "outtime        datetime64[ns]\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_d_labitems\n",
      "--------------------------------\n",
      "itemid        float64\n",
      "label          object\n",
      "fluid          object\n",
      "category       object\n",
      "loinc_code     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_d_icd_procedures\n",
      "--------------------------------\n",
      "icd_code       object\n",
      "icd_version    object\n",
      "long_title     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_d_icd_diagnoses\n",
      "--------------------------------\n",
      "icd_code       object\n",
      "icd_version    object\n",
      "long_title     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_d_hcpcs\n",
      "--------------------------------\n",
      "code                 object\n",
      "category             object\n",
      "long_description     object\n",
      "short_description    object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_diagnoses_icd\n",
      "--------------------------------\n",
      "subject_id     float64\n",
      "hadm_id        float64\n",
      "seq_num        float64\n",
      "icd_code        object\n",
      "icd_version     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_drgcodes\n",
      "--------------------------------\n",
      "subject_id       float64\n",
      "hadm_id          float64\n",
      "drg_type          object\n",
      "drg_code         float64\n",
      "description       object\n",
      "drg_severity     float64\n",
      "drg_mortality    float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_emar\n",
      "--------------------------------\n",
      "subject_id             float64\n",
      "hadm_id                float64\n",
      "emar_id                 object\n",
      "emar_seq               float64\n",
      "poe_id                  object\n",
      "pharmacy_id            float64\n",
      "charttime       datetime64[ns]\n",
      "medication              object\n",
      "event_txt               object\n",
      "scheduletime    datetime64[ns]\n",
      "storetime       datetime64[ns]\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_emar_detail\n",
      "--------------------------------\n",
      "subject_id                              float64\n",
      "emar_id                                  object\n",
      "emar_seq                                float64\n",
      "parent_field_ordinal                    float64\n",
      "administration_type                      object\n",
      "pharmacy_id                             float64\n",
      "barcode_type                             object\n",
      "reason_for_no_barcode                    object\n",
      "complete_dose_not_given                  object\n",
      "dose_due                                 object\n",
      "dose_due_unit                            object\n",
      "dose_given                               object\n",
      "dose_given_unit                          object\n",
      "will_remainder_of_dose_be_given          object\n",
      "product_amount_given                     object\n",
      "product_unit                             object\n",
      "product_code                             object\n",
      "product_description                      object\n",
      "product_description_other                object\n",
      "prior_infusion_rate                      object\n",
      "infusion_rate                            object\n",
      "infusion_rate_adjustment                 object\n",
      "infusion_rate_adjustment_amount          object\n",
      "infusion_rate_unit                       object\n",
      "route                                    object\n",
      "infusion_complete                        object\n",
      "completion_interval                      object\n",
      "new_iv_bag_hung                          object\n",
      "continued_infusion_in_other_location     object\n",
      "restart_interval                         object\n",
      "side                                     object\n",
      "site                                     object\n",
      "non_formulary_visual_verification        object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_hcpcsevents\n",
      "--------------------------------\n",
      "subject_id           float64\n",
      "hadm_id              float64\n",
      "chartdate             object\n",
      "hcpcs_cd              object\n",
      "seq_num              float64\n",
      "short_description     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_labevents\n",
      "--------------------------------\n",
      "labevent_id               float64\n",
      "subject_id                float64\n",
      "hadm_id                   float64\n",
      "specimen_id               float64\n",
      "itemid                    float64\n",
      "charttime          datetime64[ns]\n",
      "storetime          datetime64[ns]\n",
      "value                      object\n",
      "valuenum                  float64\n",
      "valueuom                   object\n",
      "ref_range_lower           float64\n",
      "ref_range_upper           float64\n",
      "flag                       object\n",
      "priority                   object\n",
      "comments                   object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_microbiologyevents\n",
      "--------------------------------\n",
      "microevent_id                 float64\n",
      "subject_id                    float64\n",
      "hadm_id                       float64\n",
      "micro_specimen_id             float64\n",
      "chartdate              datetime64[ns]\n",
      "charttime              datetime64[ns]\n",
      "spec_itemid                   float64\n",
      "spec_type_desc                 object\n",
      "test_seq                      float64\n",
      "storedate              datetime64[ns]\n",
      "storetime              datetime64[ns]\n",
      "test_itemid                   float64\n",
      "test_name                      object\n",
      "org_itemid                    float64\n",
      "org_name                       object\n",
      "isolate_num                   float64\n",
      "quantity                       object\n",
      "ab_itemid                     float64\n",
      "ab_name                        object\n",
      "dilution_text                  object\n",
      "dilution_comparison            object\n",
      "dilution_value                float64\n",
      "interpretation                 object\n",
      "comments                       object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_poe\n",
      "--------------------------------\n",
      "poe_id                            object\n",
      "poe_seq                          float64\n",
      "subject_id                       float64\n",
      "hadm_id                          float64\n",
      "ordertime                 datetime64[ns]\n",
      "order_type                        object\n",
      "order_subtype                     object\n",
      "transaction_type                  object\n",
      "discontinue_of_poe_id             object\n",
      "discontinued_by_poe_id            object\n",
      "order_status                      object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_poe_detail\n",
      "--------------------------------\n",
      "poe_id          object\n",
      "poe_seq        float64\n",
      "subject_id     float64\n",
      "field_name      object\n",
      "field_value     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_prescriptions\n",
      "--------------------------------\n",
      "subject_id                 float64\n",
      "hadm_id                    float64\n",
      "pharmacy_id                float64\n",
      "starttime           datetime64[ns]\n",
      "stoptime            datetime64[ns]\n",
      "drug_type                   object\n",
      "drug                        object\n",
      "gsn                         object\n",
      "ndc                        float64\n",
      "prod_strength               object\n",
      "form_rx                     object\n",
      "dose_val_rx                 object\n",
      "dose_unit_rx                object\n",
      "form_val_disp               object\n",
      "form_unit_disp              object\n",
      "doses_per_24_hrs           float64\n",
      "route                       object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_procedures_icd\n",
      "--------------------------------\n",
      "subject_id     float64\n",
      "hadm_id        float64\n",
      "seq_num        float64\n",
      "chartdate       object\n",
      "icd_code        object\n",
      "icd_version     object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- HOSP > df_services\n",
      "--------------------------------\n",
      "subject_id             float64\n",
      "hadm_id                float64\n",
      "transfertime    datetime64[ns]\n",
      "prev_service            object\n",
      "curr_service            object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_procedureevents\n",
      "--------------------------------\n",
      "subject_id                           float64\n",
      "hadm_id                              float64\n",
      "stay_id                              float64\n",
      "starttime                     datetime64[ns]\n",
      "endtime                       datetime64[ns]\n",
      "storetime                     datetime64[ns]\n",
      "itemid                               float64\n",
      "value                                 object\n",
      "valueuom                              object\n",
      "location                              object\n",
      "locationcategory                      object\n",
      "orderid                              float64\n",
      "linkorderid                          float64\n",
      "ordercategoryname                     object\n",
      "secondaryordercategoryname            object\n",
      "ordercategorydescription              object\n",
      "patientweight                        float64\n",
      "totalamount                          float64\n",
      "totalamountuom                        object\n",
      "isopenbag                            float64\n",
      "continueinnextdept                   float64\n",
      "cancelreason                         float64\n",
      "statusdescription                     object\n",
      "comments_date                 datetime64[ns]\n",
      "originalamount                       float64\n",
      "originalrate                         float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_outputevents\n",
      "--------------------------------\n",
      "subject_id           float64\n",
      "hadm_id              float64\n",
      "stay_id              float64\n",
      "charttime     datetime64[ns]\n",
      "storetime     datetime64[ns]\n",
      "itemid               float64\n",
      "value                 object\n",
      "valueuom              object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_inputevents\n",
      "--------------------------------\n",
      "subject_id                              float64\n",
      "hadm_id                                 float64\n",
      "stay_id                                 float64\n",
      "starttime                        datetime64[ns]\n",
      "endtime                          datetime64[ns]\n",
      "storetime                        datetime64[ns]\n",
      "itemid                                  float64\n",
      "amount                                  float64\n",
      "amountuom                                object\n",
      "rate                                    float64\n",
      "rateuom                                  object\n",
      "orderid                                 float64\n",
      "linkorderid                             float64\n",
      "ordercategoryname                        object\n",
      "secondaryordercategoryname               object\n",
      "ordercomponenttypedescription            object\n",
      "ordercategorydescription                 object\n",
      "patientweight                           float64\n",
      "totalamount                             float64\n",
      "totalamountuom                           object\n",
      "isopenbag                               float64\n",
      "continueinnextdept                      float64\n",
      "cancelreason                            float64\n",
      "statusdescription                        object\n",
      "originalamount                          float64\n",
      "originalrate                            float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_icustays\n",
      "--------------------------------\n",
      "subject_id               float64\n",
      "hadm_id                  float64\n",
      "stay_id                  float64\n",
      "first_careunit            object\n",
      "last_careunit             object\n",
      "intime            datetime64[ns]\n",
      "outtime           datetime64[ns]\n",
      "los                      float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_datetimeevents\n",
      "--------------------------------\n",
      "subject_id           float64\n",
      "hadm_id              float64\n",
      "stay_id              float64\n",
      "charttime     datetime64[ns]\n",
      "storetime     datetime64[ns]\n",
      "itemid               float64\n",
      "value                 object\n",
      "valueuom              object\n",
      "warning              float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_d_items\n",
      "--------------------------------\n",
      "itemid             float64\n",
      "label               object\n",
      "abbreviation        object\n",
      "linksto             object\n",
      "category            object\n",
      "unitname            object\n",
      "param_type          object\n",
      "lownormalvalue     float64\n",
      "highnormalvalue    float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- ICU > df_chartevents\n",
      "--------------------------------\n",
      "subject_id           float64\n",
      "hadm_id              float64\n",
      "stay_id              float64\n",
      "charttime     datetime64[ns]\n",
      "storetime     datetime64[ns]\n",
      "itemid               float64\n",
      "value                 object\n",
      "valuenum             float64\n",
      "valueuom              object\n",
      "warning              float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- CXR > df_mimic_cxr_split\n",
      "--------------------------------\n",
      "dicom_id       object\n",
      "study_id      float64\n",
      "subject_id    float64\n",
      "split          object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- CXR > df_mimic_cxr_chexpert\n",
      "--------------------------------\n",
      "subject_id                    float64\n",
      "study_id                      float64\n",
      "Atelectasis                   float64\n",
      "Cardiomegaly                  float64\n",
      "Consolidation                 float64\n",
      "Edema                         float64\n",
      "Enlarged Cardiomediastinum    float64\n",
      "Fracture                      float64\n",
      "Lung Lesion                   float64\n",
      "Lung Opacity                  float64\n",
      "No Finding                    float64\n",
      "Pleural Effusion              float64\n",
      "Pleural Other                 float64\n",
      "Pneumonia                     float64\n",
      "Pneumothorax                  float64\n",
      "Support Devices               float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- CXR > df_mimic_cxr_metadata\n",
      "--------------------------------\n",
      "dicom_id                                              object\n",
      "subject_id                                           float64\n",
      "study_id                                             float64\n",
      "PerformedProcedureStepDescription                     object\n",
      "ViewPosition                                          object\n",
      "Rows                                                 float64\n",
      "Columns                                              float64\n",
      "StudyDate                                            float64\n",
      "StudyTime                                            float64\n",
      "ProcedureCodeSequence_CodeMeaning                     object\n",
      "ViewCodeSequence_CodeMeaning                          object\n",
      "PatientOrientationCodeSequence_CodeMeaning            object\n",
      "StudyDateForm                                         object\n",
      "StudyTimeForm                                         object\n",
      "cxrtime                                       datetime64[ns]\n",
      "Unnamed: 0                                           float64\n",
      "Img_Filename                                          object\n",
      "Img_Folder                                            object\n",
      "Note                                                  object\n",
      "Note_file                                             object\n",
      "Note_folder                                           object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- CXR > df_mimic_cxr_negbio\n",
      "--------------------------------\n",
      "subject_id                    float64\n",
      "study_id                      float64\n",
      "Atelectasis                   float64\n",
      "Cardiomegaly                  float64\n",
      "Consolidation                 float64\n",
      "Edema                         float64\n",
      "Enlarged Cardiomediastinum    float64\n",
      "Fracture                      float64\n",
      "Lung Lesion                   float64\n",
      "Lung Opacity                  float64\n",
      "No Finding                    float64\n",
      "Pleural Effusion              float64\n",
      "Pleural Other                 float64\n",
      "Pneumonia                     float64\n",
      "Pneumothorax                  float64\n",
      "Support Devices               float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- NOTES > df_noteevents\n",
      "--------------------------------\n",
      "Unnamed: 0              int64\n",
      "row_id                  int64\n",
      "subject_id              int64\n",
      "hadm_id               float64\n",
      "chartdate      datetime64[ns]\n",
      "charttime      datetime64[ns]\n",
      "storetime      datetime64[ns]\n",
      "category               object\n",
      "description            object\n",
      "cgid                  float64\n",
      "iserror               float64\n",
      "text                   object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- NOTES > df_icunotes\n",
      "--------------------------------\n",
      "subject_id                  int64\n",
      "hadm_id                     int64\n",
      "stay_id                     int64\n",
      "note_id                    object\n",
      "note_type                  object\n",
      "note_seq                  float64\n",
      "charttime     datetime64[ns, UTC]\n",
      "storetime     datetime64[ns, UTC]\n",
      "text                       object\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- NOTES > df_ecgnotes\n",
      "--------------------------------\n",
      "subject_id             int64\n",
      "hadm_id                int64\n",
      "stay_id                int64\n",
      "intime                object\n",
      "outtime               object\n",
      "note_id               object\n",
      "note_type             object\n",
      "note_seq             float64\n",
      "charttime             object\n",
      "storetime     datetime64[ns]\n",
      "text                  object\n",
      "offset               float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- NOTES > df_echonotes\n",
      "--------------------------------\n",
      "subject_id      int64\n",
      "hadm_id         int64\n",
      "stay_id         int64\n",
      "intime         object\n",
      "outtime        object\n",
      "note_id        object\n",
      "note_type      object\n",
      "note_seq      float64\n",
      "charttime      object\n",
      "storetime      object\n",
      "text           object\n",
      "offset        float64\n",
      "offset.num    float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n",
      "- NOTES > df_radnotes\n",
      "--------------------------------\n",
      "subject_id                    int64\n",
      "hadm_id                       int64\n",
      "stay_id                       int64\n",
      "intime                       object\n",
      "outtime                      object\n",
      "note_id                      object\n",
      "subject_id_1                float64\n",
      "hadm_id_1                   float64\n",
      "note_type                    object\n",
      "note_seq                    float64\n",
      "charttime       datetime64[ns, UTC]\n",
      "storetime       datetime64[ns, UTC]\n",
      "text                         object\n",
      "offset                      float64\n",
      "offset.num                  float64\n",
      "dtype: object\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "## CORE\n",
    "print('- CORE > df_admissions')\n",
    "print('--------------------------------')\n",
    "print(df_admissions.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- CORE > df_patients')\n",
    "print('--------------------------------')\n",
    "print(df_patients.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- CORE > df_transfers')\n",
    "print('--------------------------------')\n",
    "print(df_transfers.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "\n",
    "## HOSP\n",
    "print('- HOSP > df_d_labitems')\n",
    "print('--------------------------------')\n",
    "print(df_d_labitems.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_d_icd_procedures')\n",
    "print('--------------------------------')\n",
    "print(df_d_icd_procedures.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_d_icd_diagnoses')\n",
    "print('--------------------------------')\n",
    "print(df_d_icd_diagnoses.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_d_hcpcs')\n",
    "print('--------------------------------')\n",
    "print(df_d_hcpcs.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_diagnoses_icd')\n",
    "print('--------------------------------')\n",
    "print(df_diagnoses_icd.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_drgcodes')\n",
    "print('--------------------------------')\n",
    "print(df_drgcodes.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_emar')\n",
    "print('--------------------------------')\n",
    "print(df_emar.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_emar_detail')\n",
    "print('--------------------------------')\n",
    "print(df_emar_detail.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_hcpcsevents')\n",
    "print('--------------------------------')\n",
    "print(df_hcpcsevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_labevents')\n",
    "print('--------------------------------')\n",
    "print(df_labevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_microbiologyevents')\n",
    "print('--------------------------------')\n",
    "print(df_microbiologyevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_poe')\n",
    "print('--------------------------------')\n",
    "print(df_poe.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_poe_detail')\n",
    "print('--------------------------------')\n",
    "print(df_poe_detail.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_prescriptions')\n",
    "print('--------------------------------')\n",
    "print(df_prescriptions.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_procedures_icd')\n",
    "print('--------------------------------')\n",
    "print(df_procedures_icd.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- HOSP > df_services')\n",
    "print('--------------------------------')\n",
    "print(df_services.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "\n",
    "## ICU\n",
    "print('- ICU > df_procedureevents')\n",
    "print('--------------------------------')\n",
    "print(df_procedureevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- ICU > df_outputevents')\n",
    "print('--------------------------------')\n",
    "print(df_outputevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- ICU > df_inputevents')\n",
    "print('--------------------------------')\n",
    "print(df_inputevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- ICU > df_icustays')\n",
    "print('--------------------------------')\n",
    "print(df_icustays.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- ICU > df_datetimeevents')\n",
    "print('--------------------------------')\n",
    "print(df_datetimeevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- ICU > df_d_items')\n",
    "print('--------------------------------')\n",
    "print(df_d_items.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- ICU > df_chartevents')\n",
    "print('--------------------------------')\n",
    "print(df_chartevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "\n",
    "## CXR\n",
    "print('- CXR > df_mimic_cxr_split')\n",
    "print('--------------------------------')\n",
    "print(df_mimic_cxr_split.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- CXR > df_mimic_cxr_chexpert')\n",
    "print('--------------------------------')\n",
    "print(df_mimic_cxr_chexpert.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- CXR > df_mimic_cxr_metadata')\n",
    "print('--------------------------------')\n",
    "print(df_mimic_cxr_metadata.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- CXR > df_mimic_cxr_negbio')\n",
    "print('--------------------------------')\n",
    "print(df_mimic_cxr_negbio.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "\n",
    "## NOTES\n",
    "print('- NOTES > df_noteevents')\n",
    "print('--------------------------------')\n",
    "print(df_noteevents.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- NOTES > df_icunotes')\n",
    "print('--------------------------------')\n",
    "print(df_dsnotes.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- NOTES > df_ecgnotes')\n",
    "print('--------------------------------')\n",
    "print(df_ecgnotes.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- NOTES > df_echonotes')\n",
    "print('--------------------------------')\n",
    "print(df_echonotes.dtypes)\n",
    "print('\\n\\n')\n",
    "\n",
    "print('- NOTES > df_radnotes')\n",
    "print('--------------------------------')\n",
    "print(df_radnotes.dtypes)\n",
    "print('\\n\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## -> GET LIST OF ALL UNIQUE ID COMBINATIONS IN MIMIC-IV (subject_id, hadm_id, stay_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_base_core = df_admissions.merge(df_patients, how='left').merge(df_transfers, how='left')\n",
    "df_base_core.to_csv(core_mimiciv_path + 'core/core.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get Unique Subject/HospAdmission/Stay Combinations\n",
    "df_ids = pd.concat([pd.DataFrame(), df_procedureevents[['subject_id','hadm_id','stay_id']]], sort=False).drop_duplicates()\n",
    "df_ids = pd.concat([df_ids, df_outputevents[['subject_id','hadm_id','stay_id']]], sort=False).drop_duplicates()\n",
    "df_ids = pd.concat([df_ids, df_inputevents[['subject_id','hadm_id','stay_id']]], sort=False).drop_duplicates()\n",
    "df_ids = pd.concat([df_ids, df_icustays[['subject_id','hadm_id','stay_id']]], sort=False).drop_duplicates()\n",
    "df_ids = pd.concat([df_ids, df_datetimeevents[['subject_id','hadm_id','stay_id']]], sort=False).drop_duplicates()\n",
    "df_ids = pd.concat([df_ids, df_chartevents[['subject_id','hadm_id','stay_id']]], sort=True).drop_duplicates()\n",
    "\n",
    "# Get Unique Subjects with Chest Xrays\n",
    "df_cxr_ids = pd.concat([pd.DataFrame(), df_mimic_cxr_chexpert[['subject_id']]], sort=True).drop_duplicates()\n",
    "\n",
    "# Get Unique Subject/HospAdmission/Stay Combinations with Chest Xrays\n",
    "df_haim_ids = df_ids[df_ids['subject_id'].isin(df_cxr_ids['subject_id'].unique())] \n",
    "\n",
    "# Save Unique Subject/HospAdmission/Stay Combinations with Chest Xrays    \n",
    "df_haim_ids.to_csv(core_mimiciv_path + 'haim_mimiciv_key_ids.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique Subjects: 382278\n",
      "Unique Subjects/HospAdmissions/Stays Combinations: 76540\n",
      "Unique Subjects with Chest Xrays Available: 65379\n"
     ]
    }
   ],
   "source": [
    "print('Unique Subjects: ' + str(len(df_patients['subject_id'].unique())))\n",
    "print('Unique Subjects/HospAdmissions/Stays Combinations: ' + str(len(df_ids)))\n",
    "print('Unique Subjects with Chest Xrays Available: ' + str(len(df_cxr_ids)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique HAIM Records Available: 34539\n"
     ]
    }
   ],
   "source": [
    "# Save Unique Subject/HospAdmission/Stay Combinations with Chest Xrays    \n",
    "df_haim_ids = pd.read_csv(core_mimiciv_path + 'haim_mimiciv_key_ids.csv')\n",
    "print('Unique HAIM Records Available: ' + str(len(df_haim_ids)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## -> SAVE ALL SINGLE PATIENT FILES FOR LATER ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GET FULL MIMIC IV PATIENT RECORD USING DATABASE KEYS\n",
    "def get_patient_icustay(key_subject_id, key_hadm_id, key_stay_id):\n",
    "        # Inputs:\n",
    "        #   key_subject_id -> subject_id is unique to a patient\n",
    "        #   key_hadm_id    -> hadm_id is unique to a patient hospital stay\n",
    "        #   key_stay_id    -> stay_id is unique to a patient ward stay\n",
    "        #   \n",
    "        #   NOTES: Identifiers which specify the patient. More information about \n",
    "        #   these identifiers is available at https://mimic-iv.mit.edu/basics/identifiers\n",
    "    \n",
    "        # Outputs:\n",
    "        #   Patient_ICUstay -> ICU patient stay structure\n",
    "    \n",
    "        #-> FILTER data\n",
    "        ##-> CORE\n",
    "        f_df_base_core = df_base_core[(df_base_core.subject_id == key_subject_id) & (df_base_core.hadm_id == key_hadm_id)]\n",
    "        f_df_admissions = df_admissions[(df_admissions.subject_id == key_subject_id) & (df_admissions.hadm_id == key_hadm_id)]\n",
    "        f_df_patients = df_patients[(df_patients.subject_id == key_subject_id)]\n",
    "        f_df_transfers = df_transfers[(df_transfers.subject_id == key_subject_id) & (df_transfers.hadm_id == key_hadm_id)]\n",
    "        ###-> Merge data into single patient structure\n",
    "        f_df_core = f_df_base_core\n",
    "        f_df_core = f_df_core.merge(f_df_admissions, how='left')\n",
    "        f_df_core = f_df_core.merge(f_df_patients, how='left')\n",
    "        f_df_core = f_df_core.merge(f_df_transfers, how='left')\n",
    "    \n",
    "        ##-> HOSP\n",
    "        f_df_diagnoses_icd = df_diagnoses_icd[(df_diagnoses_icd.subject_id == key_subject_id)]\n",
    "        f_df_drgcodes = df_drgcodes[(df_drgcodes.subject_id == key_subject_id) & (df_drgcodes.hadm_id == key_hadm_id)]\n",
    "        f_df_emar = df_emar[(df_emar.subject_id == key_subject_id) & (df_emar.hadm_id == key_hadm_id)]\n",
    "        f_df_emar_detail = df_emar_detail[(df_emar_detail.subject_id == key_subject_id)]\n",
    "        f_df_hcpcsevents = df_hcpcsevents[(df_hcpcsevents.subject_id == key_subject_id) & (df_hcpcsevents.hadm_id == key_hadm_id)]\n",
    "        f_df_labevents = df_labevents[(df_labevents.subject_id == key_subject_id) & (df_labevents.hadm_id == key_hadm_id)]\n",
    "        f_df_microbiologyevents = df_microbiologyevents[(df_microbiologyevents.subject_id == key_subject_id) & (df_microbiologyevents.hadm_id == key_hadm_id)]\n",
    "        f_df_poe = df_poe[(df_poe.subject_id == key_subject_id) & (df_poe.hadm_id == key_hadm_id)]\n",
    "        f_df_poe_detail = df_poe_detail[(df_poe_detail.subject_id == key_subject_id)]\n",
    "        f_df_prescriptions = df_prescriptions[(df_prescriptions.subject_id == key_subject_id) & (df_prescriptions.hadm_id == key_hadm_id)]\n",
    "        f_df_procedures_icd = df_procedures_icd[(df_procedures_icd.subject_id == key_subject_id) & (df_procedures_icd.hadm_id == key_hadm_id)]\n",
    "        f_df_services = df_services[(df_services.subject_id == key_subject_id) & (df_services.hadm_id == key_hadm_id)]\n",
    "        ###-> Merge content from dictionaries\n",
    "        f_df_diagnoses_icd = f_df_diagnoses_icd.merge(df_d_icd_diagnoses, how='left') \n",
    "        f_df_procedures_icd = f_df_procedures_icd.merge(df_d_icd_procedures, how='left')\n",
    "        f_df_hcpcsevents = f_df_hcpcsevents.merge(df_d_hcpcs, how='left')\n",
    "        f_df_labevents = f_df_labevents.merge(df_d_labitems, how='left')\n",
    "    \n",
    "        ##-> ICU\n",
    "        f_df_procedureevents = df_procedureevents[(df_procedureevents.subject_id == key_subject_id) & (df_procedureevents.hadm_id == key_hadm_id) & (df_procedureevents.stay_id == key_stay_id)]\n",
    "        f_df_outputevents = df_outputevents[(df_outputevents.subject_id == key_subject_id) & (df_outputevents.hadm_id == key_hadm_id) & (df_outputevents.stay_id == key_stay_id)]\n",
    "        f_df_inputevents = df_inputevents[(df_inputevents.subject_id == key_subject_id) & (df_inputevents.hadm_id == key_hadm_id) & (df_inputevents.stay_id == key_stay_id)]\n",
    "        f_df_icustays = df_icustays[(df_icustays.subject_id == key_subject_id) & (df_icustays.hadm_id == key_hadm_id) & (df_icustays.stay_id == key_stay_id)]\n",
    "        f_df_datetimeevents = df_datetimeevents[(df_datetimeevents.subject_id == key_subject_id) & (df_datetimeevents.hadm_id == key_hadm_id) & (df_datetimeevents.stay_id == key_stay_id)]\n",
    "        f_df_chartevents = df_chartevents[(df_chartevents.subject_id == key_subject_id) & (df_chartevents.hadm_id == key_hadm_id) & (df_chartevents.stay_id == key_stay_id)]\n",
    "        ###-> Merge content from dictionaries\n",
    "        f_df_procedureevents = f_df_procedureevents.merge(df_d_items, how='left')\n",
    "        f_df_outputevents = f_df_outputevents.merge(df_d_items, how='left')\n",
    "        f_df_inputevents = f_df_inputevents.merge(df_d_items, how='left')\n",
    "        f_df_datetimeevents = f_df_datetimeevents.merge(df_d_items, how='left')\n",
    "        f_df_chartevents = f_df_chartevents.merge(df_d_items, how='left')       \n",
    "    \n",
    "        ##-> CXR\n",
    "        f_df_mimic_cxr_split = df_mimic_cxr_split[(df_mimic_cxr_split.subject_id == key_subject_id)]\n",
    "        f_df_mimic_cxr_chexpert = df_mimic_cxr_chexpert[(df_mimic_cxr_chexpert.subject_id == key_subject_id)]\n",
    "        f_df_mimic_cxr_metadata = df_mimic_cxr_metadata[(df_mimic_cxr_metadata.subject_id == key_subject_id)]\n",
    "        f_df_mimic_cxr_negbio = df_mimic_cxr_negbio[(df_mimic_cxr_negbio.subject_id == key_subject_id)]\n",
    "        ###-> Merge data into single patient structure\n",
    "        f_df_cxr = f_df_mimic_cxr_split\n",
    "        f_df_cxr = f_df_cxr.merge(f_df_mimic_cxr_chexpert, how='left')\n",
    "        f_df_cxr = f_df_cxr.merge(f_df_mimic_cxr_metadata, how='left')\n",
    "        f_df_cxr = f_df_cxr.merge(f_df_mimic_cxr_negbio, how='left')\n",
    "        ###-> Get images of that timebound patient\n",
    "        f_df_imcxr = []\n",
    "        for img_idx, img_row in f_df_cxr.iterrows():\n",
    "            img_path = core_mimiciv_imgcxr_path + str(img_row['Img_Folder']) + '/' + str(img_row['Img_Filename'])\n",
    "            img_cxr_shape = [224, 224]\n",
    "            img_cxr = cv2.resize(cv2.imread(img_path, cv2.IMREAD_GRAYSCALE), (img_cxr_shape[0], img_cxr_shape[1]))\n",
    "            f_df_imcxr.append(np.array(img_cxr))\n",
    "    \n",
    "        ##-> NOTES\n",
    "        f_df_noteevents = df_noteevents[(df_noteevents.subject_id == key_subject_id) & (df_noteevents.hadm_id == key_hadm_id)]\n",
    "        f_df_dsnotes = df_dsnotes[(df_dsnotes.subject_id == key_subject_id) & (df_dsnotes.hadm_id == key_hadm_id) & (df_dsnotes.stay_id == key_stay_id)]\n",
    "        f_df_ecgnotes = df_ecgnotes[(df_ecgnotes.subject_id == key_subject_id) & (df_ecgnotes.hadm_id == key_hadm_id) & (df_ecgnotes.stay_id == key_stay_id)]\n",
    "        f_df_echonotes = df_echonotes[(df_echonotes.subject_id == key_subject_id) & (df_echonotes.hadm_id == key_hadm_id) & (df_echonotes.stay_id == key_stay_id)]\n",
    "        f_df_radnotes = df_radnotes[(df_radnotes.subject_id == key_subject_id) & (df_radnotes.hadm_id == key_hadm_id) & (df_radnotes.stay_id == key_stay_id)]\n",
    "        \n",
    "        ###-> Merge data into single patient structure\n",
    "        #--None\n",
    "    \n",
    "    \n",
    "        # -> Create & Populate patient structure\n",
    "        ## CORE\n",
    "        admissions = f_df_admissions\n",
    "        demographics = f_df_patients\n",
    "        transfers = f_df_transfers\n",
    "        core = f_df_core\n",
    "    \n",
    "        ## HOSP\n",
    "        diagnoses_icd = f_df_diagnoses_icd\n",
    "        drgcodes = f_df_diagnoses_icd\n",
    "        emar = f_df_emar\n",
    "        emar_detail = f_df_emar_detail\n",
    "        hcpcsevents = f_df_hcpcsevents\n",
    "        labevents = f_df_labevents\n",
    "        microbiologyevents = f_df_microbiologyevents\n",
    "        poe = f_df_poe\n",
    "        poe_detail = f_df_poe_detail\n",
    "        prescriptions = f_df_prescriptions\n",
    "        procedures_icd = f_df_procedures_icd\n",
    "        services = f_df_services\n",
    "    \n",
    "        ## ICU\n",
    "        procedureevents = f_df_procedureevents\n",
    "        outputevents = f_df_outputevents\n",
    "        inputevents = f_df_inputevents\n",
    "        icustays = f_df_icustays\n",
    "        datetimeevents = f_df_datetimeevents\n",
    "        chartevents = f_df_chartevents\n",
    "    \n",
    "        ## CXR\n",
    "        cxr = f_df_cxr \n",
    "        imcxr = f_df_imcxr\n",
    "    \n",
    "        ## NOTES\n",
    "        noteevents = f_df_noteevents\n",
    "        dsnotes = f_df_dsnotes\n",
    "        ecgnotes = f_df_ecgnotes\n",
    "        echonotes = f_df_echonotes\n",
    "        radnotes = f_df_radnotes\n",
    "        \n",
    "        \n",
    "        # Create patient object and return\n",
    "        Patient_ICUstay = Patient_ICU(admissions, demographics, transfers, core, \\\n",
    "                                      diagnoses_icd, drgcodes, emar, emar_detail, hcpcsevents, \\\n",
    "                                      labevents, microbiologyevents, poe, poe_detail, \\\n",
    "                                      prescriptions, procedures_icd, services, procedureevents, \\\n",
    "                                      outputevents, inputevents, icustays, datetimeevents, \\\n",
    "                                      chartevents, cxr, imcxr, noteevents, dsnotes, ecgnotes, \\\n",
    "                                      echonotes, radnotes)\n",
    "    \n",
    "        return Patient_ICUstay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# EXTRACT ALL INFO OF A SINGLE PATIENT FROM MIMIC-IV DATASET USING HAIM ID\n",
    "def extract_single_patient_records_mimiciv(haim_patient_idx, df_haim_ids, start_hr, end_hr):\n",
    "    # Inputs:\n",
    "    #   haim_patient_idx -> Ordered number of HAIM patient\n",
    "    #   df_haim_ids -> Dataframe with all unique available HAIM_MIMICIV records by key identifiers\n",
    "    #   start_hr -> start_hr indicates the first valid time (in hours) from the admition time \"admittime\" for all retreived features, input \"None\" to avoid time bounding\n",
    "    #   end_hr -> end_hr indicates the last valid time (in hours) from the admition time \"admittime\" for all retreived features, input \"None\" to avoid time bounding\n",
    "    #\n",
    "    # Outputs:\n",
    "    #   key_subject_id -> MIMIC-IV Subject ID of selected patient\n",
    "    #   key_hadm_id -> MIMIC-IV Hospital Admission ID of selected patient\n",
    "    #   key_stay_id -> MIMIC-IV ICU Stay ID of selected patient\n",
    "    #   patient -> Full ICU patient ICU stay structure\n",
    "    #   dt_patient -> Timebound ICU patient stay structure filtered by max_time_stamp or min_time_stamp if any\n",
    "    \n",
    "    # Extract information for patient\n",
    "    key_subject_id = df_haim_ids.iloc[haim_patient_idx].subject_id\n",
    "    key_hadm_id = df_haim_ids.iloc[haim_patient_idx].hadm_id\n",
    "    key_stay_id = df_haim_ids.iloc[haim_patient_idx].stay_id\n",
    "    start_hr = start_hr # Select timestamps\n",
    "    end_hr = end_hr   # Select timestamps\n",
    "    patient = get_patient_icustay(key_subject_id, key_hadm_id, key_stay_id)\n",
    "    dt_patient = get_timebound_patient_icustay(patient, start_hr , end_hr)\n",
    "    \n",
    "    return key_subject_id, key_hadm_id, key_stay_id, patient, dt_patient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GENERATE ALL SINGLE PATIENT ICU STAY RECORDS FOR ENTIRE MIMIC-IV DATABASE\n",
    "def generate_all_mimiciv_patient_object(df_haim_ids, core_mimiciv_path):\n",
    "    # Inputs:\n",
    "    #   df_haim_ids -> Dataframe with all unique available HAIM_MIMICIV records by key identifiers\n",
    "    #   core_mimiciv_path -> Path to structured MIMIC IV databases in CSV files\n",
    "    #\n",
    "    # Outputs:\n",
    "    #   nfiles -> Number of single patient HAIM files produced\n",
    "    \n",
    "    # Clean out\n",
    "    sys.stdout.flush()\n",
    "    \n",
    "    # Extract information for patient\n",
    "    nfiles = len(df_haim_ids)\n",
    "    with tqdm(total = nfiles) as pbar:\n",
    "        # Update process bar\n",
    "        nbase= 6513\n",
    "        pbar.update(nbase)\n",
    "        #Iterate through all patients\n",
    "        for haim_patient_idx in range(nbase, nfiles):\n",
    "            # Let's select each single patient and extract patient object\n",
    "            start_hr = None # Select timestamps\n",
    "            end_hr = None   # Select timestamps\n",
    "            key_subject_id, key_hadm_id, key_stay_id, patient, dt_patient = extract_single_patient_records_mimiciv(haim_patient_idx, df_haim_ids, start_hr, end_hr)\n",
    "            \n",
    "            # Save\n",
    "            filename = f\"{haim_patient_idx:08d}\" + '.pkl'\n",
    "            save_patient_object(dt_patient, core_mimiciv_path + 'pickle/' + filename)\n",
    "            # Update process bar\n",
    "            pbar.update(1)\n",
    "    return nfiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 34539/34539 [27:38:17<00:00,  2.88s/it]   \n"
     ]
    }
   ],
   "source": [
    "# GENERATE ALL SINGLE PATIENT ICU STAY RECORDS FOR ENTIRE MIMIC-IV DATABASE\n",
    "nfiles = generate_all_mimiciv_patient_object(df_haim_ids, core_mimiciv_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## -> CHECK EVERYTHING WAS EXTRACTED CORRECTLY BY TESTING A SINGLE PATIENT RETRIEVAL AND ANALYSIS FROM HAIM-MIMIC-MM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's select a single HAIM Patient from pickle files and check if it fits inclusion criteria\n",
    "haim_patient_idx = 0\n",
    "\n",
    "# Select allowed timestamp range\n",
    "start_hr = None\n",
    "end_hr = None\n",
    "\n",
    "#Load precomputed file\n",
    "filename = f\"{haim_patient_idx:08d}\" + '.pkl'\n",
    "patient = load_patient_object(core_mimiciv_pathç + 'pickle/' + filename)\n",
    "dt_patient = get_timebound_patient_icustay(patient, start_hr , end_hr)\n",
    "\n",
    "# Define inclusion criteria\n",
    "inclusion_criteria =[['ischemic heart disease', 'heart disease (ischemic)', 'heart disease'], ['acute respiratory failure', 'respiratory failure'], ['hypertension'],[\"died\"]]\n",
    "is_included, inclusion_criteria_mask = is_haim_patient_inclusion_criteria_match(dt_patient, inclusion_criteria, verbose=0)\n",
    "get_visioin_embedding(dt_patient)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "haim_py37",
   "language": "python",
   "name": "haim_py37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}