encdd_v0.2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cf8a31c2-d560-4a61-832e-94643bf3799d",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08ea20fa-68b0-4fd0-8c81-8a9db45bbbbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# encdd_v0.2.ipynb\n",
    "# From load_model_v0.1.1.ipynb\n",
    "# Goal: Inkscape now for stitching, then to slides\n",
    "\n",
    "    # logreg x 2 - 30 cross validation repeat folds (next version) v0.1.2 - Star bars\n",
    "    # Euclidean plots - modify per whiteboard to get mean line for comparsion - Sig_ridge\n",
    "\n",
    "# Non-mega stitch of log regs and Euc mean lines for doing the stat sig stars - later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "627d1ca0-f8a3-4299-b1bd-632764c4225f",
   "metadata": {},
   "outputs": [],
   "source": [
    "v = 'v0.2'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b27bd417-cb79-4e0e-b773-0f53df5b9593",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## VAE, monolithic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c88ebbc6-c470-4389-baa4-b61837b766b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import datetime\n",
    "log_dir = Path(\"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c78949de-497b-4ed7-8f71-529eefd273ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b56097a3-5d9f-455d-817e-8f90e838aa75",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = Path(\"logs\", \"prot-\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e8460b0-fed6-4db3-bfbf-89f806510c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# From vae_v0.1.py\n",
    "# Turned off argparse import\n",
    "# Commented if __name__ == '__main__':\n",
    "# Commented argparse lines under that ^\n",
    "# Begin interactive work on line 232\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "import datetime\n",
    "from pathlib import Path # log dir\n",
    "# from argparse import ArgumentParser\n",
    "from scipy.spatial import distance  # for cosine similarity calc\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# construct a log dir name, formatted as a string in the format \"YearMonthDay-HourMinuteSecond\".\n",
    "# ie logs/20231025-142536\n",
    "# Supports TensorBoard\n",
    "log_dir = Path(\"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "\n",
    "# Check if dir exist, okay if it does\n",
    "if not log_dir.exists():\n",
    "    log_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# Yes, it looks like cancer type encoder exist, whereto from here?\n",
    "model_type_encoder = LabelEncoder()\n",
    "cancer_type_encoder = LabelEncoder()\n",
    "\n",
    "# Here is a problem for re-using the cosine sim function for the cancer type distance\n",
    "# Is there a conceptual benefit to using the same or two diffent types of distance for the model mechanics and evaluation?\n",
    "def calculate_cosine_similarity(df: pd.DataFrame, model_type_column='model_type') -> (dict, dict):\n",
    "    # Separate the DataFrame into two based on the model_type\n",
    "    df_cell_line = df[df[model_type_column] == 'cell line'].drop(columns=[model_type_column])\n",
    "    df_tumor = df[df[model_type_column] == 'Tumor'].drop(columns=[model_type_column])\n",
    "\n",
    "    # Initialize dictionaries to store the results\n",
    "    cosine_similarities_cell_line = {}\n",
    "    cosine_similarities_tumor = {}\n",
    "\n",
    "    # Identify the columns that are common and valid for mean calculation\n",
    "    valid_columns = df_cell_line.select_dtypes(include=[np.number]).columns\n",
    "\n",
    "    # Calculate the mean vector for each class for the relevant columns\n",
    "    mean_vector_cell_line = df_cell_line[valid_columns].mean(axis=0).values\n",
    "    mean_vector_tumor = df_tumor[valid_columns].mean(axis=0).values\n",
    "\n",
    "    # Loop through each sample in cell_line and calculate cosine similarity to both mean vectors\n",
    "    for index, row in df_cell_line.iterrows():\n",
    "        sample_vector = row[valid_columns].values\n",
    "        sim_to_cell_line = distance.cosine(sample_vector, mean_vector_cell_line)\n",
    "        sim_to_tumor = distance.cosine(sample_vector, mean_vector_tumor)\n",
    "        cosine_similarities_cell_line[index] = (sim_to_cell_line, sim_to_tumor)\n",
    "\n",
    "    # Loop through each sample in tumor and calculate cosine similarity to both mean vectors\n",
    "    for index, row in df_tumor.iterrows():\n",
    "        sample_vector = row[valid_columns].values\n",
    "        sim_to_cell_line = distance.cosine(sample_vector, mean_vector_cell_line)\n",
    "        sim_to_tumor = distance.cosine(sample_vector, mean_vector_tumor)\n",
    "        cosine_similarities_tumor[index] = (sim_to_cell_line, sim_to_tumor)\n",
    "\n",
    "    # Combine the 2 dist dicts into one dict\n",
    "    cosine_similarities_tumor.update(cosine_similarities_cell_line)\n",
    "    cosine_similarities = cosine_similarities_tumor\n",
    "\n",
    "    # Prefilled 0s lists, number of dictionary keys\n",
    "    intra_cluster_tensor = list(range(df.shape[0]))\n",
    "    inter_cluster_tensor = list(range(df.shape[0]))\n",
    "\n",
    "    assert len(intra_cluster_tensor) == len(df), \"Length of list is not as expected\"\n",
    "    assert len(inter_cluster_tensor) == len(df), \"Length of list is not as expected\"\n",
    "\n",
    "    # Populate the zero lists with dist scores\n",
    "    for key in cosine_similarities.keys():\n",
    "        intra_cluster_tensor[key] = cosine_similarities[key][0]\n",
    "        inter_cluster_tensor[key] = cosine_similarities[key][1]\n",
    "\n",
    "    # Convert lists to tensors\n",
    "    intra_cluster_tensor = tf.convert_to_tensor(intra_cluster_tensor, dtype=np.float32)\n",
    "    inter_cluster_tensor = tf.convert_to_tensor(inter_cluster_tensor, dtype=np.float32)\n",
    "\n",
    "    return intra_cluster_tensor, inter_cluster_tensor\n",
    "\n",
    "\n",
    "# Define the Sampling Layer\n",
    "class Sampling(layers.Layer):\n",
    "    \"\"\"Uses (z_mean, z_log_var) to sample z, the vector encoding a digit.\"\"\"\n",
    "    # Stock TensorFlow\n",
    "    def call(self, inputs):\n",
    "        z_mean, z_log_var = inputs\n",
    "        batch = tf.shape(z_mean)[0]\n",
    "        dim = tf.shape(z_mean)[1]\n",
    "        epsilon = tf.random.normal(shape=(batch, dim))\n",
    "        return z_mean + tf.exp(0.5 * z_log_var) * epsilon\n",
    "\n",
    "\n",
    "# Define the VAE class\n",
    "class VAE(keras.Model):\n",
    "    def __init__(self, encoder, decoder, columns, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.encoder: keras.Model = encoder\n",
    "        self.decoder: keras.Model = decoder\n",
    "        self.total_loss_tracker = keras.metrics.Mean(name=\"total_loss\")\n",
    "        self.reconstruction_loss_tracker = keras.metrics.Mean(name=\"reconstruction_loss\")\n",
    "        self.kl_loss_tracker = keras.metrics.Mean(name=\"kl_loss\")\n",
    "        self.distance_loss_tracker = keras.metrics.Mean(name=\"distance_loss\") # added, will need to rename to distinguish from the cancer type distance\n",
    "                                                                                # Method: copy out to new notebook file, mod, put into new quad_loss script\n",
    "        self.columns = columns  # added, argument to class, can use for both model type and cancer type distance (?)\n",
    "\n",
    "    @property\n",
    "    def metrics(self):\n",
    "        return [\n",
    "            self.total_loss_tracker,\n",
    "            self.reconstruction_loss_tracker,\n",
    "            self.kl_loss_tracker,\n",
    "            self.distance_loss_tracker, # added, again this is specific to model type. Idea: just add zero for the fourth cancer type term to toggle off?\n",
    "        ]                                  # set 4th function / value to zero for decel toggle?\n",
    "\n",
    "    def train_step(self, data):\n",
    "        with tf.GradientTape() as tape:\n",
    "            converted_data: pd.DataFrame = pd.DataFrame(data.numpy(), columns=self.columns)\n",
    "            model_type = converted_data[\"model_type\"]\n",
    "            model_type = model_type.astype(int)\n",
    "            model_type = model_type_encoder.inverse_transform(model_type)\n",
    "            data = converted_data.drop(columns=[\"model_type\"])  # DROP\n",
    "            assert \"model_type\" not in data.columns, \"model_type should not be in data\"\n",
    "\n",
    "            # Is this the functionality for getting the cancer type distance as fourth term into the composite loss?\n",
    "            cancer_type = converted_data[\"cancer_type\"]\n",
    "            cancer_type = cancer_type.astype(int)\n",
    "            cancer_type = cancer_type_encoder.inverse_transform(cancer_type)\n",
    "            data = data.drop(columns=[\"cancer_type\"])  # DROP\n",
    "            assert \"cancer_type\" not in data.columns, \"cancer_type should not be in data\"\n",
    "\n",
    "            data = tf.convert_to_tensor(data)\n",
    "\n",
    "            z_mean, z_log_var, z = self.encoder(data)\n",
    "\n",
    "            labeled_embeddings: pd.DataFrame = pd.DataFrame(z.numpy())\n",
    "            labeled_embeddings[\"model_type\"] = model_type\n",
    "            # print('\\n\\n\\n')\n",
    "            # print(\"Labeled Embeddings\")\n",
    "            # print(labeled_embeddings)\n",
    "            # input() # \"return break\"\n",
    "\n",
    "            # write\n",
    "\n",
    "            intra_cluster_distance, inter_cluster_distance = calculate_cosine_similarity(\n",
    "                df=labeled_embeddings,\n",
    "                model_type_column='model_type')\n",
    "\n",
    "            # print(\"Distances:\")\n",
    "            # print(\"Intra Cluster:\")\n",
    "            # print(intra_cluster_distance)\n",
    "            # print(\"Inter Cluster:\")\n",
    "            # print(inter_cluster_distance)\n",
    "            # input()\n",
    "\n",
    "            # Stock VAE\n",
    "            reconstruction = self.decoder(z)\n",
    "            reconstruction_loss = .1 * data.shape[1] * keras.losses.binary_crossentropy(data, reconstruction)\n",
    "            kl_loss = - 0.5 * tf.reduce_sum(1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var), axis=-1)\n",
    "\n",
    "            # Added\n",
    "            distance_loss = - 100 * inter_cluster_distance\n",
    "\n",
    "            total_loss = reconstruction_loss + kl_loss + distance_loss\n",
    "            # total_loss = reconstruction_loss + kl_loss + distance_loss + cncr_dist_loss # v0.4\n",
    "\n",
    "            # print(\"Rec Loss\")\n",
    "            # print(reconstruction_loss)\n",
    "            # print(\"kl Loss\")\n",
    "            # print(kl_loss)\n",
    "            # print(\"Distance loss\")\n",
    "            # print(distance_loss)\n",
    "            # print(\"Total loss\")\n",
    "            # print(total_loss)\n",
    "            # input()\n",
    "\n",
    "        grads = tape.gradient(total_loss, self.trainable_weights)\n",
    "        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n",
    "        self.total_loss_tracker.update_state(total_loss)\n",
    "        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n",
    "        self.kl_loss_tracker.update_state(kl_loss)\n",
    "        self.distance_loss_tracker.update_state(distance_loss)\n",
    "        return {\n",
    "            \"loss\": self.total_loss_tracker.result(),\n",
    "            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n",
    "            \"kl_loss\": self.kl_loss_tracker.result(),\n",
    "            \"distance_loss\": self.distance_loss_tracker.result(),\n",
    "        }\n",
    "\n",
    "\n",
    "# Build Encoder\n",
    "def build_encoder(feature_dim, latent_dim) -> keras.Model:\n",
    "    encoder_inputs = keras.Input(shape=(feature_dim,), name=\"input_1\")\n",
    "    x = keras.layers.Dense(latent_dim, kernel_initializer='glorot_uniform', name=\"encoder_dense_1\")(encoder_inputs)\n",
    "    x = keras.layers.BatchNormalization(name=\"batchnorm\")(x)\n",
    "    z_mean = layers.Dense(latent_dim, name=\"z_mean\")(x)\n",
    "    z_log_var = layers.Dense(latent_dim, name=\"z_log_var\")(x)\n",
    "    z = Sampling()([z_mean, z_log_var])\n",
    "    encoder = keras.Model(encoder_inputs, [z_mean, z_log_var, z], name=\"encoder\")\n",
    "    return encoder\n",
    "\n",
    "\n",
    "# Build Decoder\n",
    "def build_decoder(feature_dim, latent_dim) -> keras.Model:\n",
    "    latent_inputs = keras.Input(shape=(latent_dim,))\n",
    "    x = keras.layers.Dense(feature_dim, kernel_initializer='glorot_uniform', activation='sigmoid')(latent_inputs)\n",
    "    decoder_outputs = x\n",
    "    decoder = keras.Model(latent_inputs, decoder_outputs, name=\"decoder\")\n",
    "    return decoder\n",
    "\n",
    "\n",
    "# Parameters\n",
    "latent_dim = 50\n",
    "learning_rate = 0.001\n",
    "epochs = 30\n",
    "epochs = 5\n",
    "batch_size = 128\n",
    "\n",
    "# if __name__ == '__main__':\n",
    "# parser = ArgumentParser()\n",
    "# parser.add_argument(\"-f\", \"--file\", action=\"store\", type=str, required=True)\n",
    "\n",
    "# args = parser.parse_args()\n",
    "\n",
    "                                #############  ^ This could all be a separate, collapsed cell  ############\n",
    "\n",
    "# file: str = args.file\n",
    "file = 'data/cl_cp_prot_850.tsv' # <^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^Hard code file read\n",
    "\n",
    "# d_typ_obj = pd.read_csv('data/cl_cp_cnvr_996.tsv', sep='\\t', index_col=0)\n",
    "df = pd.read_csv(file, sep='\\t', index_col=0)\n",
    "\n",
    "# v0 - scale data for baseline encoding\n",
    "selected_df = df.iloc[:, 2:]\n",
    "\n",
    "feature_count = selected_df.shape[1]\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "selected_df = pd.DataFrame(\n",
    "    scaler.fit_transform(selected_df),\n",
    "    columns=selected_df.columns,\n",
    "    index=selected_df.index)\n",
    "\n",
    "selected_df[\"model_type\"] = df[\"model_type\"]\n",
    "selected_df[\"cancer_type\"] = df[\"cancer_type\"]\n",
    "\n",
    "selected_df[\"model_type\"] = model_type_encoder.fit_transform(selected_df[\"model_type\"])\n",
    "# convert model type to int\n",
    "selected_df[\"model_type\"] = selected_df[\"model_type\"].astype(int)\n",
    "assert selected_df[\"model_type\"].nunique() == 2, \"There should be two classes\"\n",
    "\n",
    "selected_df[\"cancer_type\"] = cancer_type_encoder.fit_transform(selected_df[\"cancer_type\"])\n",
    "# convert model type to int\n",
    "selected_df[\"cancer_type\"] = selected_df[\"cancer_type\"].astype(int)\n",
    "\n",
    "print(selected_df.shape)\n",
    "\n",
    "# Build VAE\n",
    "encoder = build_encoder(feature_count, latent_dim)  # feat count set above, lat dim is a var\n",
    "decoder = build_decoder(feature_count, latent_dim)\n",
    "vae = VAE(encoder, decoder, columns=selected_df.columns)\n",
    "vae.compile(optimizer=keras.optimizers.Adam(learning_rate=learning_rate), run_eagerly=True)\n",
    "\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
    "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=Path(log_dir, \"weights\"),\n",
    "                                                 save_weights_only=True,\n",
    "                                                 verbose=1)\n",
    "\n",
    "history = vae.fit(selected_df, epochs=epochs, batch_size=batch_size, shuffle=True,\n",
    "                  callbacks=[tensorboard_callback, cp_callback])\n",
    "\n",
    "# save history\n",
    "history_df = pd.DataFrame(history.history)\n",
    "# history_df.to_csv(Path(log_dir, \"history.tsv\", sep='\\t'), index=False) # sep in bracket, comma delim?\n",
    "history_df.to_csv(Path(log_dir, \"history.tsv\"), sep='\\t', index=False)\n",
    "# save model\n",
    "vae.encoder.save(Path(log_dir, \"encoder\"))\n",
    "vae.decoder.save(Path(log_dir, \"decoder\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff9dea3c-1593-4327-b274-89d20bf65d0a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Loss curve plot, #0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10c47656-fe5e-47ab-9bdc-83384dddad3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add cosine similarity label to distance loss curve sub component"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b88e6229-8962-435b-a4fb-5455a730cd68",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecb59569-4a42-41d9-8f13-552140ac7e16",
   "metadata": {},
   "outputs": [],
   "source": [
    "prot_hist = pd.read_csv('logs/20231026-121134/history.tsv', sep = '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ff81652-c170-4c71-868d-4d84b8c0519a",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_title = 'proteomics'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e27f25f4-1646-4e30-982a-7e9a2ac7687b",
   "metadata": {},
   "outputs": [],
   "source": [
    "prot_hist # Observations\n",
    "            # recon flatlines, KL goes down, distance becomes increasingly negative\n",
    "            # Thoughts\n",
    "                # Why is distance negative?\n",
    "                # How big a priority is building in the cancer type loss?\n",
    "                # How difficult is it to build in the cancer type loss?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1db60853-9e8b-42ca-90bb-1944ec07875e",
   "metadata": {},
   "outputs": [],
   "source": [
    "prot_hist.to_csv('prompt_files/proteomic_encoding_loss_data.tsv', sep = '\\t', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03a5b928-34ed-4660-abd8-b9fa61395dcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_loss(loss_dict, latent_dim, selected_df,\n",
    "              epochs, plat, data_type, plot_title):\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.plot(loss_dict['cycle_n'], marker='o',\n",
    "             linestyle='-', color='b')\n",
    "    plt.grid(True, linestyle='--', alpha=0.6)\n",
    "    plt.title(plot_title+' '+data_type+' VAE loss',\n",
    "              fontsize=16, fontweight='bold')\n",
    "    plt.xlabel('Epoch', fontsize=14)\n",
    "    plt.ylabel('Loss', fontsize=14)\n",
    "    plt.annotate(\n",
    "        'Samples: {}'.format(\n",
    "            len(selected_df))+'\\nLatent dim: '+str(\n",
    "            latent_dim)+'\\nRaw features: {}'.format(\n",
    "            len(selected_df.columns)), \n",
    "                 xy=(0.7, 0.3), xycoords='axes fraction', \n",
    "                 bbox=dict(boxstyle='round, pad=0.5',\n",
    "                           fc='white', ec='black'),\n",
    "                 fontsize=12)\n",
    "    plt.savefig(\n",
    "        '../results_encoded/'+plat+'_'+data_name+'_dim.'+str(\n",
    "            latent_dim)+'_epochs.'+str(\n",
    "            epochs)+'_vae_loss.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b2b41d3-17e7-4e40-b76e-163eca98814b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create main plot for overall loss\n",
    "xlab = 'Epoch'\n",
    "axis_font_size = 18\n",
    "\n",
    "fig, main_ax = plt.subplots(figsize=(10, 8))\n",
    "main_ax.plot(df['loss'], label='Total Loss', color='blue')\n",
    "main_ax.set_title('Overall Loss and Individual Loss Components, ' + plot_title, fontsize = 20)\n",
    "main_ax.set_xlabel(xlab, fontsize = axis_font_size)\n",
    "main_ax.set_ylabel('Total Loss', fontsize = axis_font_size)\n",
    "main_ax.legend()\n",
    "\n",
    "width, height = .25, .25\n",
    "a, b = .3, .65\n",
    "inset_title_font_size = 14\n",
    "\n",
    "# Reconstruction loss, upper left\n",
    "ax1 = main_ax.inset_axes([a, b, width, height])  # x, y, width, height\n",
    "ax1.plot(df['reconstruction_loss'], label='Recon Loss', color='green')\n",
    "ax1.set_title('Reconstruction Loss', fontsize = inset_title_font_size)\n",
    "ax1.set_xlabel(xlab)\n",
    "ax1.set_ylabel('Recon Loss')\n",
    "\n",
    "# KL loss, upper right\n",
    "ax2 = main_ax.inset_axes([b, b, width, height])\n",
    "ax2.plot(df['kl_loss'], label='KL Loss', color='red')\n",
    "ax2.set_title('KL Loss', fontsize = inset_title_font_size)\n",
    "ax2.set_xlabel(xlab)\n",
    "ax2.set_ylabel('KL Loss')\n",
    "\n",
    "# Distance loss, lower left - move to lower right\n",
    "ax3 = main_ax.inset_axes([b, a-.02, width, height])\n",
    "ax3.plot(df['distance_loss'], label='Distance Loss', color='orange')\n",
    "ax3.set_title('Distance Loss', fontsize = inset_title_font_size)\n",
    "ax3.set_xlabel(xlab)\n",
    "ax3.set_ylabel('Distance Loss')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('loss_plots/prot_add_auto_file_naming_dvl_1.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d9e890b-d746-4852-a4a7-47faf47169d3",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Quant setups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d083d556-fe39-4be7-9dde-e02e5e04e520",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Non-redundant here for all 5 plots, want specific import section for each of the 5 plot sections\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "import umap\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, f1_score\n",
    "\n",
    "from scipy.spatial.distance import euclidean\n",
    "from matplotlib.colors import ListedColormap\n",
    "from joypy import joyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68596fea-490b-445a-a900-00b07c7b48fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "grey_colors = pd.read_csv('plot_color_files/grey_scale.tsv', sep = '\\t', index_col = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46394f7b-fd74-437b-8c00-328d8931223a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TCGA colors on plots, build versions for alt y-mappings\n",
    "cancer_type_abbreviation_mapping = {\n",
    "    'Clear cell renal cell carcinoma': 'KIRCKICH', # combined-in chromophobe\n",
    "    'Colon adenocarcinoma': 'COADREAD', # combined-in rectal adeno\n",
    "    'Pancreatic ductal adenocarcinoma': 'PAAD',\n",
    "    'Breast carcinoma': 'BRCA',\n",
    "    'Glioblastoma': 'LGGGBM', # combined-in low grade glioma\n",
    "    'Lung squamous cell carcinoma': 'LUSC',\n",
    "    'Lung adenocarcinoma': 'LUAD',\n",
    "    'Endometrial carcinoma': 'UCEC',\n",
    "    'Head and neck squamous cell carcinoma': 'HNSC',\n",
    "    'Ovarian carcinoma': 'OV'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb8616a0-3695-4a15-a43b-c2e421c6bc64",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Template for unenc_v0.2, encdd_v0.2\n",
    "def plot_func_n(inp_obj_n):\n",
    "        do something\n",
    "        return plot written to disk\n",
    "\n",
    "plot_func_n(unenc)\n",
    "plot_func_n(encdd)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f27a008a-af2e-4e8f-8e37-c68dd5eedfee",
   "metadata": {},
   "source": [
    "A plot design change should update both the pre and post plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3156eab-dbc8-42fe-8111-cb61c8557016",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build unenc_v0.2 from this? (and front_end_plots_v0.1.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44bafeee-b8b7-4de5-9ba1-7c1569716466",
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_typ -> dstnc_typ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "962d6a63-1f6c-4237-ba07-cf9879430d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = tf.keras.models.load_model(\"logs/20231026-121134/encoder\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2474cfc-74ca-44d5-9b28-301594650609",
   "metadata": {},
   "source": [
    "## Plot labeling, naming, disk reads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e3bfcc8-443a-4169-8c21-3e493df3c5ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_dim = 50 # Already in RAM if VAE run\n",
    "epochs = 5      # ' ' ' ' '"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cccb809-2798-4b2c-bc36-647cbb7029d7",
   "metadata": {},
   "source": [
    "### Toggle zero  \n",
    "Toggle after plot two, back after plot four"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67b8eccf-0329-4156-bfdb-3211e45079f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'model type'\n",
    "mode = 'model_type'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41a156ea-b92f-48e3-b9d4-5242017f37bf",
   "metadata": {},
   "source": [
    "### Toggle one  \n",
    "Leave for all five plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7a6c73e-af6e-4e7b-a3a5-ca1a390d4d1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl = 'Cell line + CPTAC'\n",
    "mdls = 'cptac_+_cell_line'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e123186c-7dc0-47a6-8f5e-c862a54e1b18",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl = 'Cell line + CPTAC + HCMI'\n",
    "mdls = 'cptac_+_cell_line_+_hcmi'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07234555-2ccc-4fd2-8406-c4d705fa1ae6",
   "metadata": {},
   "source": [
    "### Toggle set two  \n",
    "Leave for all five plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccffbca5-51df-457e-baa6-a43dcddde2ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'copy number'\n",
    "dta_typ = 'cnvr'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_cnvr_996.tsv',\n",
    "                       sep = '\\t', index_col = 0)\n",
    "dta_typ_obj = pd.DataFrame(encoder.predict(dta_typ_unenc.iloc[:, 2:])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58b4c975-b49a-4d3c-9ef7-f0b613b6d472",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'gene expression'\n",
    "dta_typ = 'gexp'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_gexp_998.tsv',\n",
    "                       sep = '\\t', index_col = 0)\n",
    "dta_typ_obj = pd.DataFrame(encoder.predict(dta_typ_unenc.iloc[:, 2:])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0b660e7-8be2-4c6e-b1f7-56817f656f14",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_ttl = 'proteomics'\n",
    "dta_typ = 'prot'\n",
    "dta_typ_unenc = pd.read_csv('data/cl_cp_prot_850.tsv',\n",
    "                   sep = '\\t', index_col = 0)\n",
    "dta_typ_obj = pd.DataFrame(encoder.predict(dta_typ_unenc.iloc[:, 2:])[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c36ffa7b-4be3-4060-98e1-ae1123e01c67",
   "metadata": {},
   "source": [
    "### Continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9ee08fb-bad7-49ee-b9ce-48fe107596f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dta_ttl)\n",
    "dta_typ_obj.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e4f3315-5965-45fd-836b-6698f2c8f71e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Labels onto latent object\n",
    "feature_columns = dta_typ_obj.columns\n",
    "dta_typ_obj['cancer_type'] =  list(dta_typ_unenc['cancer_type'])\n",
    "dta_typ_obj['model_type'] =  list(dta_typ_unenc['model_type'])\n",
    "new_cols = ['cancer_type', 'model_type'] + list(feature_columns)\n",
    "dta_typ_obj = dta_typ_obj[new_cols]\n",
    "dta_typ_obj.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7c8395a-08bc-4121-bbac-537a2921fdf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.columns # Why does this run with numeric column names the logreg function now?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81ba3493-09ba-4441-8cc9-3cab66e0fecf",
   "metadata": {},
   "source": [
    "## UMAP #1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa8ef009-ea8a-42f3-babb-a3f274f4f376",
   "metadata": {},
   "outputs": [],
   "source": [
    "reducer = umap.UMAP(n_components=2)\n",
    "numeric_cols = dta_typ_obj.select_dtypes(include=['number']).columns # Will break if reverting to dta_typ appended column headers\n",
    "scaled_data = StandardScaler().fit_transform(dta_typ_obj.loc[:, numeric_cols])\n",
    "embedding = reducer.fit_transform(scaled_data)\n",
    "emb_df = pd.DataFrame(embedding, index = dta_typ_obj.index)\n",
    "emb_lbld = pd.concat([emb_df, dta_typ_obj[['cancer_type', 'model_type']]], axis = 1)\n",
    "emb_lbld.columns = ['UMAP_1', 'UMAP_2', 'cancer_type', 'model_type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c7587e9-b1fd-44fe-ae1f-56588ed3ef86",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0) # OVERWRITE FOR INDEXING\n",
    "unique_cancer_types = emb_lbld['cancer_type'].unique()\n",
    "custom_palette = {cancer_type: tcga_colors.loc[cancer_type_abbreviation_mapping.get(cancer_type, 'Unknown'), 'cohort_color'] for cancer_type in unique_cancer_types}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7617b4b-02dd-4abd-b029-123343c141d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl = 'Cell line + CPTAC' # from version 0.1, determine where to place\n",
    "mdls = 'cptac_+_cell_line'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a722ba5-a7bf-457b-a6c0-f45bd5f914be",
   "metadata": {},
   "outputs": [],
   "source": [
    "mdls_ttl, dta_ttl, dta_typ, mdls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79dfcf94-3f20-4d38-88b1-7d177af03e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def umap_plot_to_disk(emb_lbld, mdls_ttl, dta_ttl, dta_typ, mdls):\n",
    "    tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0) # OVERWRITE FOR INDEXING\n",
    "    unique_cancer_types = emb_lbld['cancer_type'].unique()\n",
    "    custom_palette = {cancer_type: tcga_colors.loc[cancer_type_abbreviation_mapping.get(cancer_type, 'Unknown'), 'cohort_color'] for cancer_type in unique_cancer_types}\n",
    "    emb_lbld.columns = ['UMAP_1', 'UMAP_2', 'Cancer type', 'Model type']\n",
    "    plt.figure(figsize=(5, 5))\n",
    "    marker_dict = {'Tumor': '^', 'cell line': 'o'}\n",
    "    sns.scatterplot(data=emb_lbld, x='UMAP_1', y='UMAP_2',\n",
    "                    hue='Cancer type', style='Model type', markers=marker_dict,\n",
    "                    palette=custom_palette, legend='full',\n",
    "                    s = 200)\n",
    "    plt.xlabel('UMAP_2', fontsize=16)\n",
    "    plt.ylabel('UMAP_2', fontsize=16)\n",
    "    plt.legend(title='Cancer Type', loc='upper left', bbox_to_anchor=(1, 1))\n",
    "    plt.suptitle(mdls_ttl +', '+ dta_ttl, y = 1.002, fontsize = 20)\n",
    "    plt.title('n = '+str(len(emb_lbld)), fontsize = 18)\n",
    "    legend = plt.legend(title='Sample attributes', title_fontsize='14', loc='upper left', bbox_to_anchor=(1, 1), fontsize=12)\n",
    "    headers_to_bold = ['Cancer type', 'Model type']\n",
    "    for text in legend.texts:\n",
    "        if text.get_text() in headers_to_bold:\n",
    "            text.set_weight('bold')\n",
    "    plt.savefig('results_encdd/umap_function_'+dta_typ+'_'+mdls+'.png', bbox_inches = 'tight', dpi = 300)\n",
    "    return 'UMAP written to disk'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dad4db98-ab8d-49be-a266-069e8c2b1052",
   "metadata": {},
   "outputs": [],
   "source": [
    "umap_plot_to_disk(emb_lbld, mdls_ttl, dta_ttl, dta_typ, mdls)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6f35e8f-8347-4e05-812b-5118831dad37",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## LogReg function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afd814ae-822d-4f57-9759-58f5a721423c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Takes dta_typ_obj and label mode as args\n",
    "def log_reg(dta_typ_obj, mode):\n",
    "    col_X_strt = 2 # <-- this skips the dual-label columns\n",
    "    f1_stor_frm = pd.DataFrame()\n",
    "\n",
    "    # Logistic regression repeat loop, to get error\n",
    "    for i in list(range(0, 5)): # <-- turn this up to 30, get the bar stars\n",
    "        print(i)\n",
    "\n",
    "        # Train test split\n",
    "        trn = dta_typ_obj.sample(round(len(dta_typ_obj) * .8))\n",
    "        tst = dta_typ_obj.loc[~dta_typ_obj.index.isin(trn.index)]\n",
    "    \n",
    "        X_trn = trn.iloc[:, col_X_strt:]\n",
    "        X_tst = tst.iloc[:, col_X_strt:]\n",
    "\n",
    "        # Prediction targets, y is either model system or cancer type\n",
    "        y_trn = trn[mode]\n",
    "        y_tst = tst[mode]\n",
    "        \n",
    "        clf = LogisticRegression().fit(X_trn, y_trn)\n",
    "        y_pred = clf.predict(X_tst)\n",
    "        \n",
    "        f1_by_class = f1_score(y_tst, y_pred, average=None)\n",
    "        f1_df = pd.DataFrame({'Label': list(y_tst.unique()),\n",
    "                              'F1_Score': f1_by_class})\n",
    "        f1_stor_frm = pd.concat([f1_stor_frm, f1_df], axis = 0)\n",
    "    return f1_stor_frm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff6e6a9e-ab23-4d84-ab1d-b4364ce4cae7",
   "metadata": {},
   "source": [
    "## Logreg model-type plot - greyscale #2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61ac64a5-abe6-40f5-b645-480940f18718",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### setup checks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14088ce8-e573-4ccb-8a78-6e762786adcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "711c244e-e6e8-4b96-99e0-11e8c16c17f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dta_ttl)\n",
    "f1_stor_frm = log_reg(dta_typ_obj, mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1b598fb-b70c-475f-90e7-e8493146c29e",
   "metadata": {},
   "source": [
    "### LogReg Grey"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38e9587a-d4db-4edc-a060-c1e90b7fb800",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Sample count devel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2a56239-3774-4ee8-aae5-c40192fb6aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.model_type.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "487a7044-0975-4c42-956c-dbcc2b99970c",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(dta_typ_obj.model_type.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1faf91c5-c95b-460a-8833-7215bfc0e17f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.model_type.value_counts().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "555331ff-fed1-4638-ae8c-d334dd00e61a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.model_type.value_counts().index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c00d7ee2-13d3-4c39-8f74-24c60a11438c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.model_type.value_counts()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acd6ed44-6008-49c4-89e7-206c0d9c6fe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.model_type.value_counts().index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef664630-9f2c-4699-b9ac-728b02492f3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dta_typ_obj.model_type.value_counts().index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcecfdd7-ff95-4746-9fad-e295ef929933",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_stor_frm.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d93f8baa-2fb8-4ffd-9276-f01242401231",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2170b9a2-59c0-417b-8016-a588d5b86bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_stor_frm.to_csv('results_encdd/log_reg_f1_'+mode+'_'+dta_typ+'_unencoded_v0.2.tsv',\n",
    "                       sep = '\\t', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "156240b3-923d-4678-b958-f2c6d5cf3069",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_stor_frm = pd.read_csv('results_encdd/log_reg_f1_'+mode+'_'+dta_typ+'_unencoded_v0.2.tsv',\n",
    "                           sep = '\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d23f16e2-0f6d-43dd-94f0-46a75c69e308",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_counts = dict(dta_typ_obj.model_type.value_counts())\n",
    "sample_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34c738e4-04d3-4236-b5d5-a8385569eafe",
   "metadata": {},
   "outputs": [],
   "source": [
    "del(sample_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "834eace3-8f2e-45d4-9169-cc1b2b5293e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_counts = dict(dta_typ_obj.model_type.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02abd913-d393-45d5-a98c-ed389743d3c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LogReg greyscale - add sample count\n",
    "def logreg_model_plot(f1_stor_frm, mdls, dta_typ, latent_dim, epochs, mode):\n",
    "    plt.figure(figsize=(8, 4.5))\n",
    "    sns.set_style(\"whitegrid\")\n",
    "    sns.set(font_scale=1.5)\n",
    "    \n",
    "    sns.barplot(x='Label', y='F1_Score', data=f1_stor_frm, palette=['#666666', '#999999'],\n",
    "               errorbar=None)\n",
    "    sns.swarmplot(x='Label', y='F1_Score', data=f1_stor_frm, color='#333333', size=14)\n",
    "    \n",
    "    plt.suptitle('Logistic regression, '+mode_ttl+', '+dta_ttl,\n",
    "                 fontsize=24, y = 1.03)\n",
    "    plt.title(mdls_ttl, fontsize=20)\n",
    "    plt.xlabel('Model Type', fontsize=20)\n",
    "    plt.ylabel('F1 Score', fontsize=20)\n",
    "\n",
    "    new_labels = [f\"{label}, n = {sample_counts[label]}\" for label in sample_counts.keys()]\n",
    "    plt.xticks(ticks=range(len(new_labels)), labels=new_labels, fontsize=20)\n",
    "    \n",
    "    plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)\n",
    "    \n",
    "    plt.savefig('results_encdd/log_reg_'+mode+'_'+dta_typ+'_load_model_v0.2_smp_cnt.png',\n",
    "                bbox_inches='tight')\n",
    "logreg_model_plot(f1_stor_frm, mdls, dta_typ, latent_dim, epochs, mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c6cb7a4-631c-41c1-b4fc-94ec85f279e1",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### LogReg string column fix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e765191a-0b48-4ee1-8e60-a5b65d7db68b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.columns = dta_typ_obj.columns.astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6305adf0-f89a-4127-8219-1bf5540ac485",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26a76859-1e64-41e9-a10c-746ad86a4ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dta_typ_obj.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8abe740-c7fb-43f2-b4e0-0cf65892b059",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_stor_frm = log_reg(dta_typ_obj, mode) # Single reuse function, no errors expected (?)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9adc195-4962-4edb-b018-c3f3e8028346",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## LogReg cancer-type plot - TCGA colors #3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e13dc9a2-4422-40cd-ac99-4c8e4de8123a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'cancer type'\n",
    "mode = 'cancer_type'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76cf0f97-e7a9-441f-9db1-a580173b2759",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dta_ttl)\n",
    "f1_stor_frm = log_reg(dta_typ_obj, mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "696a1786-ece0-4064-ae0f-2baf028a3831",
   "metadata": {},
   "outputs": [],
   "source": [
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv',\n",
    "                          sep = '\\t')\n",
    "tcga_color_mapping = dict(zip(tcga_colors['tcga_cohorts'], tcga_colors['cohort_color']))\n",
    "unique_labels = f1_stor_frm['Label'].unique()\n",
    "palette_dict = {}\n",
    "for label in unique_labels:\n",
    "    tcga_abbreviation = cancer_type_abbreviation_mapping.get(label)\n",
    "    color = tcga_color_mapping.get(tcga_abbreviation)\n",
    "    if color:\n",
    "        palette_dict[label] = color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fdaacdd-7f3a-4eb1-8ffe-b4266c9d0c6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace file naming to start with logreg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5febdaa3-93ef-467f-946e-a2db8f882ab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cancer Type Logistic Regression Plot Function, I want these functions in-line to modify with the call\n",
    "def lgrg_plt_fnc(f1_stor_frm, mdls, data_name, latent_dim, epochs, mode):\n",
    "    plt.figure(figsize=(8, 4))\n",
    "    sns.set_style(\"whitegrid\")\n",
    "    # sns.set(font_scale=2)  # Increase the font scale for larger fonts\n",
    "    \n",
    "    ax = sns.barplot(\n",
    "        x='Label', y='F1_Score', data=f1_stor_frm,\n",
    "        palette=palette_dict,\n",
    "        errorbar=None)\n",
    "    sns.swarmplot(x='Label', y='F1_Score', data=f1_stor_frm,\n",
    "                  color='#333333', size=8)  # Increase point size\n",
    "    \n",
    "    plt.suptitle('Logistic regression, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=24, y = 1.04)  # Increase title font \n",
    "    plt.title(mdls_ttl, fontsize=20)\n",
    "    plt.xlabel('Cancer type', fontsize=20)  # Increase x-axis label font size\n",
    "    plt.ylabel('F1 Score', fontsize=20)  # Increase y-axis label font size\n",
    "    \n",
    "    plt.xticks(rotation=45, ha = 'right', fontsize = 16)  # Rotate x-axis labels for better visibility\n",
    "    plt.grid(color='grey', linestyle='-', linewidth=0.25, alpha=0.5)\n",
    "    \n",
    "    # Modify the save path to use the provided parameters\n",
    "    plt.savefig('results_encdd/mdl_systms.'+mdls+'_'+dta_typ+'_dim.'+str(\n",
    "                latent_dim)+'_epochs.'+str(\n",
    "                epochs)+'_mode.'+mode+'_v0.2.png', bbox_inches='tight', dpi = 300)\n",
    "    # plt.show()\n",
    "lgrg_plt_fnc(f1_stor_frm, mdls, dta_typ, latent_dim, epochs, mode)    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac8f826a-277f-4292-becd-62ccf160b95d",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Euclidean, #4 & 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6621c737-b8da-4ab6-a2f7-6d6dd7b2c9c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl, mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f9708c8-92ef-4868-b22f-1047deb97351",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dta_typ_obj"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a89335-3a79-412c-9678-ec9f741273f2",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "#### Nested function devel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a045bd8-64ba-4f09-bc6b-963378d417a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Euclidean distance, pass platform mode column\n",
    "def euc_dstnc(sample, features, df):\n",
    "    other_types = df[df[mode_colmn] != sample[mode_colmn]]\n",
    "    mean_features_other_types = other_types[features].mean()\n",
    "    distance = euclidean(sample[features], mean_features_other_types)\n",
    "    return distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "157dca47-23da-4c3d-8ce2-1016978d1d8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dstnc_typs = ['cncr_typ_dstncs', 'mdl_typ_dstncs']\n",
    "mode_colmns = ['cancer_type', 'model_type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "198f3e0e-485f-479a-b922-06ad6c032917",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nested function test\n",
    "for dstnc_typ, mode_colmn in dstnc_typs, mode_colmns:\n",
    "    print(dstnc_typ, mode_colmn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21968ff5-9609-4a63-9cda-f02a18398e21",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_columns = dta_typ_obj.columns[2:]\n",
    "for dstnc_typ, mode_colmn in dstnc_typs, mode_colmns:\n",
    "    dta_typ_obj[dstnc_typ] = dta_typ_obj.apply(\n",
    "        lambda row: euc_dstnc(row, feature_columns, dta_typ_obj), axis=1)\n",
    "new_cols = ['cancer_type', 'model_type', 'cncr_typ_dstncs', 'mdl_typ_dstncs'] + list(feature_columns)\n",
    "dta_typ_obj = dta_typ_obj[new_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afdc7852-6046-4f36-9ad3-e3d41293bdd2",
   "metadata": {},
   "source": [
    "#### Double function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d261dd4-7e9d-4283-819d-500225f3910c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Euclicean distance, model type\n",
    "def mdl_typ_dist(sample, features, df):\n",
    "    other_types = df[df['model_type'] != sample['model_type']]\n",
    "    mean_features_other_types = other_types[features].mean()\n",
    "    distance = euclidean(sample[features], mean_features_other_types)\n",
    "    return distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14dbb4e6-2c1c-416c-88aa-6e54fbbd1e26",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Euclidean distance, cancer type\n",
    "def cncr_typ_dist(sample, features, df):\n",
    "    other_types = df[df['cancer_type'] != sample['cancer_type']]\n",
    "    mean_features_other_types = other_types[features].mean()\n",
    "    distance = euclidean(sample[features], mean_features_other_types)\n",
    "    return distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "681c8db2-d41b-49cd-b23c-564eafb4b2da",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_columns = dta_typ_obj.columns[2:]\n",
    "dta_typ_obj['mdl_typ_dstncs'] = dta_typ_obj.apply(\n",
    "    lambda row: mdl_typ_dist(row, feature_columns, dta_typ_obj), axis=1)\n",
    "dta_typ_obj['cncr_typ_dstncs'] = dta_typ_obj.apply(\n",
    "    lambda row: cncr_typ_dist(row, feature_columns, dta_typ_obj), axis=1)\n",
    "new_cols = ['cancer_type', 'model_type', 'cncr_typ_dstncs', 'mdl_typ_dstncs'] + list(feature_columns)\n",
    "dta_typ_obj = dta_typ_obj[new_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eef16874-9588-45a6-a9e1-cf90803308da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Devel \n",
    "def avrg_dstnc(mode, dstnc_typ):\n",
    "    average_distances = dta_typ_obj.groupby(\n",
    "        mode)[dstnc_typ].mean().sort_values(ascending=False)\n",
    "    sorted_df = dta_typ_obj.loc[dta_typ_obj[mode].isin(average_distances.index)]\n",
    "    sorted_df[mode] = pd.Categorical(\n",
    "        sorted_df[mode], categories=average_distances.index, ordered=True)\n",
    "    sorted_df = sorted_df.sort_values(mode)\n",
    "    tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0)\n",
    "    custom_color_list = [tcga_colors.loc[cancer_type_abbreviation_mapping[cancer_type],'cohort_color'] for cancer_type in average_distances.index]\n",
    "    custom_colormap = ListedColormap(custom_color_list)\n",
    "    return sorted_df, custom_colormap"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8433e6c4-9083-4f3d-939b-196b3b4028a9",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Eucldn Colrs, #4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6505bfe9-2123-4e8e-8224-f03bb3c2c32c",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'cancer type'\n",
    "mode = 'cancer_type'\n",
    "dstnc_typ = 'cncr_typ_dstncs'\n",
    "average_distances = dta_typ_obj.groupby(\n",
    "    mode)[dstnc_typ].mean().sort_values(ascending=False)\n",
    "sorted_df = dta_typ_obj.loc[dta_typ_obj[mode].isin(average_distances.index)]\n",
    "sorted_df[mode] = pd.Categorical(\n",
    "    sorted_df[mode], categories=average_distances.index, ordered=True)\n",
    "sorted_df = sorted_df.sort_values(mode)\n",
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0)\n",
    "custom_color_list = [tcga_colors.loc[cancer_type_abbreviation_mapping[cancer_type],'cohort_color'] for cancer_type in average_distances.index]\n",
    "custom_colormap = ListedColormap(custom_color_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49c4996f-1f2d-456b-8713-e35401fd7512",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d296ad4f-9633-482e-9dde-7f863824c6ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "round(average_distances.var(), 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e34e0f9c-843f-4971-8772-48413cd91c2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(average_distances.var())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a2393d8-f389-4669-b625-6bc77d7a42d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_distances.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38cdc803-eee7-4cfe-ae10-f187519cb308",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_distances.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ecee4b4-f41a-4359-8ab9-1a1c6a563e39",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79774ade-e209-4fde-8f08-894f0dea3a81",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddd52da9-2529-4a95-9bd0-bbdb5f8d1dd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "f1_stor_frm.to_csv('results_encdd/log_reg_f1_'+mode+'_'+dta_typ+'_unencoded_v0.2.tsv',\n",
    "                       sep = '\\t', index = Falea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5a5c69a-a182-496a-9ecd-ce2d27168be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e3eca9b-69c2-4707-badd-61ed00ed288e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3dc3685-3d57-4b88-88a7-96ee914a7f3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5da38ee1-d370-4c49-9fa8-440ef10993a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_values = np.linspace(0.75, 0.068, 7)\n",
    "plt.figure()\n",
    "joyplot(data=sorted_df[[mode, dstnc_typ]], by=mode,\n",
    "    figsize=(10, 6.5), colormap=custom_colormap,\n",
    "    fade=True)\n",
    "\n",
    "for y_value, cancer_type in zip(y_values, sorted_df[mode].unique()):\n",
    "    count = dict(sorted_df[mode].value_counts())[cancer_type]\n",
    "    x_position = sorted_df[dstnc_typ].max()\n",
    "    plt.annotate(f\"n={count}\", xy=(x_position, y_value), verticalalignment='center')\n",
    "\n",
    "plt.suptitle('Euclidean Distances, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=30, y = 1.01)\n",
    "plt.title(mdls_ttl+ ', n = '+str(dta_typ_obj.shape[0]),\n",
    "          y = .92, x = .22, fontsize = 26)\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "\n",
    "plt.annotate(\n",
    "    r'Variance of means: $\\mathbf{' + f'{average_distances.var():.3f}' + '}$',\n",
    "    xy=(0.01, 0.87), xycoords='axes fraction',\n",
    "    ha='right', va='top')\n",
    "\n",
    "plt.savefig('results_encdd/rdgln_'+mode+'_'+mdls+'_'+dta_typ+'_v0.2.png',\n",
    "            bbox_inches = 'tight', dpi = 300)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5efee8c-a192-49ca-8090-699fe5476b5c",
   "metadata": {},
   "source": [
    "## Eucldn Gry, #5  \n",
    "Toggle zero back to og grey setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4e313b1-95a8-451b-a9b5-ec03ca338fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_ttl = 'model type'\n",
    "mode = 'model_type'\n",
    "dist_typ = 'mdl_typ_dstncs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d6fb36c-f3fa-466e-b721-dda2eb812fd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_df, custom_colormap = avrg_dstnc(mode, dstnc_typ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "242c2ff0-6f1b-4370-bfd2-1425c34708a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "abbreviation_mapping = {\n",
    "    'cell line': 'cell line',\n",
    "    'Tumor': 'Tumor',\n",
    "    'HCMI': 'HCMI', # test\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13836b34-3c0c-4623-b0d5-496123ddee95",
   "metadata": {},
   "outputs": [],
   "source": [
    "average_distances = dta_typ_obj.groupby(\n",
    "    mode)[dstnc_typ].mean().sort_values(ascending=False)\n",
    "sorted_df = dta_typ_obj.loc[dta_typ_obj[mode].isin(average_distances.index)]\n",
    "sorted_df[mode] = pd.Categorical(\n",
    "    sorted_df[mode], categories=average_distances.index, ordered=True)\n",
    "sorted_df = sorted_df.sort_values(mode)\n",
    "tcga_colors = pd.read_csv('plot_color_files/tcga_colors.tsv', sep = '\\t', index_col = 0) #\n",
    "custom_color_list = [grey_colors.loc[\n",
    "                     abbreviation_mapping[\n",
    "                     model_type],'quant_mode_color'] for model_type in average_distances.index]\n",
    "custom_colormap = ListedColormap(custom_color_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53481f46-3058-4c74-b10e-404fd555a720",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_values = np.linspace(0.52, 0.15, 2)\n",
    "plt.figure()\n",
    "joyplot(data=sorted_df[[mode, dstnc_typ]], by=mode,\n",
    "    figsize=(10, 6.5), colormap=custom_colormap,\n",
    "    fade=True)\n",
    "\n",
    "for y_value, cancer_type in zip(y_values, sorted_df[mode].unique()):\n",
    "    count = dict(sorted_df[mode].value_counts())[cancer_type]\n",
    "    x_position = sorted_df[dstnc_typ].max()\n",
    "    plt.annotate(f\"n={count}\", xy=(x_position, y_value), verticalalignment='center', fontsize = 24)\n",
    "\n",
    "plt.suptitle('Euclidean Distances, '+mode_ttl+', '+dta_ttl,\n",
    "             fontsize=30, y = 1.06)\n",
    "plt.title(mdls_ttl+ ', n = '+str(dta_typ_obj.shape[0]),\n",
    "          y = .97, x = .4, fontsize = 26)\n",
    "\n",
    "plt.rcParams['text.usetex'] = True\n",
    "\n",
    "plt.annotate(\n",
    "    r'Variance of means: $\\mathbf{' + f'{average_distances.var():.3f}' + '}$',\n",
    "    xy=(0.2, 0.87), xycoords='axes fraction',\n",
    "    ha='right', va='top')\n",
    "\n",
    "plt.savefig('results_encdd/rdgln_'+mode+'_'+mdls+'_'+dta_typ+'_v0.2.png',\n",
    "            bbox_inches = 'tight', dpi = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10d21cd8-40fa-41ff-82af-d2fff5ed1463",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}