classifier-visualizations-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a5b1e907",
   "metadata": {},
   "source": [
    "# Importing the classifier pickle\n",
    "NOTE: pickle must contain the trained __classifier__, not just the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e08590c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import json\n",
    "import pickle\n",
    "import numpy as np\n",
    "import statistics as stats\n",
    "\n",
    "from sklearn import tree\n",
    "from pandas import DataFrame\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "# ------ Fill this in to specify which pickle should be opened ------\n",
    "# --- NOTE: should include full path and filename but NO extension ---\n",
    "file = \"stored_models/trained_classifiers/dtc/dtc-head-16-cdiac_multilabel_gt_copy.csv-2022-04-21-16.40.41\"\n",
    "\n",
    "# --------------------------------------------------------------\n",
    "\n",
    "byte_count = int(file.split(\"/\")[-1].split(\"-\")[2])\n",
    "\n",
    "classifier_path = f\"{file}.pkl\"\n",
    "info_metrics_path = f\"{file}-info-{byte_count}Bytes.json\"\n",
    "\n",
    "info_metrics = json.load(open(info_metrics_path))\n",
    "\n",
    "classifier = pickle.load(open(classifier_path, \"rb\"))\n",
    "\n",
    "for key in info_metrics.keys():\n",
    "   print(key)\n",
    "   print(f'\\t{info_metrics[key]}')\n",
    "\n",
    "precision = info_metrics['Model precision']\n",
    "recall = info_metrics['Model recall']\n",
    "print(f'F1-Score\\n\\t{2*precision*recall/(precision+recall)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "165cfbce",
   "metadata": {},
   "source": [
    "# Decision tree visualization\n",
    "Acceptable classifiers:\n",
    "- Decision tree classifier (dtc)\n",
    "- Single-tree extra tree classifier (t_etc)\n",
    "\n",
    "NOTE: If decision tree has too high of a maximum depth, this visualization will be illegible. I recommend setting max_depth to 3."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49770e61",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure(figsize=(25, 25))\n",
    "_ = tree.plot_tree(classifier.model,\n",
    "                   filled=True)\n",
    "props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\n",
    "label_legend = \"\"\n",
    "for label in classifier.unique_labels:\n",
    "    label_legend += (label + \"\\n\")\n",
    "fig.text(0.13, 0.7, label_legend[:-1], fontsize=25, bbox=props)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7f89b35",
   "metadata": {},
   "source": [
    "# HeadByte heatmap visualization\n",
    "Acceptable classifiers: any"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "479dcc10",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = classifier.model.predict(classifier.X_test)\n",
    "\n",
    "# Building a dictionary of the predicted (multi)labels and their corresponding binary lists\n",
    "\n",
    "unique_rows = np.unique(y_pred, axis=0)\n",
    "unique_dict = {}\n",
    "for unique_row in unique_rows:\n",
    "    labels = []\n",
    "    for pair in zip(unique_row, classifier.unique_labels):\n",
    "        if pair[0]:\n",
    "            labels.append(pair[1])\n",
    "    if len(labels) == 0:\n",
    "        labels.append(\"unknown\")\n",
    "    unique_dict[\", \".join(labels)] = unique_row\n",
    "    \n",
    "# Building a dictionary of all the feature info for each predicted label\n",
    "\n",
    "large_dict = {}\n",
    "for key in unique_dict.keys():\n",
    "    large_dict[key] = []\n",
    "\n",
    "for i in range(len(classifier.X_test)):\n",
    "    for key in unique_dict.keys():\n",
    "        if np.all(y_pred[i]==unique_dict[key]):\n",
    "            large_dict[key].append(classifier.X_test[i].tolist())\n",
    "            break\n",
    "\n",
    "sample_counts = \"Predicted sample counts\\n\\n\"\n",
    "for key in large_dict.keys():\n",
    "    sample_counts += key + \" : \" + str(len(large_dict[key])) + \"\\n\"\n",
    "\n",
    "print(\"The labels and their binarized lists:\")\n",
    "for key in unique_dict.keys():\n",
    "    print(key)\n",
    "    print(unique_dict[key])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc902821",
   "metadata": {},
   "source": [
    "## Average value heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0e5aafb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Averaging the features for each bit\n",
    "\n",
    "avgs = []\n",
    "for key in large_dict.keys():\n",
    "    lst = large_dict[key]\n",
    "    lst = [list(x) for x in zip(*lst)]\n",
    "    lst = [stats.mean(x) for x in lst]\n",
    "    avgs.append(lst)\n",
    "    \n",
    "# UNCOMMENT to fill in empty labels in case you run on a limited range of samples\n",
    "\n",
    "# for i in range(len(avgs)):\n",
    "#     if (len(avgs[i])==0):\n",
    "#         avgs[i] = [0.0] * byte_count\n",
    "\n",
    "# Creating the plot\n",
    "\n",
    "fig, ax1 = plt.subplots()\n",
    "\n",
    "# Label y axis side with the class name\n",
    "\n",
    "y1_axis_labels = large_dict.keys()\n",
    "ax1.set_yticks(np.arange(len(y1_axis_labels)), labels=y1_axis_labels)\n",
    "\n",
    "# Plot the heatmap set good x-axis ticks\n",
    "\n",
    "# fig.set_size_inches((2*byte_count)/10, 10.5) # UNCOMMENT FOR LARGER BYTECOUNTS\n",
    "plt.imshow(np.array(avgs))\n",
    "plt.xticks(ticks=[i for i in range(byte_count)], labels=[i+1 for i in range(byte_count)])\n",
    "\n",
    "# Adding title and sample count legend\n",
    "\n",
    "ax1.set_title(\"Average scalar value of n-th head byte for each predicted multilabel\")\n",
    "plt.tight_layout()\n",
    "\n",
    "props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\n",
    "plt.text(byte_count, -0.25, sample_counts[:-1], fontsize=10,\n",
    "         verticalalignment='top', bbox=props)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be416262",
   "metadata": {},
   "source": [
    "## Standard deviation heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ea20a2a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Finding the standard deviation for each bit\n",
    "\n",
    "stdevs = []\n",
    "y_axis_labels = []\n",
    "for key in large_dict.keys():\n",
    "    lst = large_dict[key]\n",
    "    lst = [list(x) for x in zip(*lst)]\n",
    "    if (len(lst[0])>1):\n",
    "        lst = [stats.stdev(x) for x in lst]\n",
    "        stdevs.append(lst)\n",
    "        y_axis_labels.append(key)\n",
    "    \n",
    "# UNCOMMENT to fill in empty labels in case you run on a limited range of samples\n",
    "\n",
    "# for i in range(len(stdevs)):\n",
    "#     if (len(stdevs[i])==0):\n",
    "#         stdevs[i] = [0.0] * byte_count\n",
    "\n",
    "# Creating the plot\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "im = ax.imshow(np.array(stdevs))\n",
    "\n",
    "# Show all ticks and label them with the respective list entries\n",
    "\n",
    "ax.set_yticks(np.arange(len(y_axis_labels)), labels=y_axis_labels)\n",
    "\n",
    "# Plot the heatmap set good x-axis ticks\n",
    "\n",
    "# fig.set_size_inches(18.5, 10.5) # UNCOMMENT FOR LARGER BYTECOUNTS\n",
    "plt.imshow(np.array(stdevs))\n",
    "plt.xticks(ticks=[i for i in range(byte_count)], labels=[i+1 for i in range(byte_count)])\n",
    "\n",
    "# Adding title and sample count legend\n",
    "\n",
    "ax1.set_title(\"Standard deviation of scalar values of n-th head byte\")\n",
    "plt.tight_layout()\n",
    "\n",
    "props = dict(boxstyle='round', facecolor='wheat', alpha=0.5)\n",
    "plt.text(byte_count, -0.25, sample_counts[:-1], fontsize=10,\n",
    "         verticalalignment='top', bbox=props)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1071c819",
   "metadata": {},
   "source": [
    "# Multilabel Confusion Matrices\n",
    "Acceptable classifiers: any"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73884a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "def print_confusion_matrix(confusion_matrix, axes, class_label, class_names, fontsize=14):\n",
    "\n",
    "    df_cm = pd.DataFrame(\n",
    "        confusion_matrix, index=class_names, columns=class_names,\n",
    "    )\n",
    "\n",
    "    try:\n",
    "        heatmap = sns.heatmap(df_cm, annot=True, fmt=\"d\", cbar=False, ax=axes, cmap='crest', annot_kws={\"fontsize\":16})\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Confusion matrix values must be integers.\")\n",
    "    heatmap.yaxis.set_ticklabels(heatmap.yaxis.get_ticklabels(), rotation=0, ha='right', fontsize=16)\n",
    "    heatmap.xaxis.set_ticklabels(heatmap.xaxis.get_ticklabels(), rotation=45, ha='right', fontsize=16)\n",
    "    axes.set_ylabel('True label')\n",
    "    axes.set_xlabel('Predicted label')\n",
    "    axes.set_title(class_label, fontsize=20)\n",
    "\n",
    "y_pred = classifier.model.predict(classifier.X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85aef8d0",
   "metadata": {},
   "source": [
    "## Individual Label Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f802575",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import multilabel_confusion_matrix\n",
    "\n",
    "# Some customizable values here, first two values should multiply to at least label count\n",
    "fig, ax = plt.subplots(2, 5, figsize=(20, 7))\n",
    "    \n",
    "multilabel_confusion_matrix = multilabel_confusion_matrix(y_pred, classifier.Y_test)\n",
    "multilabel_labels = classifier.unique_labels\n",
    "\n",
    "for axes, cfs_matrix, label in zip(ax.flatten(), multilabel_confusion_matrix, multilabel_labels):\n",
    "    print_confusion_matrix(cfs_matrix, axes, label, [\"N\", \"Y\"])\n",
    "    \n",
    "    axes.xaxis.label.set_size(18)\n",
    "    axes.yaxis.label.set_size(18)\n",
    "    axes.title\n",
    "    \n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5bdd001",
   "metadata": {},
   "source": [
    "## Predicted Multilabel Matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "223d2f7d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import sympy\n",
    "import primefac\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import multilabel_confusion_matrix\n",
    "\n",
    "unique_rows = np.unique(np.concatenate([y_pred, classifier.Y_test]), axis=0)\n",
    "row_hashtable = dict()\n",
    "\n",
    "primes_master = []\n",
    "for i in range(len(unique_rows[0])):\n",
    "    primes_master.append(sympy.prime(i+1))\n",
    "    \n",
    "def row_hasher(row):\n",
    "    primes = []\n",
    "    hash_labels = []\n",
    "    for i in range(len(row)):\n",
    "        if row[i] == 1:\n",
    "            primes.append(primes_master[i])\n",
    "    return np.prod(primes) if (primes != []) else 0\n",
    "\n",
    "y_pred = classifier.model.predict(classifier.X_test)\n",
    "y_pred_hashed = []\n",
    "Y_test_hashed = []\n",
    "\n",
    "for i in range(len(y_pred)):\n",
    "    y_pred_hashed.append(row_hasher(y_pred[i]))\n",
    "    Y_test_hashed.append(row_hasher(classifier.Y_test[i]))\n",
    "\n",
    "unique_hashes = y_pred_hashed + Y_test_hashed\n",
    "unique_hashes = np.unique(unique_hashes)\n",
    "hash_to_multiclass_dict = dict()\n",
    "hash_to_multiclass_dict[0] = b = [0] * len(unique_hashes)\n",
    "\n",
    "for i in range(len(unique_hashes)):\n",
    "    b = [0] * len(unique_hashes)\n",
    "    b[i] = 1\n",
    "    hash_to_multiclass_dict[unique_hashes[i]] = b\n",
    "    \n",
    "multiclass_y_pred = []\n",
    "multiclass_Y_test = []\n",
    "\n",
    "for i in range(len(y_pred)):\n",
    "    curr_mc_ypred = hash_to_multiclass_dict[y_pred_hashed[i]]\n",
    "    curr_mc_Ytest = hash_to_multiclass_dict[Y_test_hashed[i]]\n",
    "    multiclass_y_pred.append(curr_mc_ypred)\n",
    "    multiclass_Y_test.append(curr_mc_Ytest)\n",
    "\n",
    "labels_gt_dict = dict()\n",
    "for i in range(len(unique_rows)):\n",
    "    currlabels = []\n",
    "    for j in range(len(unique_rows[0])):\n",
    "        if unique_rows[i][j] == 1:\n",
    "            currlabels.append(classifier.unique_labels[j])\n",
    "    labels_gt_dict[row_hasher(unique_rows[i])] = \", \".join(currlabels)\n",
    "labels_gt_dict[0] = 'UNKNOWN'\n",
    "    \n",
    "# Some customizable values here, first two values should multiply to at least label count\n",
    "fig, ax = plt.subplots(3, 5, figsize=(20, 14))\n",
    "    \n",
    "multilabel_confusion_matrix = multilabel_confusion_matrix(multiclass_y_pred, multiclass_Y_test)\n",
    "multilabel_labels = [val[1] for val in sorted(labels_gt_dict.items())]\n",
    "\n",
    "for axes, cfs_matrix, label in zip(ax.flatten(), multilabel_confusion_matrix, multilabel_labels):\n",
    "    print_confusion_matrix(cfs_matrix, axes, label, [\"N\", \"Y\"])\n",
    "    \n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54795907",
   "metadata": {},
   "source": [
    "# Precision-Recall Curve\n",
    "NOTE: This creates new test and train datasets and trains a new OneVsRest Classifier, so the imported classifier model is irrelevant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0262b0c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import PrecisionRecallDisplay\n",
    "\n",
    "random_state = np.random.RandomState(0)\n",
    "\n",
    "y = []\n",
    "y.extend(classifier.Y_train)\n",
    "y.extend(classifier.Y_test)\n",
    "\n",
    "X = []\n",
    "X.extend(classifier.X_train)\n",
    "X.extend(classifier.X_test)\n",
    "\n",
    "# Use label_binarize to be multi-label like settings\n",
    "Y = label_binarize(y, classes=[num for num in range(len(classifier.Y_train[0]))])\n",
    "n_classes = Y.shape[1]\n",
    "\n",
    "# Split into training and test\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(\n",
    "    X, Y, test_size=0.33, random_state=random_state\n",
    ")\n",
    "\n",
    "ovr_classifier = OneVsRestClassifier(\n",
    "    make_pipeline(StandardScaler(), LinearSVC(random_state=random_state))\n",
    ")\n",
    "ovr_classifier.fit(X_train, Y_train)\n",
    "y_score = ovr_classifier.decision_function(X_test)\n",
    "\n",
    "# For each class\n",
    "precision = dict()\n",
    "recall = dict()\n",
    "average_precision = dict()\n",
    "for i in range(n_classes):\n",
    "    precision[i], recall[i], _ = precision_recall_curve(Y_test[:, i], y_score[:, i])\n",
    "    average_precision[i] = average_precision_score(Y_test[:, i], y_score[:, i])\n",
    "\n",
    "# A \"micro-average\": quantifying score on all classes jointly\n",
    "precision[\"micro\"], recall[\"micro\"], _ = precision_recall_curve(\n",
    "    Y_test.ravel(), y_score.ravel()\n",
    ")\n",
    "average_precision[\"micro\"] = average_precision_score(Y_test, y_score, average=\"micro\")\n",
    "\n",
    "display = PrecisionRecallDisplay(\n",
    "    recall=recall[\"micro\"],\n",
    "    precision=precision[\"micro\"],\n",
    "    average_precision=average_precision[\"micro\"],\n",
    ")\n",
    "display.plot()\n",
    "_ = display.ax_.set_title(\"Micro-averaged over all classes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89a62639",
   "metadata": {},
   "source": [
    "# T-distributed Stochastic Neighbor Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e01f07a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "from time import time\n",
    "from sklearn import manifold, datasets\n",
    "from matplotlib.ticker import NullFormatter\n",
    "from mpl_toolkits import mplot3d\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "X = []\n",
    "X.extend(classifier.X_train)\n",
    "X.extend(classifier.X_test)\n",
    "\n",
    "y = []\n",
    "y.extend(classifier.Y_train)\n",
    "y.extend(classifier.Y_test)\n",
    "unique_rows = np.unique(y, axis=0)\n",
    "\n",
    "unique_dict = dict()\n",
    "for unique_row in unique_rows:\n",
    "    labels = []\n",
    "    for pair in zip(unique_row, classifier.unique_labels):\n",
    "        if (pair[0]==1):\n",
    "            labels.append(pair[1])\n",
    "    if len(labels) == 0:\n",
    "        labels.append(\"unknown\")\n",
    "    unique_dict[\", \".join(labels)] = unique_row\n",
    "\n",
    "organized_features = dict()\n",
    "for key in unique_dict.keys():\n",
    "    organized_features[key] = []\n",
    "for i in range(len(X)):\n",
    "    for key in unique_dict.keys():\n",
    "        if (np.array_equal(y[i],unique_dict[key])):\n",
    "            organized_features[key].append(X[i])\n",
    "\n",
    "\n",
    "fig = plt.figure(figsize=(8, 6))\n",
    "ax = plt.axes(projection='3d')\n",
    "cmap = matplotlib.cm.get_cmap('hsv')\n",
    "\n",
    "all_x, all_y, all_z = [], [], []\n",
    "ci = 0\n",
    "for key in organized_features.keys():\n",
    "    tsne = manifold.TSNE(\n",
    "        n_components=3,\n",
    "        init=\"random\",\n",
    "        random_state=0,\n",
    "        perplexity=50,\n",
    "        learning_rate=\"auto\",\n",
    "        n_iter=300,\n",
    "        angle=0.5\n",
    "    )\n",
    "    tsne_embedded = tsne.fit_transform(np.array(organized_features[key][:1000]))\n",
    "    \n",
    "    xdata = [el[0] for el in tsne_embedded]\n",
    "    ydata = [el[1] for el in tsne_embedded]\n",
    "    zdata = [el[2] for el in tsne_embedded]\n",
    "    \n",
    "    color = cmap(ci/len(organized_features))\n",
    "    ci += 1\n",
    "    ax.scatter3D(xdata, ydata, zdata, color=color, label=key)\n",
    "    \n",
    "    all_x.extend(xdata)\n",
    "    all_y.extend(ydata)\n",
    "    all_z.extend(zdata)\n",
    "\n",
    "def calculate_bounds(data):\n",
    "    std = np.std(data)\n",
    "    mean = np.mean(data)\n",
    "    return mean-3*std, mean+3*std\n",
    "#     q75, q25 = np.percentile(data, [75 ,25])\n",
    "#     iqr = q75 - q25\n",
    "#     upper_bound = iqr * 1.5 + q75\n",
    "#     lower_bound = iqr * (-1.5) + q25\n",
    "#     return lower_bound, upper_bound\n",
    "\n",
    "xlb, xub = calculate_bounds(all_x)\n",
    "ylb, yub = calculate_bounds(all_y)\n",
    "zlb, zub = calculate_bounds(all_z)\n",
    "ax.set_xlim(xlb, xub)\n",
    "ax.set_ylim(ylb, yub)\n",
    "ax.set_zlim(zlb, zub)\n",
    "\n",
    "ax.legend()\n",
    "plt.legend(bbox_to_anchor=(1.05, 1), loc='lower right', borderaxespad=0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml1",
   "language": "python",
   "name": "ml1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}