10.1 - CICD Pipeline Custom Time Series Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef2a9de8-37cd-4a22-9ae4-1f9617a93ec5",
   "metadata": {},
   "source": [
    "## Part X.I - Configure CI/CD Pipeline for Custom Model\n",
    "\n",
    "University of San Diego - MS Applied AI\n",
    "\n",
    "AAI-540 Team 5\n",
    "\n",
    "October 21, 2024"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "293b053d-faf7-42c3-95d8-02640fd86084",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Environment Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "id": "c09f7a37-49c1-4cca-a2ab-e3e83b7ba5e4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0mStored 's3_datalake_path_csv' (str)\n",
      "Stored 'local_data_path_csv' (str)\n",
      "Stored 's3_datalake_path_parquet' (str)\n"
     ]
    }
   ],
   "source": [
    "# Setup environment\n",
    "%run 0-Environment_Setup.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e8e60ff-13eb-4057-b0ff-19a0f173925b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Upgrade sagemaker version\n",
    "!pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "83e6baa5-cf0e-41cb-8ddd-f8229dace1cb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Import modules\n",
    "import sys\n",
    "\n",
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker.workflow.pipeline_context import PipelineSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "3dc1bffb-aacd-452c-8f85-859e73f5ccaf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set session variables\n",
    "sm_client = boto3.client('sagemaker', region_name=region)\n",
    "sagemaker_session = sagemaker.Session()\n",
    "role = sagemaker.get_execution_role()\n",
    "region = sagemaker_session.boto_session.region_name\n",
    "bucket = sess.default_bucket()\n",
    "pipeline_session = PipelineSession()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "c9090b1a-3a5b-4d89-b0b8-7ad6c988a690",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://sagemaker-us-east-1-342408968837/store-sales-forecasting/pipelines/input/\n",
      "s3://sagemaker-us-east-1-342408968837/store-sales-forecasting/pipelines/output/\n"
     ]
    }
   ],
   "source": [
    "# Set model package group name to register successful models to\n",
    "model_package_group_name = \"custom-model-package-group\"\n",
    "\n",
    "# Set input and output S3 paths for the pipelines\n",
    "base_path = f\"s3://{bucket}/store-sales-forecasting/pipelines\"\n",
    "input_data_path = f\"{base_path}/input/\"\n",
    "output_data_path = f\"{base_path}/output/\"\n",
    "print(input_data_path)\n",
    "print(output_data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "id": "2dd1f169-242d-4316-8d18-34a00f77e25d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running \n",
      "    SELECT *\n",
      "    FROM\n",
      "        \"store_sales_feature_group_offline_1728878780\"\n",
      "    ORDER BY\n",
      "        date ASC, store_nbr ASC\n",
      "    \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker:Query f3b608c4-8584-480c-9912-34c19b4baa57 is being executed.\n",
      "INFO:sagemaker:Query f3b608c4-8584-480c-9912-34c19b4baa57 successfully executed.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>store_nbr</th>\n",
       "      <th>sales</th>\n",
       "      <th>oil</th>\n",
       "      <th>onpromotion</th>\n",
       "      <th>is_holiday</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>cluster</th>\n",
       "      <th>year</th>\n",
       "      <th>...</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>dow_cos</th>\n",
       "      <th>dow_sin</th>\n",
       "      <th>sales_record_id</th>\n",
       "      <th>event_time</th>\n",
       "      <th>write_time</th>\n",
       "      <th>api_invocation_time</th>\n",
       "      <th>is_deleted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:1</td>\n",
       "      <td>1.728879e+09</td>\n",
       "      <td>2024-10-14 04:11:58.235</td>\n",
       "      <td>2024-10-14 04:06:44.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:2</td>\n",
       "      <td>1.728879e+09</td>\n",
       "      <td>2024-10-14 04:11:58.188</td>\n",
       "      <td>2024-10-14 04:06:45.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>8</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:3</td>\n",
       "      <td>1.728879e+09</td>\n",
       "      <td>2024-10-14 04:11:58.119</td>\n",
       "      <td>2024-10-14 04:06:45.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>9</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:4</td>\n",
       "      <td>1.728879e+09</td>\n",
       "      <td>2024-10-14 04:11:57.992</td>\n",
       "      <td>2024-10-14 04:06:45.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>21</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:5</td>\n",
       "      <td>1.728879e+09</td>\n",
       "      <td>2024-10-14 04:11:58.307</td>\n",
       "      <td>2024-10-14 04:06:45.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         date  store_nbr  sales    oil  onpromotion  is_holiday  city  state  \\\n",
       "0  2013-01-01          1    0.0  93.14            0           1    18     12   \n",
       "1  2013-01-01          2    0.0  93.14            0           1    18     12   \n",
       "2  2013-01-01          3    0.0  93.14            0           1    18     12   \n",
       "3  2013-01-01          4    0.0  93.14            0           1    18     12   \n",
       "4  2013-01-01          5    0.0  93.14            0           1    21     14   \n",
       "\n",
       "   cluster  year  ...  month_sin  day_cos   day_sin  dow_cos   dow_sin  \\\n",
       "0       13  2013  ...        0.5  0.97953  0.201299  0.62349  0.781831   \n",
       "1       13  2013  ...        0.5  0.97953  0.201299  0.62349  0.781831   \n",
       "2        8  2013  ...        0.5  0.97953  0.201299  0.62349  0.781831   \n",
       "3        9  2013  ...        0.5  0.97953  0.201299  0.62349  0.781831   \n",
       "4        4  2013  ...        0.5  0.97953  0.201299  0.62349  0.781831   \n",
       "\n",
       "   sales_record_id    event_time               write_time  \\\n",
       "0     2013-01-01:1  1.728879e+09  2024-10-14 04:11:58.235   \n",
       "1     2013-01-01:2  1.728879e+09  2024-10-14 04:11:58.188   \n",
       "2     2013-01-01:3  1.728879e+09  2024-10-14 04:11:58.119   \n",
       "3     2013-01-01:4  1.728879e+09  2024-10-14 04:11:57.992   \n",
       "4     2013-01-01:5  1.728879e+09  2024-10-14 04:11:58.307   \n",
       "\n",
       "       api_invocation_time  is_deleted  \n",
       "0  2024-10-14 04:06:44.000       False  \n",
       "1  2024-10-14 04:06:45.000       False  \n",
       "2  2024-10-14 04:06:45.000       False  \n",
       "3  2024-10-14 04:06:45.000       False  \n",
       "4  2024-10-14 04:06:45.000       False  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 306,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pull the data from the feature store sorted by date and then store number\n",
    "sales_features_store_df = get_store_dataset_from_offline_feature_group_date_sort(store_sales_feature_group)\n",
    "sales_features_store_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "id": "f2b42583-4c48-4688-a8c5-a89eb5ae1123",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Save the data from the feature store locally and upload to the pipeline input S3 path\n",
    "sales_features_store_df.to_csv(\"input_data.csv\")\n",
    "!aws s3 cp \"input_data.csv\" $input_data_path"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8277cd9-ab8f-4b21-9fb3-c336ff6e848d",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Pipeline Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "id": "868de40e-aed2-4a7b-a883-737c22b104b7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.workflow.parameters import (\n",
    "    ParameterInteger,\n",
    "    ParameterString,\n",
    "    ParameterFloat,\n",
    ")\n",
    "\n",
    "# Set pipeline parameters\n",
    "processing_instance_count = ParameterInteger(name=\"ProcessingInstanceCount\", default_value=1)\n",
    "instance_type = ParameterString(name=\"TrainingInstanceType\", default_value=\"ml.m5.xlarge\")\n",
    "model_approval_status = ParameterString(\n",
    "    name=\"ModelApprovalStatus\", default_value=\"PendingManualApproval\"\n",
    ")\n",
    "input_data = ParameterString(\n",
    "    name=\"InputData\",\n",
    "    default_value=input_data_path,\n",
    ")\n",
    "batch_data = ParameterString(\n",
    "    name=\"BatchData\",\n",
    "    default_value=output_data_path,\n",
    ")\n",
    "\n",
    "# Define the RMSE score threshold that determines if we keep the model\n",
    "rmse_threshold = ParameterFloat(name=\"RmseThreshold\", default_value=0.65)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d62993a-439d-4e4b-953c-5a1c53322ba7",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Data Preprocessing Pipeline Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "id": "eb1f8a7c-10b9-4ffe-992a-e7fa238622f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting custom-model-code/preprocessing.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile custom-model-code/preprocessing.py\n",
    "\n",
    "# Define a preprocessing script that will run in the pipeline\n",
    "# This script will take the data in the feature store, split it, and transform\n",
    "# it into the format expected by the model\n",
    "\n",
    "import json\n",
    "import argparse\n",
    "import os\n",
    "import requests\n",
    "import tempfile\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Select features to use in the model\n",
    "def get_store_features(row):\n",
    "    return [\n",
    "      row[\"sales\"], \n",
    "      row[\"oil\"], \n",
    "      row[\"onpromotion\"],\n",
    "      row[\"is_holiday\"], \n",
    "      row[\"hash_0\"], \n",
    "      row[\"hash_1\"], \n",
    "      row[\"hash_2\"], \n",
    "      row[\"hash_3\"], \n",
    "      row[\"hash_4\"], \n",
    "      row[\"hash_5\"], \n",
    "      row[\"hash_6\"], \n",
    "      row[\"hash_7\"], \n",
    "      row[\"hash_8\"], \n",
    "      row[\"hash_9\"], \n",
    "      row[\"month_cos\"],\n",
    "      row[\"month_sin\"],\n",
    "      # row[\"day_cos\"],\n",
    "      # row[\"day_sin\"],\n",
    "      row[\"dow_cos\"],\n",
    "      row[\"dow_sin\"]\n",
    "]\n",
    "\n",
    "# Split the data sets into input windows and associated targets\n",
    "def generate_windows(data, input_seq_length, target_seq_length, stride):\n",
    "    windows = []\n",
    "    targets = []\n",
    "    num_days = data.shape[1]\n",
    "    \n",
    "    for i in range(0, num_days, stride):\n",
    "        if (i+input_seq_length+target_seq_length) <= num_days:\n",
    "            input_window_end = i + input_seq_length\n",
    "            target_window_end = input_window_end + target_seq_length\n",
    "            \n",
    "            input_window = data[:, i:input_window_end, :]\n",
    "            target_window = data[:, input_window_end:target_window_end, 0]\n",
    "            \n",
    "            windows.append(input_window)\n",
    "            targets.append(target_window)\n",
    "            \n",
    "    return np.array(windows), np.array(targets)\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    # Base directory inside the pipeline\n",
    "    base_dir = \"/opt/ml/processing\"\n",
    "    \n",
    "    # Load the data\n",
    "    df = pd.read_csv(f\"{base_dir}/input/input_data.csv\", index_col=0)\n",
    "    \n",
    "    # Apply feature selection function\n",
    "    df[\"features\"] = df.apply(get_store_features, axis=1)\n",
    "    num_continuous_features = 3\n",
    "    \n",
    "    # Drop uneeded columns\n",
    "    drop_columns = [col for col in df.columns if col not in [\"date\", \"store_nbr\", \"features\"]]\n",
    "    df.drop(columns=drop_columns, inplace=True)\n",
    "    \n",
    "    # Pivot the data to be in the format (store number, date, features)\n",
    "    df_pivoted = df.pivot(index=\"store_nbr\", columns=\"date\", values=\"features\")\n",
    "    \n",
    "    # Convert the data to an array\n",
    "    stacked_df = np.array(df_pivoted.values.tolist())\n",
    "    \n",
    "    # Split the data into test/train/val sets with a 80/10/10 split\n",
    "    n = stacked_df.shape[1]\n",
    "    train_data = stacked_df[:, :int(n*0.8), :]\n",
    "    test_data = stacked_df[:, int(n*0.8):int(n*0.9), :]\n",
    "    val_data = stacked_df[:, int(n*0.9):-7, :]\n",
    "    \n",
    "    # Withold the last 7 days of the data for forecasting\n",
    "    forecast_data = stacked_df[:, -7:, :]\n",
    "    \n",
    "    # Get the mean and standard deviation for normalization\n",
    "    scaler = StandardScaler()\n",
    "\n",
    "    # Flatten the first 2 dimensions into (stores*instances, features)\n",
    "    train_data_2d = train_data.reshape(-1, train_data.shape[2])\n",
    "    test_data_2d = test_data.reshape(-1, test_data.shape[2])\n",
    "    val_data_2d = val_data.reshape(-1, val_data.shape[2])\n",
    "    forecast_data_2d = forecast_data.reshape(-1, forecast_data.shape[2])\n",
    "\n",
    "    # Scale just the continuous features\n",
    "    train_data_2d[:, :num_continuous_features] = scaler.fit_transform(train_data_2d[:, :num_continuous_features])\n",
    "    test_data_2d[:, :num_continuous_features] = scaler.transform(test_data_2d[:, :num_continuous_features])\n",
    "    val_data_2d[:, :num_continuous_features] = scaler.transform(val_data_2d[:, :num_continuous_features])\n",
    "    forecast_data_2d[:, :num_continuous_features] = scaler.transform(forecast_data_2d[:, :num_continuous_features])\n",
    "\n",
    "    # Add Gaussian noise to the continuous features\n",
    "    train_data_2d[:, :num_continuous_features] = train_data_2d[:, :num_continuous_features] + np.random.normal(0, 0.2, train_data_2d[:, :num_continuous_features].shape)\n",
    "    test_data_2d[:, :num_continuous_features] = test_data_2d[:, :num_continuous_features] + np.random.normal(0, 0.2, test_data_2d[:, :num_continuous_features].shape)\n",
    "    val_data_2d[:, :num_continuous_features] = val_data_2d[:, :num_continuous_features] + np.random.normal(0, 0.2, val_data_2d[:, :num_continuous_features].shape)\n",
    "    forecast_data_2d[:, :num_continuous_features] = forecast_data_2d[:, :num_continuous_features] + np.random.normal(0, 0.2, forecast_data_2d[:, :num_continuous_features].shape)\n",
    "\n",
    "    # Reshape the data back to its original dimensions\n",
    "    train_data = train_data_2d.reshape(train_data.shape)\n",
    "    test_data = test_data_2d.reshape(test_data.shape)\n",
    "    val_data = val_data_2d.reshape(val_data.shape)\n",
    "    forecast_data = forecast_data_2d.reshape(forecast_data.shape)\n",
    "    \n",
    "    # Generate windows for train/test/val sets\n",
    "    input_seq_length = 7\n",
    "    target_seq_length = 1\n",
    "    stride = 1\n",
    "\n",
    "    # Create the input and target windows for the data splits\n",
    "    train_inputs, train_targets = generate_windows(train_data, input_seq_length, target_seq_length, stride)\n",
    "    print(f\"Train inputs shape: {train_inputs.shape}\")\n",
    "    print(f\"Train targets shape: {train_targets.shape}\")\n",
    "\n",
    "    test_inputs, test_targets = generate_windows(test_data, input_seq_length, target_seq_length, stride)\n",
    "    print(f\"Test inputs shape: {test_inputs.shape}\")\n",
    "    print(f\"Test targets shape: {test_targets.shape}\")\n",
    "\n",
    "    val_inputs, val_targets = generate_windows(val_data, input_seq_length, target_seq_length, stride)\n",
    "    print(f\"Validation inputs shape: {val_inputs.shape}\")\n",
    "    print(f\"Validation inputs shape: {val_targets.shape}\")\n",
    "    \n",
    "    # Save data splits\n",
    "    np.save(f\"{base_dir}/train/train_inputs.npy\", train_inputs)\n",
    "    np.save(f\"{base_dir}/train/train_targets.npy\", train_targets)\n",
    "\n",
    "    np.save(f\"{base_dir}/test/test_inputs.npy\", test_inputs)\n",
    "    np.save(f\"{base_dir}/test/test_targets.npy\", test_targets)\n",
    "\n",
    "    np.save(f\"{base_dir}/validation/val_inputs.npy\", val_inputs)\n",
    "    np.save(f\"{base_dir}/validation/val_targets.npy\", val_targets)\n",
    "\n",
    "    # Save the evaluation data for the batch transform evaluation job\n",
    "    with open(f\"{base_dir}/transform-input/validation_data.ndjson\", \"w\") as f:\n",
    "        for i, window in enumerate(val_inputs):\n",
    "            instance = {\"input_1\": window.tolist()}\n",
    "            json_line = json.dumps(instance)\n",
    "            if i < len(val_inputs) - 1:\n",
    "                f.write(json_line + \"\\n\")\n",
    "            else:\n",
    "                f.write(json_line)\n",
    "    \n",
    "    \n",
    "    # Save the forecasting data\n",
    "    with open(f\"{base_dir}/forecast-input/forecast_data.ndjson\", \"w\") as f:\n",
    "        instance = {\"input_1\": forecast_data.tolist()}\n",
    "        json_line = json.dumps(instance)\n",
    "        f.write(json_line)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "id": "60803293-8d8e-4e63-bd27-30b1fb94a220",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.image_uris:Defaulting to only available Python version: py3\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.sklearn.processing import SKLearnProcessor\n",
    "\n",
    "# Define a sklearn processor container to run the preprocessing script\n",
    "sklearn_processor = SKLearnProcessor(\n",
    "    framework_version=\"1.2-1\",\n",
    "    instance_type=\"ml.m5.xlarge\",\n",
    "    instance_count=processing_instance_count,\n",
    "    base_job_name=\"sklearn-custom-model-process\",\n",
    "    role=role,\n",
    "    sagemaker_session=pipeline_session,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "id": "a9d4f597-5a5c-4dae-9f9b-facc19819872",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
    "from sagemaker.workflow.steps import ProcessingStep\n",
    "\n",
    "# Set the arguments for the data processing step\n",
    "processor_args = sklearn_processor.run(\n",
    "    inputs=[\n",
    "        ProcessingInput(source=input_data, destination=\"/opt/ml/processing/input\"),\n",
    "    ],\n",
    "    outputs=[\n",
    "        ProcessingOutput(output_name=\"train\", source=\"/opt/ml/processing/train\"),\n",
    "        ProcessingOutput(output_name=\"validation\", source=\"/opt/ml/processing/validation\"),\n",
    "        ProcessingOutput(output_name=\"test\", source=\"/opt/ml/processing/test\"),\n",
    "        ProcessingOutput(output_name=\"transform-input\", source=\"/opt/ml/processing/transform-input\"),\n",
    "        ProcessingOutput(output_name=\"forecast-input\", source=\"/opt/ml/processing/forecast-input\"),\n",
    "    ],\n",
    "    code=\"custom-model-code/preprocessing.py\",\n",
    ")\n",
    "\n",
    "# Define the data processing step\n",
    "step_process = ProcessingStep(name=\"CustomModelProcess\", step_args=processor_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "29f7a9d7-567a-4b62-8de8-e47b12f69eeb",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Model Training Pipeline Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "id": "2f9006bc-8e37-46c1-a808-320f3089b340",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.image_uris:Defaulting to only available Python version: py38\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.estimator import Estimator\n",
    "from sagemaker.inputs import TrainingInput\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "\n",
    "# Define regex patterns for capturing training metrics\n",
    "metric_definitions=[\n",
    "    {'Name': 'loss', 'Regex': \"loss: ([0-9]+(.|e\\-)[0-9]+),?\"},\n",
    "    {'Name': 'root_mean_squared_error', 'Regex': \"root_mean_squared_error: ([0-9]+(.|e\\-)[0-9]+),?\"},\n",
    "    {'Name': 'mean_absolute_error', 'Regex': \"mean_absolute_error: ([0-9]+(.|e\\-)[0-9]+),?\"},\n",
    "    {'Name': 'val_loss', 'Regex': \"val_loss: ([0-9]+(.|e\\-)[0-9]+),?\"},\n",
    "    {'Name': 'val_root_mean_squared_error', 'Regex': \"val_root_mean_squared_error: ([0-9]+(.|e\\-)[0-9]+),?\"},\n",
    "    {'Name': 'val_mean_absolute_error', 'Regex': \"val_mean_absolute_error: ([0-9]+(.|e\\-)[0-9]+),?\"},\n",
    "    {'Name': 'epoch', 'Regex': \"Epoch ([0-9]+(.|e\\-)[0-9]+),?\"}]\n",
    "\n",
    "# Specify tensorflow image to train the model in\n",
    "image_uri = sagemaker.image_uris.retrieve(\n",
    "    framework='tensorflow',\n",
    "    region=region,\n",
    "    version='2.6.0',\n",
    "    image_scope='training',\n",
    "    instance_type='ml.m5.xlarge'\n",
    ")\n",
    "\n",
    "# Define an estimator using custom training logic\n",
    "model_path = f\"s3://{bucket}/CustomModelTrain\"\n",
    "custom_model_train = Estimator(\n",
    "    entry_point='custom-model-code/train.py',\n",
    "    image_uri=image_uri,\n",
    "    instance_type='ml.m5.xlarge',\n",
    "    instance_count=1,\n",
    "    output_path=model_path,\n",
    "    role=role,\n",
    "    sagemaker_session=pipeline_session,\n",
    "    hyperparameters={\n",
    "        'batch_size': 10,\n",
    "        'epochs': 50,\n",
    "        'learning_rate': 0.002,\n",
    "        'l2_regularization': 0.004,\n",
    "        'dropout': 0.2\n",
    "    },\n",
    "    metric_definitions=metric_definitions\n",
    ")\n",
    "\n",
    "train_args = custom_model_train.fit(\n",
    "    inputs={\n",
    "        \"train\": TrainingInput(\n",
    "            s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\"train\"].S3Output.S3Uri,\n",
    "            content_type=\"application/x-npy\",\n",
    "        ),\n",
    "        \"test\": TrainingInput(\n",
    "            s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\"test\"].S3Output.S3Uri,\n",
    "            content_type=\"application/x-npy\",\n",
    "        ),\n",
    "        \"validation\": TrainingInput(\n",
    "            s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\n",
    "                \"validation\"\n",
    "            ].S3Output.S3Uri,\n",
    "            content_type=\"application/x-npy\",\n",
    "        ),\n",
    "    }\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "id": "1f5eb051-a2ad-4f91-aee5-7e566b1410b7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.inputs import TrainingInput\n",
    "from sagemaker.workflow.steps import TrainingStep\n",
    "\n",
    "# Define the training step\n",
    "step_train = TrainingStep(\n",
    "    name=\"CustomModelTrain\",\n",
    "    step_args=train_args,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "id": "77e18241-a205-4900-be42-9002e7ec065a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.model import Model\n",
    "\n",
    "# Create a model from the training job in the previous step\n",
    "custom_model = Model(\n",
    "    image_uri='763104351884.dkr.ecr.us-east-1.amazonaws.com/tensorflow-inference:2.6-cpu',\n",
    "    model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
    "    sagemaker_session=pipeline_session,\n",
    "    role=role,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "id": "b16d9fa4-9175-4b2c-9809-b1443a6e562c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.inputs import CreateModelInput\n",
    "from sagemaker.workflow.model_step import ModelStep\n",
    "\n",
    "# Create the model creation step\n",
    "step_create_model = ModelStep(\n",
    "    name=\"CustomModel\",\n",
    "    step_args=custom_model.create(instance_type=\"ml.m5.xlarge\"),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "974b7723-c109-4181-8642-dda539ea0d70",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Batch Transform Evaluation Pipeline Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "id": "b780dcb1-5c8c-4b0b-be05-73e607fe0282",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.transformer import Transformer\n",
    "from sagemaker.inputs import TransformInput\n",
    "from sagemaker.workflow.steps import TransformStep\n",
    "\n",
    "# Create a batch transform step that generates predictions on the validation set for evaluation\n",
    "transformer = Transformer(\n",
    "    model_name=step_create_model.properties.ModelName,\n",
    "    instance_type=\"ml.m5.xlarge\",\n",
    "    instance_count=1,\n",
    "    strategy=\"MultiRecord\",\n",
    "    assemble_with=\"Line\",\n",
    "    output_path=f\"{output_data_path}transform-results\",\n",
    "    accept=\"application/jsonlines\"\n",
    ")\n",
    "\n",
    "# Specify the validation data written out in the preprocessing step as input\n",
    "transform_input = TransformInput(\n",
    "    data=step_process.properties.ProcessingOutputConfig.Outputs[\"transform-input\"].S3Output.S3Uri, \n",
    "    split_type=\"Line\",\n",
    "    content_type=\"application/jsonlines\"\n",
    ")\n",
    "\n",
    "# Create the batch transform step\n",
    "step_transform_eval = TransformStep(\n",
    "    name=\"CustomModelBatchTransform\", transformer=transformer, inputs=transform_input\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 482,
   "id": "2e689d88-f23d-4e6f-804b-2973cc7173ae",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting custom-model-code/evaluation.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile custom-model-code/evaluation.py\n",
    "\n",
    "# Define an evaluation script that will run in the pipeline\n",
    "# This script evaluates the predictions made on the validation set and\n",
    "# logs an evaluation report with RMSE and MAE scores\n",
    "\n",
    "import os\n",
    "import json\n",
    "import pathlib\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    # Set base directory inside the pipeline\n",
    "    base_dir = \"/opt/ml/processing\"\n",
    "    print(os.getcwd())\n",
    "    \n",
    "    # Load validation set true target values\n",
    "    val_targets = np.load(os.path.join(f\"{base_dir}/validation\", \"val_targets.npy\"))\n",
    "    print(val_targets.shape)\n",
    "    \n",
    "    # Load predictions from the batch transform job\n",
    "    with open(f\"{base_dir}/transform-results/validation_data.ndjson.out\", \"r\") as f:\n",
    "        predictions = []\n",
    "        for line in f:\n",
    "            obj = json.loads(line.strip())\n",
    "            predictions.extend(obj[\"predictions\"])\n",
    "    \n",
    "    # Convert the predictions back into a numpy array\n",
    "    predictions_array = np.array(predictions)\n",
    "    print(predictions_array.shape)\n",
    "    \n",
    "    # Flatten the targets and predictions for computing metrics\n",
    "    targets_flat = val_targets.flatten()\n",
    "    predictions_flat = predictions_array.flatten()\n",
    "\n",
    "    # Compute the RMSE, MAE, and standard deviation of the residuals\n",
    "    rmse = mean_squared_error(targets_flat, predictions_flat, squared=False)\n",
    "    mae = mean_absolute_error(targets_flat, predictions_flat)\n",
    "    std = np.std(targets_flat - predictions_flat)\n",
    "    print(f\"RMSE: {rmse} MAE: {mae}\")\n",
    "\n",
    "    # Write the evaluation metrics out to an evaluation report\n",
    "    report_dict = {\n",
    "        \"regression_metrics\": {\n",
    "            \"rmse\": {\"value\": rmse, \"standard_deviation\": std},\n",
    "            \"mae\": {\"value\": mae, \"standard_deviation\": std}\n",
    "        },\n",
    "    }\n",
    "\n",
    "    output_dir = \"/opt/ml/processing/evaluation\"\n",
    "    pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    evaluation_path = f\"{output_dir}/evaluation.json\"\n",
    "    with open(evaluation_path, \"w\") as f:\n",
    "        f.write(json.dumps(report_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "id": "337849b2-5563-46bc-9f09-d1cbfd3c8124",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:sagemaker.image_uris:Defaulting to only available Python version: py3\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.sklearn.processing import SKLearnProcessor\n",
    "\n",
    "# Define a sklearn processor container to run the evaluation script\n",
    "sklearn_eval_processor = SKLearnProcessor(\n",
    "    framework_version=\"1.2-1\",\n",
    "    instance_type=\"ml.m5.xlarge\",\n",
    "    instance_count=processing_instance_count,\n",
    "    base_job_name=\"sklearn-custom-eval-process\",\n",
    "    role=role,\n",
    "    sagemaker_session=pipeline_session,\n",
    ")\n",
    "\n",
    "# Set the input, output, and script for the evaluation step\n",
    "eval_args = sklearn_eval_processor.run(\n",
    "    inputs=[\n",
    "        ProcessingInput(\n",
    "            source=step_process.properties.ProcessingOutputConfig.Outputs[\"validation\"].S3Output.S3Uri,\n",
    "            destination=\"/opt/ml/processing/validation\",\n",
    "        ),\n",
    "        ProcessingInput(\n",
    "            source=step_transform_eval.properties.TransformOutput.S3OutputPath,\n",
    "            destination=\"/opt/ml/processing/transform-results\",\n",
    "        ),\n",
    "    ],\n",
    "    outputs=[\n",
    "        ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\"),\n",
    "    ],\n",
    "    code=\"custom-model-code/evaluation.py\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "id": "8f1613a3-f125-493e-ba2a-99ea8bd37a69",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.workflow.properties import PropertyFile\n",
    "\n",
    "# Define the file the evaluation report will be written to\n",
    "evaluation_report = PropertyFile(\n",
    "    name=\"EvaluationReport\", output_name=\"evaluation\", path=\"evaluation.json\"\n",
    ")\n",
    "\n",
    "# Create the evaluation step\n",
    "step_eval = ProcessingStep(\n",
    "    name=\"CustomModelEval\",\n",
    "    step_args=eval_args,\n",
    "    property_files=[evaluation_report],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d1b5b311-525e-4557-9866-0afd7481d501",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Model Registration Pipeline Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "id": "b3731a41-3930-4316-a490-98adc2f4ed9b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:sagemaker.workflow.utilities:Popping out 'ProcessingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.model_metrics import MetricsSource, ModelMetrics\n",
    "\n",
    "# Load the metrics computed in the evaluation step\n",
    "model_metrics = ModelMetrics(\n",
    "    model_statistics=MetricsSource(\n",
    "        s3_uri=\"{}/evaluation.json\".format(\n",
    "            step_eval.arguments[\"ProcessingOutputConfig\"][\"Outputs\"][0][\"S3Output\"][\"S3Uri\"]\n",
    "        ),\n",
    "        content_type=\"application/json\",\n",
    "    )\n",
    ")\n",
    "\n",
    "# Register the model to the custom model package group\n",
    "register_args = custom_model.register(\n",
    "    inference_instances=[\"ml.t2.medium\", \"ml.m5.xlarge\"],\n",
    "    transform_instances=[\"ml.m5.xlarge\"],\n",
    "    model_package_group_name=model_package_group_name,\n",
    "    approval_status=model_approval_status,\n",
    "    model_metrics=model_metrics\n",
    ")\n",
    "\n",
    "# Create the model registration step\n",
    "# NOTE: This step will only run if the RMSE threshold is not exceeded in the eval step\n",
    "step_register = ModelStep(name=\"CustomModelRegister\", step_args=register_args)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff5e2f8b-626a-43fb-90f5-299ffc09c686",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Forecasting Pipeline Steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71780eac-c464-4bd8-a581-3118dbba703a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Get the container image and S3 artifacts for the last model \n",
    "model_packages = sm_client.list_model_packages(\n",
    "    ModelPackageGroupName=model_package_group_name, SortBy=\"CreationTime\", SortOrder=\"Descending\")\n",
    "\n",
    "model_package = sm_client.describe_model_package(ModelPackageName=model_packages['ModelPackageSummaryList'][0][\"ModelPackageArn\"])\n",
    "previous_model_image = model_package['InferenceSpecification']['Containers'][0]['Image']\n",
    "previous_model_artifacts = model_package['InferenceSpecification']['Containers'][0]['ModelDataUrl']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 480,
   "id": "fd03cb84-cf9e-468f-9124-9210e8d2c66b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Recreate the last model that was added to the registry\n",
    "# This model will be used for forecasting if the model trained\n",
    "# In this pipeline exceeds the RMSE threshold\n",
    "custom_model_existing = Model(\n",
    "    image_uri=previous_model_image,\n",
    "    model_data=previous_model_artifacts,\n",
    "    sagemaker_session=pipeline_session,\n",
    "    role=role,\n",
    ")\n",
    "\n",
    "# Create a step that recreates an existing model for inference\n",
    "step_recreate_existing_model = ModelStep(\n",
    "    name=\"PreviousCustomModel\",\n",
    "    step_args=custom_model_existing.create(instance_type=\"ml.m5.xlarge\"),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 469,
   "id": "7dbe26eb-33ae-4a58-ae6f-15e3c607daee",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.transformer import Transformer\n",
    "\n",
    "# Define the batch transform that will run with the new model\n",
    "new_forecast_transformer = Transformer(\n",
    "    model_name=step_create_model.properties.ModelName,\n",
    "    instance_type=\"ml.m5.xlarge\",\n",
    "    instance_count=1,\n",
    "    strategy=\"MultiRecord\",\n",
    "    assemble_with=\"Line\",\n",
    "    output_path=f\"{output_data_path}sales-forecast\",\n",
    "    accept=\"application/jsonlines\"\n",
    ")\n",
    "\n",
    "# Define the batch transform that will run with the existing model\n",
    "old_forecast_transformer = Transformer(\n",
    "    model_name=step_recreate_existing_model.properties.ModelName,\n",
    "    instance_type=\"ml.m5.xlarge\",\n",
    "    instance_count=1,\n",
    "    strategy=\"MultiRecord\",\n",
    "    assemble_with=\"Line\",\n",
    "    output_path=f\"{output_data_path}sales-forecast\",\n",
    "    accept=\"application/jsonlines\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "id": "5a3ed5e8-8196-40ac-97f7-f4657deef542",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the input for the forecasting step\n",
    "transform_input_forecast = TransformInput(\n",
    "    data=step_process.properties.ProcessingOutputConfig.Outputs[\"forecast-input\"].S3Output.S3Uri,\n",
    "    split_type=\"Line\",\n",
    "    content_type=\"application/jsonlines\"\n",
    ")\n",
    "\n",
    "# Create the batch transform step that runs if the conditional check succeeds\n",
    "step_transform_forecast_new = TransformStep(\n",
    "    name=\"CustomModelBatchForecastNew\", transformer=new_forecast_transformer, inputs=transform_input_forecast\n",
    ")\n",
    "\n",
    "# Create the batch transform step that runs if the conditional check fails\n",
    "step_transform_forecast_existing = TransformStep(\n",
    "    name=\"CustomModelBatchForecastExisting\", transformer=old_forecast_transformer, inputs=transform_input_forecast\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93895f41-1260-4c86-b94b-3be2e3e321dd",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Conditional Pipeline Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "id": "b8bb66a6-205d-4189-a2e8-0123beddb230",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.workflow.conditions import ConditionLessThanOrEqualTo\n",
    "from sagemaker.workflow.condition_step import ConditionStep\n",
    "from sagemaker.workflow.functions import JsonGet\n",
    "\n",
    "\n",
    "# Define a conditional check that checks if the RMSE computed in the evaluation\n",
    "# step is below the threshold parameter\n",
    "cond_lte = ConditionLessThanOrEqualTo(\n",
    "    left=JsonGet(\n",
    "        step_name=step_eval.name,\n",
    "        property_file=evaluation_report,\n",
    "        json_path=\"regression_metrics.rmse.value\",\n",
    "    ),\n",
    "    right=rmse_threshold,\n",
    ")\n",
    "\n",
    "# Create a step that registers the new model if the conditional check succeeds and\n",
    "# uses the new model for forecasting, and otherwise loads the most recent model from \n",
    "# the registry and uses that model for forecasting\n",
    "step_cond = ConditionStep(\n",
    "    name=\"CustomModelRMSECond\",\n",
    "    conditions=[cond_lte],\n",
    "    if_steps=[step_register, step_transform_forecast_new],\n",
    "    else_steps=[step_recreate_existing_model, step_transform_forecast_existing],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ebec7ea-4039-4bfa-8dd7-372efecae5ab",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Pipeline Execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "id": "8c535ec7-3f9c-48a5-ab9f-e1625da1264b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.workflow.pipeline import Pipeline\n",
    "\n",
    "# Configure the pipeline\n",
    "pipeline_name = f\"CustomModelPipeline\"\n",
    "pipeline = Pipeline(\n",
    "    name=pipeline_name,\n",
    "    parameters=[\n",
    "        processing_instance_count,\n",
    "        instance_type,\n",
    "        model_approval_status,\n",
    "        input_data,\n",
    "        batch_data,\n",
    "        rmse_threshold,\n",
    "    ],\n",
    "    steps=[step_process, step_train, step_create_model, step_transform_eval, step_eval, step_cond],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 475,
   "id": "70fec1a3-29ba-4545-93a9-d34db562d4a6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:sagemaker.workflow.utilities:Popping out 'ProcessingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ModelName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TransformJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ProcessingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ModelPackageName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TransformJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ModelName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TransformJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ProcessingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TrainingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ModelName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TransformJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ProcessingJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ModelPackageName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TransformJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'ModelName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n",
      "WARNING:sagemaker.workflow.utilities:Popping out 'TransformJobName' from the pipeline definition by default since it will be overridden at pipeline execution time. Please utilize the PipelineDefinitionConfig to persist this field in the pipeline definition if desired.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:342408968837:pipeline/CustomModelPipeline',\n",
       " 'ResponseMetadata': {'RequestId': '8eb65de6-893f-4afe-b891-435df14df6b7',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '8eb65de6-893f-4afe-b891-435df14df6b7',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '87',\n",
       "   'date': 'Sat, 19 Oct 2024 00:52:52 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 475,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Upsert the IAM role to the pipeline steps\n",
    "pipeline.upsert(role_arn=role)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 476,
   "id": "b9cacf46-9043-4efc-b0de-4d0d82c3da0b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'PipelineArn': 'arn:aws:sagemaker:us-east-1:342408968837:pipeline/CustomModelPipeline',\n",
       " 'PipelineExecutionArn': 'arn:aws:sagemaker:us-east-1:342408968837:pipeline/CustomModelPipeline/execution/oot6k1g9zwmg',\n",
       " 'PipelineExecutionDisplayName': 'execution-1729299173064',\n",
       " 'PipelineExecutionStatus': 'Executing',\n",
       " 'CreationTime': datetime.datetime(2024, 10, 19, 0, 52, 52, 967000, tzinfo=tzlocal()),\n",
       " 'LastModifiedTime': datetime.datetime(2024, 10, 19, 0, 52, 52, 967000, tzinfo=tzlocal()),\n",
       " 'CreatedBy': {'UserProfileArn': 'arn:aws:sagemaker:us-east-1:342408968837:user-profile/d-2cr7fbmrqyrg/jlawton',\n",
       "  'UserProfileName': 'jlawton',\n",
       "  'DomainId': 'd-2cr7fbmrqyrg',\n",
       "  'IamIdentity': {'Arn': 'arn:aws:sts::342408968837:assumed-role/LabRole/SageMaker',\n",
       "   'PrincipalId': 'AROAU7OJKHKCWCCLLHI6O:SageMaker'}},\n",
       " 'LastModifiedBy': {'UserProfileArn': 'arn:aws:sagemaker:us-east-1:342408968837:user-profile/d-2cr7fbmrqyrg/jlawton',\n",
       "  'UserProfileName': 'jlawton',\n",
       "  'DomainId': 'd-2cr7fbmrqyrg',\n",
       "  'IamIdentity': {'Arn': 'arn:aws:sts::342408968837:assumed-role/LabRole/SageMaker',\n",
       "   'PrincipalId': 'AROAU7OJKHKCWCCLLHI6O:SageMaker'}},\n",
       " 'ResponseMetadata': {'RequestId': '42dbf68e-3ed3-48be-a1e4-4d43b2e79956',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '42dbf68e-3ed3-48be-a1e4-4d43b2e79956',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '963',\n",
       "   'date': 'Sat, 19 Oct 2024 00:52:53 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 476,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Start the pipeline execution\n",
    "execution = pipeline.start()\n",
    "execution.describe()"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 57,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.trn1.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 58,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1.32xlarge",
    "vcpuNum": 128
   },
   {
    "_defaultOrder": 59,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.trn1n.32xlarge",
    "vcpuNum": 128
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (Data Science 3.0)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/sagemaker-data-science-310-v1"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}