6.2 - Data Preparation DeepAR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part VI.2 - Prepare dataset for DeepAR model\n",
    "\n",
    "University of San Diego - MS Applied AI\n",
    "\n",
    "AAI-540 Team 5\n",
    "\n",
    "October 21, 2024"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stored 's3_datalake_path_csv' (str)\n",
      "Stored 'local_data_path_csv' (str)\n",
      "Stored 's3_datalake_path_parquet' (str)\n"
     ]
    }
   ],
   "source": [
    "# setup environment\n",
    "%run 0-Environment_Setup.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from time import gmtime, strftime\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running \n",
      "    SELECT *\n",
      "    FROM\n",
      "        \"store_sales_feature_group_offline_1728336748\"\n",
      "    ORDER BY\n",
      "        store_nbr ASC, date ASC\n",
      "    \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>store_nbr</th>\n",
       "      <th>sales</th>\n",
       "      <th>oil</th>\n",
       "      <th>onpromotion</th>\n",
       "      <th>is_holiday</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>cluster</th>\n",
       "      <th>year</th>\n",
       "      <th>...</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>dow_cos</th>\n",
       "      <th>dow_sin</th>\n",
       "      <th>sales_record_id</th>\n",
       "      <th>event_time</th>\n",
       "      <th>write_time</th>\n",
       "      <th>api_invocation_time</th>\n",
       "      <th>is_deleted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.979530</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.623490</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:1</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:10.967</td>\n",
       "      <td>2024-10-07 21:33:11.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-01-02</td>\n",
       "      <td>1</td>\n",
       "      <td>7417.15</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.918958</td>\n",
       "      <td>0.394356</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>0.974928</td>\n",
       "      <td>2013-01-02:1</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:11.435</td>\n",
       "      <td>2024-10-07 21:33:12.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-01-03</td>\n",
       "      <td>1</td>\n",
       "      <td>5873.24</td>\n",
       "      <td>92.97</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.820763</td>\n",
       "      <td>0.571268</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>0.433884</td>\n",
       "      <td>2013-01-03:1</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:10.869</td>\n",
       "      <td>2024-10-07 21:33:13.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-01-04</td>\n",
       "      <td>1</td>\n",
       "      <td>5919.88</td>\n",
       "      <td>93.12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.688967</td>\n",
       "      <td>0.724793</td>\n",
       "      <td>-0.900969</td>\n",
       "      <td>-0.433884</td>\n",
       "      <td>2013-01-04:1</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:11.111</td>\n",
       "      <td>2024-10-07 21:33:14.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-01-05</td>\n",
       "      <td>1</td>\n",
       "      <td>6318.79</td>\n",
       "      <td>93.12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.528964</td>\n",
       "      <td>0.848644</td>\n",
       "      <td>-0.222521</td>\n",
       "      <td>-0.974928</td>\n",
       "      <td>2013-01-05:1</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:11.141</td>\n",
       "      <td>2024-10-07 21:33:14.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         date  store_nbr    sales    oil  onpromotion  is_holiday  city  \\\n",
       "0  2013-01-01          1     0.00  93.14            0           1    18   \n",
       "1  2013-01-02          1  7417.15  93.14            0           0    18   \n",
       "2  2013-01-03          1  5873.24  92.97            0           0    18   \n",
       "3  2013-01-04          1  5919.88  93.12            0           0    18   \n",
       "4  2013-01-05          1  6318.79  93.12            0           1    18   \n",
       "\n",
       "   state  cluster  year  ...  month_sin   day_cos   day_sin   dow_cos  \\\n",
       "0     12       13  2013  ...        0.5  0.979530  0.201299  0.623490   \n",
       "1     12       13  2013  ...        0.5  0.918958  0.394356 -0.222521   \n",
       "2     12       13  2013  ...        0.5  0.820763  0.571268 -0.900969   \n",
       "3     12       13  2013  ...        0.5  0.688967  0.724793 -0.900969   \n",
       "4     12       13  2013  ...        0.5  0.528964  0.848644 -0.222521   \n",
       "\n",
       "    dow_sin  sales_record_id    event_time               write_time  \\\n",
       "0  0.781831     2013-01-01:1  1.728337e+09  2024-10-07 21:38:10.967   \n",
       "1  0.974928     2013-01-02:1  1.728337e+09  2024-10-07 21:38:11.435   \n",
       "2  0.433884     2013-01-03:1  1.728337e+09  2024-10-07 21:38:10.869   \n",
       "3 -0.433884     2013-01-04:1  1.728337e+09  2024-10-07 21:38:11.111   \n",
       "4 -0.974928     2013-01-05:1  1.728337e+09  2024-10-07 21:38:11.141   \n",
       "\n",
       "       api_invocation_time  is_deleted  \n",
       "0  2024-10-07 21:33:11.000       False  \n",
       "1  2024-10-07 21:33:12.000       False  \n",
       "2  2024-10-07 21:33:13.000       False  \n",
       "3  2024-10-07 21:33:14.000       False  \n",
       "4  2024-10-07 21:33:14.000       False  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get features stacked by store then date\n",
    "sales_features_store = get_store_dataset_from_offline_feature_group(store_sales_feature_group)\n",
    "sales_features_store.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set dataset split parameters\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>store_nbr</th>\n",
       "      <th>sales</th>\n",
       "      <th>oil</th>\n",
       "      <th>onpromotion</th>\n",
       "      <th>is_holiday</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>cluster</th>\n",
       "      <th>year</th>\n",
       "      <th>...</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>day_cos</th>\n",
       "      <th>day_sin</th>\n",
       "      <th>dow_cos</th>\n",
       "      <th>dow_sin</th>\n",
       "      <th>sales_record_id</th>\n",
       "      <th>event_time</th>\n",
       "      <th>write_time</th>\n",
       "      <th>api_invocation_time</th>\n",
       "      <th>is_deleted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:1</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:10.967</td>\n",
       "      <td>2024-10-07 21:33:11.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52204</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>32</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:32</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:10.967</td>\n",
       "      <td>2024-10-07 21:33:12.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53888</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:33</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:11.047</td>\n",
       "      <td>2024-10-07 21:33:12.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55572</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>34</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:34</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:10.856</td>\n",
       "      <td>2024-10-07 21:33:12.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57256</th>\n",
       "      <td>2013-01-01</td>\n",
       "      <td>35</td>\n",
       "      <td>0.0</td>\n",
       "      <td>93.14</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>2013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.97953</td>\n",
       "      <td>0.201299</td>\n",
       "      <td>0.62349</td>\n",
       "      <td>0.781831</td>\n",
       "      <td>2013-01-01:35</td>\n",
       "      <td>1.728337e+09</td>\n",
       "      <td>2024-10-07 21:38:11.069</td>\n",
       "      <td>2024-10-07 21:33:12.000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            date  store_nbr  sales    oil  onpromotion  is_holiday  city  \\\n",
       "0     2013-01-01          1    0.0  93.14            0           1    18   \n",
       "52204 2013-01-01         32    0.0  93.14            0           1     8   \n",
       "53888 2013-01-01         33    0.0  93.14            0           1    17   \n",
       "55572 2013-01-01         34    0.0  93.14            0           1     8   \n",
       "57256 2013-01-01         35    0.0  93.14            0           1    15   \n",
       "\n",
       "       state  cluster  year  ...  month_sin  day_cos   day_sin  dow_cos  \\\n",
       "0         12       13  2013  ...        0.5  0.97953  0.201299  0.62349   \n",
       "52204      6        3  2013  ...        0.5  0.97953  0.201299  0.62349   \n",
       "53888      9        3  2013  ...        0.5  0.97953  0.201299  0.62349   \n",
       "55572      6        6  2013  ...        0.5  0.97953  0.201299  0.62349   \n",
       "57256      6        3  2013  ...        0.5  0.97953  0.201299  0.62349   \n",
       "\n",
       "        dow_sin  sales_record_id    event_time               write_time  \\\n",
       "0      0.781831     2013-01-01:1  1.728337e+09  2024-10-07 21:38:10.967   \n",
       "52204  0.781831    2013-01-01:32  1.728337e+09  2024-10-07 21:38:10.967   \n",
       "53888  0.781831    2013-01-01:33  1.728337e+09  2024-10-07 21:38:11.047   \n",
       "55572  0.781831    2013-01-01:34  1.728337e+09  2024-10-07 21:38:10.856   \n",
       "57256  0.781831    2013-01-01:35  1.728337e+09  2024-10-07 21:38:11.069   \n",
       "\n",
       "           api_invocation_time  is_deleted  \n",
       "0      2024-10-07 21:33:11.000       False  \n",
       "52204  2024-10-07 21:33:12.000       False  \n",
       "53888  2024-10-07 21:33:12.000       False  \n",
       "55572  2024-10-07 21:33:12.000       False  \n",
       "57256  2024-10-07 21:33:12.000       False  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sort features by date as copy to train scaler \n",
    "sales_features_date = sales_features_store.copy()\n",
    "sales_features_date['date'] = pd.to_datetime(sales_features_date['date'])\n",
    "sales_features_date = sales_features_date.sort_values(by=['date'])\n",
    "sales_features_date.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total days in series: 1687\n",
      "Days in training dataset: 1349\n",
      "Days in test data: 338\n",
      "Weeks in test data: 24\n",
      "Weeks in validation data: 24\n"
     ]
    }
   ],
   "source": [
    "# calculate the total days in the date range so we can split at 80% mark\n",
    "series_start_date = pd.to_datetime(sales_features_date['date'].min())\n",
    "series_end_date = pd.to_datetime(sales_features_date['date'].max())\n",
    "delta = series_end_date - series_start_date\n",
    "\n",
    "# set training cutoff parameters\n",
    "training_series_day_count = int(delta.days * .8)\n",
    "start_training = series_start_date\n",
    "end_training = series_start_date + datetime.timedelta(days=training_series_day_count)\n",
    "\n",
    "# set test cutoff parameters\n",
    "start_test = end_training + datetime.timedelta(days=1)\n",
    "test_days = delta.days - training_series_day_count\n",
    "test_weeks = int((delta.days - training_series_day_count) / 7)\n",
    "val_weeks = int(test_weeks / 2)\n",
    "test_weeks = val_weeks\n",
    "end_test = start_test + datetime.timedelta(days=(test_weeks * 7))\n",
    "\n",
    "print(\"Total days in series: {}\".format(delta.days)) \n",
    "print(\"Days in training dataset: {}\".format(training_series_day_count))\n",
    "print(\"Days in test data: {}\".format(test_days))\n",
    "print(\"Weeks in test data: {}\".format(test_weeks))\n",
    "print(\"Weeks in validation data: {}\".format(val_weeks))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Scale/Noarmalize the numeric features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# slice the raw data to train set (to fit scaler)\n",
    "scale_train_set = sales_features_date[sales_features_date['date'] < start_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: black;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: block;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 1ex;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>StandardScaler()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow fitted\">&nbsp;&nbsp;StandardScaler<a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.4/modules/generated/sklearn.preprocessing.StandardScaler.html\">?<span>Documentation for StandardScaler</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></label><div class=\"sk-toggleable__content fitted\"><pre>StandardScaler()</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "StandardScaler()"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# initialize standard scaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(scale_train_set[['sales', 'oil', 'onpromotion']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['deepAR-scaler.joblib']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save the scaler locally\n",
    "joblib.dump(scaler, 'deepAR-scaler.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "download: s3://sagemaker-us-east-1-053585949834/store-sales-forecasting/deepar/gold-dataset/scaler/deepAR-scaler.joblib to ./deepAR-scaler.joblib\n"
     ]
    }
   ],
   "source": [
    "# only run if scaler is already fit and saved\n",
    "LOAD_SCALER = False\n",
    "if(LOAD_SCALER):\n",
    "    scaler_filename = copy_file_from_s3(s3_deepar_gold_dataset_path + \"/scaler/deepAR-scaler.joblib\")\n",
    "    scaler = joblib.load(scaler_filename) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply scaling to our full dataset\n",
    "sales_features_store[['sales', 'oil', 'onpromotion']] = scaler.transform(sales_features_date[['sales', 'oil', 'onpromotion']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
       "       18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34,\n",
       "       35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,\n",
       "       52, 53, 54])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# capture the unique stores\n",
    "unique_store_nbrs = sales_features_store['store_nbr'].unique()\n",
    "unique_store_nbrs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to build the target time series for each store\n",
    "def build_store_timeseries(store_sales, target_col):\n",
    "    unique_stores = store_sales['store_nbr'].unique()\n",
    "    store_timeseries = []\n",
    "    for store_nbr in unique_stores:\n",
    "        # get the sales data for this store and only keep the timestep and sales number\n",
    "        store_data = store_sales[store_sales['store_nbr'] == store_nbr]\n",
    "        store_data = store_data[['date', target_col]]\n",
    "\n",
    "        # convert to datetime and then to series with timestep = 1d\n",
    "        store_data['date'] = pd.to_datetime(store_data['date'])\n",
    "        \n",
    "        store_data = store_data.set_index('date')\n",
    "        store_data = store_data.resample('D').sum()\n",
    "        store_ts = store_data.iloc[:, 0]\n",
    "\n",
    "        # add to list\n",
    "        store_timeseries.append(store_ts)    \n",
    "    return store_timeseries\n",
    "\n",
    "# helper function to write ts datasets to json\n",
    "def write_dicts_to_file(path, data):\n",
    "    with open(path, \"wb\") as fp:\n",
    "        for d in data:\n",
    "            fp.write(json.dumps(d).encode(\"utf-8\"))\n",
    "            fp.write(\"\\n\".encode(\"utf-8\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# build the target timeseries\n",
    "timeseries_stores_sales = build_store_timeseries(sales_features_store, 'sales')\n",
    "timeseries_stores_oil = build_store_timeseries(sales_features_store, 'oil')\n",
    "timeseries_stores_holidays = build_store_timeseries(sales_features_store, 'is_holiday')\n",
    "timeseries_stores_promotions = build_store_timeseries(sales_features_store, 'onpromotion')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1688\n",
      "1688\n",
      "1688\n",
      "1688\n"
     ]
    }
   ],
   "source": [
    "print(len(timeseries_stores_sales[0]))\n",
    "print(len(timeseries_stores_oil[0]))\n",
    "print(len(timeseries_stores_holidays[0]))\n",
    "print(len(timeseries_stores_promotions[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Train and Test Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset date range: 2013-01-01 00:00:00 - 2017-08-15 00:00:00\n"
     ]
    }
   ],
   "source": [
    "# inspect date range of the series\n",
    "print(\"Dataset date range: {} - {}\".format(timeseries_stores_sales[0].index.min(), timeseries_stores_sales[0].index.max()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54\n"
     ]
    }
   ],
   "source": [
    "# generate training data\n",
    "training_data = [\n",
    "    {\n",
    "        \"start\": str(start_training),\n",
    "        \"target\": ts[start_training:end_training].tolist(),\n",
    "        \"cat\": [int(unique_store_nbrs[i]) - 1],\n",
    "        \"dynamic_feat\": [\n",
    "            timeseries_stores_oil[i][start_training:end_training].tolist(),\n",
    "            timeseries_stores_holidays[i][start_training:end_training].tolist(),\n",
    "            timeseries_stores_promotions[i][start_training:end_training].tolist(),\n",
    "        ],\n",
    "    }\n",
    "    for i, ts in enumerate(timeseries_stores_sales)\n",
    "]\n",
    "print(len(training_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54\n"
     ]
    }
   ],
   "source": [
    "val_end = start_test + datetime.timedelta(days=(val_weeks*7))\n",
    "\n",
    "# generate training data\n",
    "val_data = [\n",
    "    {\n",
    "        \"start\": str(start_test),\n",
    "        \"target\": ts[start_test:val_end].tolist(),\n",
    "        \"cat\": [int(unique_store_nbrs[i]) - 1],\n",
    "        \"dynamic_feat\": [\n",
    "            timeseries_stores_oil[i][start_test:val_end].tolist(),\n",
    "            timeseries_stores_holidays[i][start_test:val_end].tolist(),\n",
    "            timeseries_stores_promotions[i][start_test:val_end].tolist(),\n",
    "        ],\n",
    "    }\n",
    "    for i, ts in enumerate(timeseries_stores_sales)\n",
    "]\n",
    "print(len(val_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1188\n"
     ]
    }
   ],
   "source": [
    "# Rolling weekly evaluations until end of test set and after val set\n",
    "test_windows = test_weeks - 2\n",
    "\n",
    "gen_test_start = start_test + datetime.timedelta(days=(val_weeks*7))\n",
    "gen_test_end = gen_test_start + datetime.timedelta(days=(test_weeks*7))\n",
    "cw = 7\n",
    "\n",
    "test_data = [\n",
    "    {\n",
    "        \"start\": str(gen_test_start + datetime.timedelta(days=((k-1) * cw))),\n",
    "        \"target\": ts[(gen_test_start + datetime.timedelta(days=((k-1) * cw))) : (gen_test_start + datetime.timedelta(days=((k * cw) - 1)))].tolist(),\n",
    "        \"cat\": [int(unique_store_nbrs[i]) - 1],\n",
    "        \"dynamic_feat\": [\n",
    "            timeseries_stores_oil[i][(gen_test_start + datetime.timedelta(days=((k-1) * cw))) : (gen_test_start + datetime.timedelta(days=((k * cw) + deepar_prediction_length - 1)))].tolist(),\n",
    "            timeseries_stores_holidays[i][(gen_test_start + datetime.timedelta(days=((k-1) * cw))) : (gen_test_start + datetime.timedelta(days=((k * cw) + deepar_prediction_length - 1)))].tolist(),\n",
    "            timeseries_stores_promotions[i][(gen_test_start + datetime.timedelta(days=((k-1) * cw))) : (gen_test_start + datetime.timedelta(days=((k * cw) + deepar_prediction_length - 1)))].tolist(),\n",
    "        ],\n",
    "    }\n",
    "    for k in range(1, test_windows + 1)\n",
    "    for i, ts in enumerate(timeseries_stores_sales)\n",
    "]\n",
    "\n",
    "print(len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write datasets to json files\n",
    "write_dicts_to_file(\"train.json\", training_data)\n",
    "write_dicts_to_file(\"val.json\", val_data)\n",
    "write_dicts_to_file(\"test.json\", test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://sagemaker-us-east-1-343218227212/store-sales-forecasting/deepar/gold-dataset'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set flag to publish to \"gold\" dataset or keep in experiments\n",
    "publish_to_gold = True\n",
    "\n",
    "# setup S3 path to store train and test datasets\n",
    "if(publish_to_gold):\n",
    "    s3_dataset_path = s3_deepar_gold_dataset_path\n",
    "else:\n",
    "    s3_dataset_path = \"s3://{}/store-sales-forecasting/deepar/experiments/{}\".format(bucket, strftime(\"%Y-%m-%d-%H-%M-%S\", gmtime()))\n",
    "s3_dataset_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting existing file\n",
      "Uploading file to s3://sagemaker-us-east-1-343218227212/store-sales-forecasting/deepar/gold-dataset/train/train.json\n",
      "Overwriting existing file\n",
      "Uploading file to s3://sagemaker-us-east-1-343218227212/store-sales-forecasting/deepar/gold-dataset/val/val.json\n",
      "Overwriting existing file\n",
      "Uploading file to s3://sagemaker-us-east-1-343218227212/store-sales-forecasting/deepar/gold-dataset/test/test.json\n"
     ]
    }
   ],
   "source": [
    "# copy the train/test files to S3\n",
    "copy_to_s3(\"train.json\", s3_dataset_path + \"/train/train.json\", True)\n",
    "copy_to_s3(\"val.json\", s3_dataset_path + \"/val/val.json\", True)\n",
    "copy_to_s3(\"test.json\", s3_dataset_path + \"/test/test.json\", True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy scaler to S3\n",
    "copy_to_s3(\"deepAR-scaler.joblib\", s3_dataset_path + \"/scaler/deepAR-scaler.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rm: cannot remove 'deepAR-scaler.joblib': No such file or directory\n"
     ]
    }
   ],
   "source": [
    "# cleanup files\n",
    "!rm train.json\n",
    "!rm val.json\n",
    "!rm test.json\n",
    "!rm deepAR-scaler.joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}