![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "6QeMfKCzuY9q"
- },
- "outputs": [],
- "source": [
- "from statsmodels.tsa.stattools import adfuller\n",
- "from statsmodels.graphics.tsaplots import plot_acf,plot_pacf\n",
- "import statsmodels.api as sm\n",
- "from statsmodels.tsa.arima_model import ARIMA\n",
- "from statsmodels.tsa.arima_model import ARMA\n",
- "from statsmodels.tools.parallel import parallel_func as Parallel\n",
- "\n",
- "import pandas as pd\n",
- "import seaborn as sns\n",
- "from matplotlib.pylab import rcParams\n",
- "import matplotlib.pyplot as plt\n",
- "import plotly.express as px\n",
- "import plotly.graph_objects as go\n",
- "from tqdm import tqdm_notebook\n",
- "import math\n",
- "import numpy as np\n",
- "import pandas as pd\n",
- "\n",
- "from itertools import product\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "from sklearn.feature_extraction.text import TfidfVectorizer\n",
- "from sklearn.feature_extraction.text import CountVectorizer\n",
- "from sklearn.datasets import fetch_20newsgroups\n",
- "from sklearn.decomposition import LatentDirichletAllocation\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "from sklearn.decomposition import PCA\n",
- "from sklearn.covariance import EllipticEnvelope\n",
- "from sklearn.ensemble import IsolationForest"
- ]
- },
- {
- "cell_type": "code",
- "source": [],
- "metadata": {
- "id": "l6e8Wwu_upH_"
- },
- "execution_count": null,
- "outputs": []
- },
- {
- "cell_type": "markdown",
- "source": [
- "#Import Monthly Index Data"
- ],
- "metadata": {
- "id": "z_ZFR4hTuzXi"
- }
- },
- {
- "cell_type": "code",
- "source": [
- "df = pd.read_csv('Hang Seng Index_Monthly.csv')\n",
- "df = df[['Date','Close']]\n",
- "df"
- ],
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 423
- },
- "id": "98ysjm-Bu12c",
- "outputId": "c9ebfee0-55dc-4085-f13a-80328bf09bd9"
- },
- "execution_count": null,
- "outputs": [
- {
- "output_type": "execute_result",
- "data": {
- "text/plain": [
- " Date Close\n",
- "0 2017-01-01 23360.779297\n",
- "1 2017-02-01 23740.730469\n",
- "2 2017-03-01 24111.589844\n",
- "3 2017-04-01 24615.130859\n",
- "4 2017-05-01 25660.650391\n",
- ".. ... ...\n",
- "64 2022-05-01 21415.199219\n",
- "65 2022-06-01 21859.789063\n",
- "66 2022-07-01 20156.509766\n",
- "67 2022-08-01 19954.390625\n",
- "68 2022-09-01 17222.830078\n",
- "\n",
- "[69 rows x 2 columns]"
- ],
- "text/html": [
- "\n",
- " | \n", - " | Date | \n", - "Close | \n", - "
|---|---|---|
| 0 | \n", - "2017-01-01 | \n", - "23360.779297 | \n", - "
| 1 | \n", - "2017-02-01 | \n", - "23740.730469 | \n", - "
| 2 | \n", - "2017-03-01 | \n", - "24111.589844 | \n", - "
| 3 | \n", - "2017-04-01 | \n", - "24615.130859 | \n", - "
| 4 | \n", - "2017-05-01 | \n", - "25660.650391 | \n", - "
| ... | \n", - "... | \n", - "... | \n", - "
| 64 | \n", - "2022-05-01 | \n", - "21415.199219 | \n", - "
| 65 | \n", - "2022-06-01 | \n", - "21859.789063 | \n", - "
| 66 | \n", - "2022-07-01 | \n", - "20156.509766 | \n", - "
| 67 | \n", - "2022-08-01 | \n", - "19954.390625 | \n", - "
| 68 | \n", - "2022-09-01 | \n", - "17222.830078 | \n", - "
69 rows × 2 columns
\n", - "| \n", - " | Close | \n", - "
|---|---|
| Date | \n", - "\n", - " |
| 2017-01-01 | \n", - "23360.779297 | \n", - "
| 2017-02-01 | \n", - "23740.730469 | \n", - "
| 2017-03-01 | \n", - "24111.589844 | \n", - "
| 2017-04-01 | \n", - "24615.130859 | \n", - "
| 2017-05-01 | \n", - "25660.650391 | \n", - "
| ... | \n", - "... | \n", - "
| 2022-05-01 | \n", - "21415.199219 | \n", - "
| 2022-06-01 | \n", - "21859.789063 | \n", - "
| 2022-07-01 | \n", - "20156.509766 | \n", - "
| 2022-08-01 | \n", - "19954.390625 | \n", - "
| 2022-09-01 | \n", - "17222.830078 | \n", - "
69 rows × 1 columns
\n", - "| \n", - " | Close | \n", - "Close_diff | \n", - "
|---|---|---|
| Date | \n", - "\n", - " | \n", - " |
| 2017-01-01 | \n", - "23360.779297 | \n", - "-90.263959 | \n", - "
| 2017-02-01 | \n", - "23740.730469 | \n", - "379.951172 | \n", - "
| 2017-03-01 | \n", - "24111.589844 | \n", - "370.859375 | \n", - "
| 2017-04-01 | \n", - "24615.130859 | \n", - "503.541015 | \n", - "
| 2017-05-01 | \n", - "25660.650391 | \n", - "1045.519532 | \n", - "
| ... | \n", - "... | \n", - "... | \n", - "
| 2022-05-01 | \n", - "21415.199219 | \n", - "325.808594 | \n", - "
| 2022-06-01 | \n", - "21859.789063 | \n", - "444.589844 | \n", - "
| 2022-07-01 | \n", - "20156.509766 | \n", - "-1703.279297 | \n", - "
| 2022-08-01 | \n", - "19954.390625 | \n", - "-202.119141 | \n", - "
| 2022-09-01 | \n", - "17222.830078 | \n", - "-2731.560547 | \n", - "
69 rows × 2 columns
\n", - "| \n", - " | (p, d, q) | \n", - "AIC | \n", - "
|---|---|---|
| 0 | \n", - "(1, 1, 1) | \n", - "1173.487398 | \n", - "
| 1 | \n", - "(2, 1, 1) | \n", - "1173.782953 | \n", - "
| 2 | \n", - "(1, 1, 2) | \n", - "1174.070595 | \n", - "
| 3 | \n", - "(0, 1, 3) | \n", - "1174.126408 | \n", - "
| 4 | \n", - "(0, 1, 2) | \n", - "1174.267185 | \n", - "
| 5 | \n", - "(0, 1, 1) | \n", - "1174.458121 | \n", - "
| 6 | \n", - "(2, 1, 2) | \n", - "1175.781483 | \n", - "
| 7 | \n", - "(3, 1, 1) | \n", - "1175.781844 | \n", - "
| 8 | \n", - "(0, 1, 4) | \n", - "1175.993243 | \n", - "
| 9 | \n", - "(3, 1, 2) | \n", - "1177.455039 | \n", - "
| 10 | \n", - "(4, 1, 1) | \n", - "1177.757546 | \n", - "
| 11 | \n", - "(0, 1, 5) | \n", - "1177.791493 | \n", - "
| 12 | \n", - "(3, 1, 3) | \n", - "1178.185211 | \n", - "
| 13 | \n", - "(4, 1, 2) | \n", - "1179.382167 | \n", - "
| 14 | \n", - "(0, 1, 6) | \n", - "1179.727750 | \n", - "
| 15 | \n", - "(5, 1, 1) | \n", - "1179.743136 | \n", - "
| 16 | \n", - "(5, 1, 2) | \n", - "1181.345726 | \n", - "
| 17 | \n", - "(6, 1, 1) | \n", - "1181.725322 | \n", - "
| 18 | \n", - "(0, 1, 7) | \n", - "1181.725832 | \n", - "
| 19 | \n", - "(5, 1, 3) | \n", - "1183.357599 | \n", - "
| 20 | \n", - "(7, 1, 1) | \n", - "1183.613455 | \n", - "
| 21 | \n", - "(7, 1, 2) | \n", - "1184.133557 | \n", - "
| 22 | \n", - "(3, 1, 0) | \n", - "1184.463428 | \n", - "
| 23 | \n", - "(4, 1, 0) | \n", - "1184.717132 | \n", - "
| 24 | \n", - "(7, 1, 3) | \n", - "1184.986355 | \n", - "
| 25 | \n", - "(5, 1, 0) | \n", - "1185.454265 | \n", - "
| 26 | \n", - "(2, 1, 0) | \n", - "1186.435658 | \n", - "
| 27 | \n", - "(6, 1, 0) | \n", - "1186.771205 | \n", - "
| 28 | \n", - "(7, 1, 0) | \n", - "1187.679874 | \n", - "
| 29 | \n", - "(1, 1, 0) | \n", - "1190.687542 | \n", - "
| 30 | \n", - "(0, 1, 0) | \n", - "1227.389901 | \n", - "
"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "5c33f96b",
- "metadata": {
- "id": "5c33f96b",
- "outputId": "560c037e-253a-4a43-e44d-57a3051f96d6"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Requirement already satisfied: plotly in c:\\users\\jane wong\\anaconda3\\lib\\site-packages (5.11.0)\n",
- "Requirement already satisfied: tenacity>=6.2.0 in c:\\users\\jane wong\\anaconda3\\lib\\site-packages (from plotly) (8.0.1)\n",
- "Note: you may need to restart the kernel to use updated packages.\n"
- ]
- }
- ],
- "source": [
- "pip install plotly"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1a23ca7e",
- "metadata": {
- "id": "1a23ca7e",
- "outputId": "f9f72459-d9d5-4187-acfc-0ac20ce8b75f"
- },
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Building prefix dict from C:\\Users\\Jane Wong\\OneDrive - HKUST Connect\\ESG\\dict.txt ...\n",
- "Loading model from cache C:\\Users\\JANEWO~1\\AppData\\Local\\Temp\\jieba.u4dcdda7161e8cb4bbea6dffafa6acf12.cache\n",
- "Loading model cost 1.510 seconds.\n",
- "Prefix dict has been built successfully.\n"
- ]
- }
- ],
- "source": [
- "import statsmodels\n",
- "import requests\n",
- "from bs4 import BeautifulSoup\n",
- "from selenium.webdriver import Chrome\n",
- "from selenium.webdriver.common.keys import Keys\n",
- "from selenium import webdriver\n",
- "\n",
- "import jieba\n",
- "import jieba.analyse\n",
- "jieba.initialize()\n",
- "jieba.set_dictionary('dict.txt')\n",
- "\n",
- "import yfinance as yf\n",
- "\n",
- "import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "import pandas as pd\n",
- "import random\n",
- "\n",
- "import numpy as np\n",
- "import tensorflow as tf\n",
- "import pandas as pd\n",
- "pd.options.mode.chained_assignment = None\n",
- "import seaborn as sns\n",
- "from matplotlib.pylab import rcParams\n",
- "import matplotlib.pyplot as plt\n",
- "import plotly\n",
- "import plotly.express as px\n",
- "import plotly.graph_objects as go\n",
- "import datetime as dt\n",
- "\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "from sklearn.feature_extraction.text import TfidfVectorizer\n",
- "from sklearn.feature_extraction.text import CountVectorizer\n",
- "import pyLDAvis\n",
- "from __future__ import print_function\n",
- "import pyLDAvis\n",
- "import pyLDAvis.sklearn\n",
- "pyLDAvis.enable_notebook()\n",
- "from sklearn.datasets import fetch_20newsgroups\n",
- "from sklearn.decomposition import LatentDirichletAllocation\n",
- "from sklearn.preprocessing import StandardScaler\n",
- "from sklearn.decomposition import PCA\n",
- "from sklearn.covariance import EllipticEnvelope\n",
- "from sklearn.ensemble import IsolationForest\n",
- "\n",
- "%matplotlib inline\n",
- "\n",
- "sns.set(style='whitegrid', palette='muted')\n",
- "rcParams['figure.figsize'] = 14, 8\n",
- "np.random.seed(0)\n",
- "tf.random.set_seed(0)\n",
- "\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ab11c0c8",
- "metadata": {
- "id": "ab11c0c8",
- "outputId": "f23983a2-c0bf-4889-af27-6151c35ae229"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "C:\\Users\\Jane Wong\\anaconda3\\lib\\site-packages\\matplotlib\\mpl-data\\matplotlibrc\n"
- ]
- }
- ],
- "source": [
- "import matplotlib\n",
- "print(matplotlib.matplotlib_fname())"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e5f09f18",
- "metadata": {
- "id": "e5f09f18"
- },
- "source": [
- "# Import Preprocessed Headlines"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ef898947",
- "metadata": {
- "id": "ef898947",
- "outputId": "3aac0a0c-75f4-4c63-cd91-65005c68f520"
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "| \n", - " | date | \n", - "headline | \n", - "segmented headline | \n", - "keywords | \n", - "
|---|---|---|---|---|
| 0 | \n", - "2017-01-01 | \n", - "房委會綠建環評社區獲鉑金評級 | \n", - "房委會 綠建 環評 社區 獲 鉑金 評級 | \n", - "房委會 綠建 環評 社區 鉑金 評級 | \n", - "
| 1 | \n", - "2017-01-01 | \n", - "發祥街西項目獲鉑金評級 | \n", - "發祥 街西 項目 獲 鉑金 評級 | \n", - "發祥 街西 項目 鉑金 評級 | \n", - "
| 2 | \n", - "2017-01-01 | \n", - "潘基文告別寄語聯國為無聲者吶喊 | \n", - "潘基文 告別 寄語 聯國為 無聲 者 吶喊 | \n", - "告別 寄語 聯國為 無聲 吶喊 潘基文 | \n", - "
| 3 | \n", - "2017-01-01 | \n", - "潘基文告別聯合國 | \n", - "潘基文 告別 聯合國 | \n", - "告別 聯合國 潘基文 | \n", - "
| 4 | \n", - "2017-01-01 | \n", - "兩岸國際潘基文告別寄語聯國為無聲者吶喊 | \n", - "兩岸 國際 潘基文 告別 寄語 聯國為 無聲 者 吶喊 | \n", - "兩岸 國際 告別 寄語 聯國為 無聲 吶喊 潘基文 | \n", - "
| ... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "
| 321941 | \n", - "2022-09-30 | \n", - "俄羅斯指西方涉蓄意破壞北溪天然氣管道美國稱推測元兇為時尚早 | \n", - "俄羅斯 指 西方 涉 蓄意 破壞 北溪 天然氣 管道 美國 稱 推測 元兇 為 時尚 早 | \n", - "俄羅斯 破壞 天然氣 美國 推測 元兇 時尚 北溪 蓄意 管道 西方 | \n", - "
| 321942 | \n", - "2022-09-30 | \n", - "美國辦太平洋島國峰會被指抗衡中國影響力中方重申不搞經濟脅迫 | \n", - "美 國辦 太平洋 島國 峰會 被 指 抗衡 中國 影響力 中方 重申 不 搞 經濟 脅迫 | \n", - "國辦 島國 峰會 中國 影響力 經濟 脅迫 抗衡 重申 中方 太平洋 | \n", - "
| 321943 | \n", - "2022-09-30 | \n", - "澳門旅議會料國慶訪澳旅客增至三萬恢復內地團後旅客人數將再增 | \n", - "澳門 旅 議會 料 國慶 訪澳 旅客 增至 三萬 恢復 內地 團後 旅客 人數 將再 增 | \n", - "旅客 澳門 議會 國慶 訪澳 三萬 恢復 內地 團後 人數 將再 增至 | \n", - "
| 321944 | \n", - "2022-09-30 | \n", - "謝安琪與母女檔拍親子裝大讚囡囡表現專業 | \n", - "謝安琪 與 母女 檔 拍 親子裝 大 讚 囡 囡 表現 專業 | \n", - "謝安琪 親子裝 表現 專業 母女 | \n", - "
| 321945 | \n", - "2022-09-30 | \n", - "有狂風雷暴日間短暫時間有陽光最高度 | \n", - "有 狂風 雷暴 日間 短暫 時間 有 陽光 最 高度 | \n", - "狂風 日間 短暫 時間 陽光 雷暴 高度 | \n", - "
321946 rows × 4 columns
\n", - "| \n", - " | index | \n", - "date | \n", - "headline | \n", - "segmented headline | \n", - "keywords | \n", - "
|---|---|---|---|---|---|
| 0 | \n", - "22 | \n", - "2017-01-01 | \n", - "中國今發布南沙海洋環境預報 | \n", - "中國 今發布 南沙 海洋環境 預報 | \n", - "中國 今發布 海洋環境 預報 南沙 | \n", - "
| 1 | \n", - "113 | \n", - "2017-01-03 | \n", - "辦理低保豈能成吃拿卡要的藉口 | \n", - "辦理 低保 豈能 成 吃拿卡要 的 藉口 | \n", - "吃拿卡要 辦理 豈能 藉口 低保 | \n", - "
| 2 | \n", - "246 | \n", - "2017-01-04 | \n", - "國立傳統藝術中心颱風影響工程對外開放時程延後 | \n", - "國立 傳統 藝術 中心 颱風 影響 工程 對外開放 時程 延後 | \n", - "國立 傳統 藝術 颱風 影響 對外開放 時程 延後 工程 中心 | \n", - "
| 3 | \n", - "335 | \n", - "2017-01-05 | \n", - "六成高空氣污染日數受內地影響環保署不排除霧霾襲港 | \n", - "六成 高 空氣污染 日數 受 內地 影響 環保署 不 排除 霧 霾 襲港 | \n", - "空氣污染 日數 內地 影響 環保署 襲港 六成 排除 | \n", - "
| 4 | \n", - "360 | \n", - "2017-01-06 | \n", - "中國最安全城市排行榜香港第一拉薩第二 | \n", - "中國 最 安全 城市 排行榜 香港 第一 拉薩 第二 | \n", - "中國 拉薩 排行榜 香港 安全 第二 第一 城市 | \n", - "
| ... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "... | \n", - "
| 3211 | \n", - "321298 | \n", - "2022-09-28 | \n", - "李家超指當局銳意發展創科注入新經濟動力 | \n", - "李家 超指 當局 銳意 發展 創科 注入 新 經濟 動力 | \n", - "超指 當局 銳意 發展 創科 經濟 動力 李家 注入 | \n", - "
| 3212 | \n", - "321543 | \n", - "2022-09-29 | \n", - "級颶風撲佛州萬人疏散 | \n", - "級 颶風 撲 佛州 萬人 疏散 | \n", - "颶風 佛州 萬人 疏散 | \n", - "
| 3213 | \n", - "321552 | \n", - "2022-09-29 | \n", - "綠色和平極端酷熱工友極危險拒絕紙老虎保障 | \n", - "綠色 和平 極端 酷熱 工友 極 危險 拒絕 紙老虎 保障 | \n", - "綠色 極端 酷熱 危險 拒絕 紙老虎 工友 和平 保障 | \n", - "
| 3214 | \n", - "321840 | \n", - "2022-09-30 | \n", - "美國公布太平洋夥伴關係宣言所羅門群島簽署 | \n", - "美國 公布 太平洋 夥伴關係 宣言 所羅門 群島 簽署 | \n", - "美國 夥伴關係 所羅門 群島 簽署 宣言 太平洋 公布 | \n", - "
| 3215 | \n", - "321850 | \n", - "2022-09-30 | \n", - "英皇集團加盟平台網上售戲飛捐支持植樹 | \n", - "英皇 集團 加盟 平台 網上 售戲 飛捐 支持 植樹 | \n", - "集團 網上 售戲 飛捐 植樹 英皇 加盟 平台 支持 | \n", - "
3216 rows × 5 columns
\n", - "