update all options cron jobs

app/cron_implied_volatility.py

@@ -1,20 +1,16 @@
 import requests
 import orjson
+import ujson
 import re
 from datetime import datetime
-from dotenv import load_dotenv
 import os
 import sqlite3
 import time
 from tqdm import tqdm
+import pandas as pd
+import numpy as np
 
-load_dotenv()
 
-api_key = os.getenv('UNUSUAL_WHALES_API_KEY')
-querystring = {"timeframe":"5Y"}
-headers = {"Accept": "application/json, text/plain", "Authorization": api_key}
-
-# Connect to the databases
 con = sqlite3.connect('stocks.db')
 etf_con = sqlite3.connect('etf.db')
 cursor = con.cursor()
@@ -47,112 +43,121 @@ def get_tickers_from_directory(directory: str):
         return []
 
 
-def save_json(data, symbol, directory_path):
+def convert_to_serializable(obj):
+    if isinstance(obj, np.float64):
+        return float(obj)
+    elif isinstance(obj, (np.int64, np.int32)):
+        return int(obj)
+    elif isinstance(obj, (list, np.ndarray)):
+        return [convert_to_serializable(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {key: convert_to_serializable(value) for key, value in obj.items()}
+    else:
+        return obj
+
+def save_json(data, symbol):
+    directory_path = "json/implied-volatility"
     os.makedirs(directory_path, exist_ok=True)  # Ensure the directory exists
+
+    # Convert numpy types to JSON-serializable types
+    serializable_data = convert_to_serializable(data)
+
     with open(f"{directory_path}/{symbol}.json", 'wb') as file:  # Use binary mode for orjson
-        file.write(orjson.dumps(data))
+        file.write(orjson.dumps(serializable_data))
 
 
-def safe_round(value, decimals=2):
-    try:
-        return round(float(value), decimals)
-    except (ValueError, TypeError):
-        return value
-
-
-def add_data(data, historical_data):
-    res_list = []
-    for item in data:
-        date = item['date']
-        for item2 in historical_data:
-            try:
-                if date == item2['date']:
-                    item['changesPercentage'] = item2['changesPercentage']
-                    item['putCallRatio'] = item2['putCallRatio']
-                    item['total_open_interest'] = item2['total_open_interest']
-                    item['changesPercentageOI'] = item2.get('changesPercentageOI',None)
-            except Exception as e:
-                print(e)
-
-        if 'changesPercentage' in item:
-            res_list.append(item)
+def compute_realized_volatility(data, window_size=20):
+    """
+    Compute the realized volatility of stock prices over a rolling window.
+    Realized volatility is the annualized standard deviation of log returns of stock prices.
+    """
+    # Sort data by date (oldest first)
+    data = sorted(data, key=lambda x: x['date'])
 
-    return res_list
-
-
-
-def prepare_data(data, symbol, directory_path, sort_by = "date"):
-    res_list = []
-    for item in data:
-        try:
-            new_item = {
-                key: safe_round(value) if isinstance(value, (int, float, str)) else value
-                for key, value in item.items()
-            }
-
-            res_list.append(new_item)
-        except:
-            pass
-
-    if res_list:
-        data = sorted(res_list, key=lambda x: x[sort_by], reverse=True)
-        with open(f"json/options-historical-data/companies/{symbol}.json", "r") as file:
-            historical_data = orjson.loads(file.read())
-
-        res_list = add_data(data,historical_data)
-        save_json(res_list, symbol, directory_path)
-
-
-
-def get_iv_data():
-    print("Starting to download iv data...")
-    directory_path = "json/implied-volatility"
-    total_symbols = get_tickers_from_directory(directory_path)
-    if len(total_symbols) < 100:
-        total_symbols = stocks_symbols+etf_symbols
-
-    counter = 0
-    for symbol in tqdm(total_symbols):
-        try:
-            url = f"https://api.unusualwhales.com/api/stock/{symbol}/volatility/realized"
-
-            response = requests.get(url, headers=headers, params=querystring)
-            if response.status_code == 200:
-                data = response.json()['data']
-                prepare_data(data, symbol, directory_path)
-
-            counter +=1
-
-            # If 50 chunks have been processed, sleep for 60 seconds
-            if counter == 260:
-                print("Sleeping...")
-                time.sleep(60)
-                counter = 0
+    # Extract stock prices and dates
+    prices = [item.get('price') for item in data]  # Use .get() to handle missing keys
+    dates = [item['date'] for item in data]
+
+    # Compute log returns of stock prices, skipping None values
+    log_returns = []
+    for i in range(1, len(prices)):
+        if prices[i] is not None and prices[i - 1] is not None and prices[i - 1] != 0:
+            log_returns.append(np.log(prices[i] / prices[i - 1]))
+        else:
+            log_returns.append(None)  # Append None if price is missing or invalid
+
+    # Compute realized volatility using a rolling window
+    realized_volatility = []
+    for i in range(len(log_returns)):
+        if i < window_size - 1:
+            # Not enough data for the window, append None
+            realized_volatility.append(None)
+        else:
+            # Collect valid log returns in the window
+            window_returns = []
+            for j in range(i - window_size + 1, i + 1):
+                if log_returns[j] is not None:
+                    window_returns.append(log_returns[j])
 
+            if len(window_returns) >= window_size:
+                # Compute standard deviation of log returns over the window
+                rv_daily = np.sqrt(np.sum(np.square(window_returns)) / window_size)
+                # Annualize the realized volatility
+                rv_annualized = rv_daily * np.sqrt(252)
+                realized_volatility.append(rv_annualized)
+            else:
+                # Not enough valid data in the window, append None
+                realized_volatility.append(None)
+
+    # Shift realized volatility FORWARD by window_size days to align with IV from window_size days ago
+    realized_volatility = realized_volatility[window_size - 1:] + [None] * (window_size - 1)
+
+    # Create the resulting list
+    rv_list = []
+    for i in range(len(data)):
+        try:
+            rv_list.append({
+                "date": data[i]["date"],
+                "price": data[i].get("price"),  # Use .get() to handle missing keys
+                "changesPercentage": data[i].get("changesPercentage", None),  # Default to None if missing
+                "putCallRatio": data[i].get("putCallRatio", None),  # Default to None if missing
+                "total_open_interest": data[i].get("total_open_interest", None),  # Default to None if missing
+                "changesPercentageOI": data[i].get("changesPercentageOI", None),  # Default to None if missing
+                "iv": data[i].get("iv", None),  # Default to None if missing
+                "rv": round(realized_volatility[i], 4) if realized_volatility[i] is not None else None
+            })
         except Exception as e:
-            print(f"Error for {symbol}:{e}")
-
+            # If any error occurs, append a dictionary with default values
+            rv_list.append({
+                "date": data[i]["date"],
+                "price": data[i].get("price", None),
+                "changesPercentage": data[i].get("changesPercentage", None),
+                "putCallRatio": data[i].get("putCallRatio", None),
+                "total_open_interest": data[i].get("total_open_interest", None),
+                "changesPercentageOI": data[i].get("changesPercentageOI", None),
+                "iv": data[i].get("iv", None),
+                "rv": None
+            })
+
+    # Sort the final list by date in descending order
+    rv_list = sorted(rv_list, key=lambda x: x['date'], reverse=True)
+
+    return rv_list
 
 if __name__ == '__main__':
-    get_iv_data()
-
-    '''
     directory_path = "json/implied-volatility"
     total_symbols = get_tickers_from_directory(directory_path)
     if len(total_symbols) < 100:
-        total_symbols = stocks_symbols+etf_symbols
+        total_symbols = stocks_symbols + etf_symbols  # Assuming these are defined elsewhere
 
     for symbol in tqdm(total_symbols):
         try:
             with open(f"json/options-historical-data/companies/{symbol}.json", "r") as file:
-                historical_data = orjson.loads(file.read())
-
-            with open(f"json/implied-volatility/{symbol}.json", "r") as file:
                 data = orjson.loads(file.read())
 
-            res_list = add_data(data,historical_data)
+            rv_list = compute_realized_volatility(data)
 
-            save_json(res_list, symbol, directory_path)
+            if rv_list:
+                save_json(rv_list, symbol)
         except:
             pass
-    '''