01_NEODyS_astro_dynamical_properties.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7962dae7-1d3b-499a-a540-7518ced890c4",
   "metadata": {},
   "source": [
    "# Project: NeoExoSphyre Detecting near Earth objects"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9225d76d",
   "metadata": {},
   "source": [
    "### Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9cbb3ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import requests\n",
    "import pathlib\n",
    "import sqlite3\n",
    "\n",
    "# Define the directory and name of the NEODyS data\n",
    "raw_data_dir = pathlib.Path(\"raw_data/\")\n",
    "raw_data_file = pathlib.Path(\"neodys.cat\")\n",
    "\n",
    "raw_data_filepath = raw_data_dir / raw_data_file\n",
    "print(f\"Our NEODyS file path: {raw_data_filepath}\")\n",
    "\n",
    "pathlib.Path.mkdir(raw_data_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "264ededa",
   "metadata": {},
   "source": [
    "### Get the current number of known NEOs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "80304643-f3e8-4025-a7e6-8e0253bd5b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "http_response = requests.get(\"https://newton.spacedys.com/neodys/index.php?pc=1.0\")\n",
    "html_content = http_response.content\n",
    "\n",
    "# Extract the number of NEOs from a specific HTML position, using a regular expression. The\n",
    "# number is displayed in bold like \"[...] <b>1000 objects</b> in the NEODys [...]\"\n",
    "neodys_nr_neos = int(re.findall(r\"<b>(.*?) objects</b> in the NEODyS\", str(html_content))[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "72528a60-d661-4f2f-9005-fdbcde4f161f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of currently known NEOs: 37416\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of currently known NEOs: {neodys_nr_neos}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b02174ca",
   "metadata": {},
   "source": [
    "### Download the NEODyS file and store it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3617f316-9d47-493e-858a-5fba0115903f",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = requests.get(\"https://newton.spacedys.com/~neodys2/neodys.cat\")\n",
    "download_file_path = pathlib.Path(raw_data_filepath)\n",
    "with download_file_path.open(mode=\"wb+\") as file_obj:\n",
    "    file_obj.write(response.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ab285dd9-39d1-4bdc-a750-1cc25b055758",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set a placeholder dictionary where the data will be stored\n",
    "neo_dict = []\n",
    "\n",
    "# Open the NEODyS file. Ignore the header (first 6 rows) and iterate through the file row-wise.\n",
    "# Read the content adn save it in the dictionary\n",
    "with open(raw_data_filepath) as f_temp:\n",
    "    neo_data = f_temp.readlines()[6:]\n",
    "    for neo_data_line_f in neo_data:\n",
    "        neo_data_line = neo_data_line_f.split()\n",
    "        neo_dict.append(\n",
    "            {\n",
    "                \"Name\": neo_data_line[0].replace(\"'\", \"\"),\n",
    "                \"Epoch_MJD\": float(neo_data_line[1]),\n",
    "                \"SemMajAxis_AU\": float(neo_data_line[2]),\n",
    "                \"Ecc_\": float(neo_data_line[3]),\n",
    "                \"Incl_deg\": float(neo_data_line[4]),\n",
    "                \"LongAscNode_deg\": float(neo_data_line[5]),\n",
    "                \"ArgP_deg\": float(neo_data_line[6]),\n",
    "                \"MeanAnom_deg\": float(neo_data_line[7]),\n",
    "                \"AbsMag_\": float(neo_data_line[8]),\n",
    "                \"SlopeParamG_\": float(neo_data_line[9]),\n",
    "            }\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f72c40e1-ec6d-44df-91b6-fe3110f65b7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Does the file contain the same number of NEOs as the NEODyS website? \n",
      ">Yes\n"
     ]
    }
   ],
   "source": [
    "print(f\"Does the file contain the same number of NEOs as the NEODyS website? \\n\"\n",
    "      f\"{'>Yes' if len(neo_dict) == neodys_nr_neos else '>No'}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25975120",
   "metadata": {},
   "source": [
    "### Let's take a look!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d88dbef2-4c16-41c3-8d2e-be3cb903d755",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'Name': '433',\n",
       "  'Epoch_MJD': 60000.0,\n",
       "  'SemMajAxis_AU': 1.458129137561811,\n",
       "  'Ecc_': 0.22278187193561982,\n",
       "  'Incl_deg': 10.827822059970396,\n",
       "  'LongAscNode_deg': 304.2870240036129,\n",
       "  'ArgP_deg': 178.92698564290663,\n",
       "  'MeanAnom_deg': 110.77766380976027,\n",
       "  'AbsMag_': 10.91,\n",
       "  'SlopeParamG_': 0.46},\n",
       " {'Name': '719',\n",
       "  'Epoch_MJD': 60600.0,\n",
       "  'SemMajAxis_AU': 2.636157149048896,\n",
       "  'Ecc_': 0.5467795293144841,\n",
       "  'Incl_deg': 11.575260341822329,\n",
       "  'LongAscNode_deg': 183.8571635400628,\n",
       "  'ArgP_deg': 156.2155185178333,\n",
       "  'MeanAnom_deg': 148.45067666976948,\n",
       "  'AbsMag_': 15.54,\n",
       "  'SlopeParamG_': 0.15}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neo_dict[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55f896ff",
   "metadata": {},
   "source": [
    "### Now we create the NEODyS SQLite database for our future project work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e09fefcf-ba84-491e-97a3-7f753d29145e",
   "metadata": {},
   "outputs": [],
   "source": [
    "database_dir = pathlib.Path(\"../databases/neos/\")\n",
    "database_file = pathlib.Path(\"neodys.db\")\n",
    "database_filepath = database_dir / database_file\n",
    "\n",
    "# Create the directory\n",
    "pathlib.Path.mkdir(database_dir, parents=True, exist_ok=True)\n",
    "\n",
    "# Establish a connection to the database and set a cursor\n",
    "neodys_db_con = sqlite3.connect(database_filepath)\n",
    "neodys_db_cur = neodys_db_con.cursor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we create the NEODyS SQLite database for our future project work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "65b7aa38-24f5-499d-a037-9271ba121832",
   "metadata": {},
   "outputs": [],
   "source": [
    "neodys_db_cur.execute(\n",
    "    \"CREATE TABLE IF NOT EXISTS main(Name TEXT PRIMARY KEY, \"\n",
    "    \"Epoch_MJD FLOAT, \"\n",
    "    \"SemMajAxis_AU FLOAT, \"\n",
    "    \"Ecc_ FLOAT, \"\n",
    "    \"Incl_deg FLOAT, \"\n",
    "    \"LongAscNode_deg FLOAT, \"\n",
    "    \"ArgP_deg FLOAT, \"\n",
    "    \"MeanAnom_deg FLOAT, \"\n",
    "    \"AbsMag_ FLOAT, \"\n",
    "    \"SlopeParamG_ FLOAT)\"\n",
    ")\n",
    "neodys_db_con.commit()\n",
    "\n",
    "# Insert the raw data into the database\n",
    "neodys_db_cur.executemany(\n",
    "    \"INSERT OR IGNORE INTO main(Name, \"\n",
    "    \"Epoch_MJD, \"\n",
    "    \"SemMajAxis_AU, \"\n",
    "    \"Ecc_, \"\n",
    "    \"Incl_deg, \"\n",
    "    \"LongAscNode_deg, \"\n",
    "    \"ArgP_deg, \"\n",
    "    \"MeanAnom_deg, \"\n",
    "    \"AbsMag_, \"\n",
    "    \"SlopeParamG_) \"\n",
    "    \"VALUES (:Name, \"\n",
    "    \":Epoch_MJD, \"\n",
    "    \":SemMajAxis_AU, \"\n",
    "    \":Ecc_, \"\n",
    "    \":Incl_deg, \"\n",
    "    \":LongAscNode_deg, \"\n",
    "    \":ArgP_deg, \"\n",
    "    \":MeanAnom_deg, \"\n",
    "    \":AbsMag_, \"\n",
    "    \":SlopeParamG_)\",\n",
    "    neo_dict,\n",
    ")\n",
    "neodys_db_con.commit()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5f01aeb",
   "metadata": {},
   "source": [
    "### Add new columns in the main table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e9b59f2f-49de-4695-a810-5c501431854e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for col_name in [\"Aphel_AU\", \"Perihel_AU\"]:\n",
    "    \n",
    "    # SQL Query for creating new columns\n",
    "    sql_col_create = f\"ALTER TABLE main ADD COLUMN {col_name} FLOAT\"\n",
    "\n",
    "    # Try to create a new column. If it exists an sqlite3.OperationalError will raise. Pass this error.\n",
    "    try:\n",
    "        neodys_db_cur.execute(sql_col_create)\n",
    "        neodys_db_con.commit()\n",
    "    except sqlite3.OperationalError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ef9d50a",
   "metadata": {},
   "source": [
    "### Get orbital elements to compute the derived parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "985670b8-63e2-4296-8744-b80aa422cc5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "neodys_db_cur.execute(\"SELECT Name, SemMajAxis_AU, Ecc_ FROM main\")\n",
    "\n",
    "# Fetch the data\n",
    "_neo_data = neodys_db_cur.fetchall()\n",
    "\n",
    "# Iterate throuh the results, compute the derived elements and put them in a list of\n",
    "# dicitionaries\n",
    "_neo_deriv_param_dict = []\n",
    "for _neo_data_line_f in _neo_data:\n",
    "    _neo_deriv_param_dict.append(\n",
    "        {\n",
    "            \"Name\": _neo_data_line_f[0],\n",
    "            \"Aphel_AU\": (1.0 + _neo_data_line_f[2]) * _neo_data_line_f[1],\n",
    "            \"Perihel_AU\": (1.0 - _neo_data_line_f[2]) * _neo_data_line_f[1],\n",
    "        }\n",
    "    )\n",
    "\n",
    "# Insert the data into the main table\n",
    "neodys_db_cur.executemany(\n",
    "    \"UPDATE main SET Aphel_AU = :Aphel_AU, Perihel_AU = :Perihel_AU \"\n",
    "    \"WHERE Name = :Name\",\n",
    "    _neo_deriv_param_dict,\n",
    ")\n",
    "neodys_db_con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2831bf1c-e81e-4ec8-8532-3ea51b9802ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "neodys_db_con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cfc217b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import modules\n",
    "import pathlib\n",
    "import sqlite3"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cafdc866",
   "metadata": {},
   "source": [
    "### Now we create the NEODyS SQLite database for our future project work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b11f0fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "database_dir = pathlib.Path(\"../databases/neos/\")\n",
    "database_file = pathlib.Path(\"neodys.db\")\n",
    "database_filepath = database_dir / database_file\n",
    "\n",
    "# Create the directory\n",
    "pathlib.Path.mkdir(database_dir, parents=True, exist_ok=True)\n",
    "\n",
    "# Establish a connection to the database and set a cursor\n",
    "neodys_db_con = sqlite3.connect(database_filepath)\n",
    "neodys_db_cur = neodys_db_con.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "edbde816",
   "metadata": {},
   "outputs": [],
   "source": [
    "def neo_class(sem_maj_axis_au: float,\n",
    "              peri_helio_au: float,\n",
    "              ap_helio_au: float) -> str:\n",
    "    \"\"\"Classify the NEO based on the orbital parameters.\n",
    "    Depending on the semi-major axis, perihelion and / or aphelion a NEO can be classified as an\n",
    "    Amor, Apollo, Aten or Atira.\n",
    "    Parameters\n",
    "    ----------\n",
    "    sem_maj_axis_au : float\n",
    "        Semi-major axis of the NEO. Given in AU\n",
    "    peri_helio_au : float\n",
    "        Perihelion of the NEO. Given in AU\n",
    "    ap_helio_au : float\n",
    "        Aphelion of the NEO. Given in AU\n",
    "    Returns\n",
    "    -------\n",
    "    neo_type : str\n",
    "        NEO class / type.\n",
    "    References\n",
    "    ----------\n",
    "    -1- Link to the NEO classifiction schema: https://cneos.jpl.nasa.gov/about/neo_groups.html\n",
    "    \"\"\"\n",
    "    # Determine the NEO class in an extensive if-else statement\n",
    "    if (sem_maj_axis_au > 1.0) & (1.017 < peri_helio_au < 1.3):\n",
    "        neo_type = 'Amor'\n",
    "\n",
    "    elif (sem_maj_axis_au > 1.0) & (peri_helio_au < 1.017):\n",
    "        neo_type = 'Apollo'\n",
    "\n",
    "    elif (sem_maj_axis_au < 1.0) & (ap_helio_au > 0.983):\n",
    "        neo_type = 'Aten'\n",
    "\n",
    "    elif (sem_maj_axis_au < 1.0) & (ap_helio_au < 0.983):\n",
    "        neo_type = 'Atira'\n",
    "\n",
    "    else:\n",
    "        neo_type = 'Other'\n",
    "\n",
    "    return neo_type"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21142d82",
   "metadata": {},
   "source": [
    "### Testing the function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "251dcd18",
   "metadata": {},
   "outputs": [],
   "source": [
    "amor1221 = [1.9191, 1.0832, 2.7550]\n",
    "apollo1862 = [1.4702, 0.64699, \t2.2935]\n",
    "aten2062 = [0.9668, 0.7901, 1.1434]\n",
    "atira163693 = [0.7411, 0.5024, 0.9798]\n",
    "\n",
    "assert \"Amor\" == neo_class(*amor1221)\n",
    "assert \"Apollo\" == neo_class(*apollo1862)\n",
    "assert \"Aten\" == neo_class(*aten2062)\n",
    "assert \"Atira\" == neo_class(*atira163693)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "30a56cd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_col(sqlitecon : sqlite3.Connection, \n",
    "               sqlitecur: sqlite3.Cursor,\n",
    "               table: str,\n",
    "               col_name: str,\n",
    "               col_type: str) -> None:\n",
    "    \"\"\"\n",
    "    Function to create new columns in tables.\n",
    "    Parameters\n",
    "    ----------\n",
    "    sqlitecon : sqlite3.Connection\n",
    "        Connection to the SQLite database.\n",
    "    sqlitecur : sqlite3.Cursor\n",
    "        Cursor that points to the SQLite database.\n",
    "    sqlitetable : str\n",
    "        Table name, where a new column shall be added.\n",
    "    col_name : str\n",
    "        Column name.\n",
    "    col_type : str\n",
    "        SQLite column type (FLOAT, INT, TEXT, etc.).\n",
    "    \"\"\"\n",
    "    # Generic f-string that represents an SQLite command to alter a table (adding a new column\n",
    "    # with its dtype).\n",
    "    sql_col_create = f\"ALTER TABLE {table} ADD COLUMN {col_name} {col_type}\"\n",
    "\n",
    "    # Try to create a new column. If is exists an sqlite3.OperationalError will raise. Pass\n",
    "    # this error.\n",
    "    try:\n",
    "        sqlitecur.execute(sql_col_create)\n",
    "        sqlitecon.commit()\n",
    "    except sqlite3.OperationalError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5d2ee50",
   "metadata": {},
   "source": [
    "### Add a new column in the main table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5c16c508",
   "metadata": {},
   "outputs": [],
   "source": [
    "create_col(sqlitecon=neodys_db_con,\n",
    "           sqlitecur=neodys_db_cur,\n",
    "           table=\"main\",\n",
    "           col_name=\"NEOClass\",\n",
    "           col_type=\"Text\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47d2ecfb",
   "metadata": {},
   "source": [
    "### Get the orbital elements to compute the NEO class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "98f4785c",
   "metadata": {},
   "outputs": [],
   "source": [
    "neodys_db_cur.execute(\"SELECT Name, SemMajAxis_AU, Perihel_AU, Aphel_AU FROM main\")\n",
    "\n",
    "# Fetch the data\n",
    "neo_data = neodys_db_cur.fetchall()\n",
    "\n",
    "# Iterate throuh the results, compute the NEO class and put in into the results\n",
    "neo_class_param_dict = []\n",
    "for neo_data_row in neo_data:\n",
    "    neo_class_param_dict.append(\n",
    "        {\n",
    "            \"Name\": neo_data_row[0],\n",
    "            \"NEOClass\": neo_class(\n",
    "                sem_maj_axis_au=neo_data_row[1],\n",
    "                peri_helio_au=neo_data_row[2],\n",
    "                ap_helio_au=neo_data_row[3]\n",
    "            )\n",
    "        }\n",
    "    )\n",
    "\n",
    "# Insert the data into the main table\n",
    "neodys_db_cur.executemany(\n",
    "    \"UPDATE main SET NEOClass = :NEOClass \"\n",
    "    \"WHERE Name = :Name\",\n",
    "    neo_class_param_dict,\n",
    ")\n",
    "neodys_db_con.commit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a2e2ef85",
   "metadata": {},
   "outputs": [],
   "source": [
    "neodys_db_con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9699e4a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}