From d83bdda60377af6add25d852d1870d5963386e35 Mon Sep 17 00:00:00 2001 From: TRPA-Sarah Date: Fri, 17 Jan 2025 09:02:03 -0800 Subject: [PATCH 1/2] AQ fire analysis with 2018 --- ...sholdEvaluation_DataEngineering_2023.ipynb | 176 ++++++++++-------- 1 file changed, 97 insertions(+), 79 deletions(-) diff --git a/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb b/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb index 15a87b3..722d9d4 100644 --- a/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb +++ b/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb @@ -1906,14 +1906,14 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 19, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "C:\\Users\\snewsome\\AppData\\Local\\Temp\\ipykernel_24192\\322407640.py:120: SettingWithCopyWarning: \n", + "C:\\Users\\snewsome\\AppData\\Local\\Temp\\ipykernel_24192\\3479171100.py:120: SettingWithCopyWarning: \n", "A value is trying to be set on a copy of a slice from a DataFrame.\n", "Try using .loc[row_indexer,col_indexer] = value instead\n", "\n", @@ -1935,8 +1935,8 @@ " r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ data 2021.xlsx\",\n", " r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ data 2020.xlsx\",\n", " r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ data 2019.xlsx\",\n", - " #r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ Data 2013-2018 Daily.xlsx\"]\n", - "]\n", + " r\"F:\\Research and Analysis\\Air Quality\\Annual Reports DRI\\Daily AQ data 2018 from AQ'd data 2018.xlsx\"]\n", + "\n", "# 2. Read Data from Each File Daily Data\n", "dfs = [] # List to store DataFrames from each file\n", "sheet_name = 'daily' # Name of the sheet to read\n", @@ -1970,7 +1970,7 @@ "DailyAir_df['STL O3max'] = pd.to_numeric(DailyAir_df['STL O3max'], errors='coerce')\n", "DailyAir_df['IV O3max'] = pd.to_numeric(DailyAir_df['IV O3max'], errors='coerce')\n", "DailyAir_df['BSP O3max'] = pd.to_numeric(DailyAir_df['BSP O3max'], errors='coerce')\n", - "#DailyAir_df['LTC O3max'] = pd.to_numeric(DailyAir_df['LTC O3max'], errors='coerce')\n", + "DailyAir_df['LTC O3max'] = pd.to_numeric(DailyAir_df['LTC O3max'], errors='coerce')\n", "\n", "#keep data from 2018-2023 --- Excel Table used has 2013-2018 data, before 2018 soo 2017 and earlier the data for STL PM2.5avg is actually PM10avg\n", "#DailyAir_df = DailyAir_df.loc[DailyAir_df['date'].dt.year >= 2018]\n", @@ -2050,7 +2050,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -2059,39 +2059,44 @@ "text": [ "With Fire Exemption Dates:\n", " Year Max_PM10 Threshold Value CA Threshold Value NAAQS\n", - "0 2019 43.0 50 150\n", - "1 2020 55.0 50 150\n", - "2 2021 46.0 50 150\n", - "3 2022 45.0 50 150\n", - "4 2023 39.0 50 150\n", + "0 2018 143.0 50 150\n", + "1 2019 43.0 50 150\n", + "2 2020 55.0 50 150\n", + "3 2021 46.0 50 150\n", + "4 2022 45.0 50 150\n", + "5 2023 39.0 50 150\n", "All Dates:\n", " Year Max_PM10 Threshold Value CA Threshold Value NAAQS\n", - "0 2019 43.0 50 150\n", - "1 2020 201.0 50 150\n", - "2 2021 492.0 50 150\n", - "3 2022 136.0 50 150\n", - "4 2023 39.0 50 150\n", + "0 2018 143.0 50 150\n", + "1 2019 43.0 50 150\n", + "2 2020 201.0 50 150\n", + "3 2021 492.0 50 150\n", + "4 2022 136.0 50 150\n", + "5 2023 39.0 50 150\n", "Combined PM10 Data:\n", " Year Exceedances_CA_exempt Exceedances_CA_all Exceedances_NAAQS_exempt \\\n", - "0 2019 0 0 0 \n", - "1 2020 1 20 0 \n", - "2 2021 0 15 0 \n", - "3 2022 0 2 0 \n", - "4 2023 0 0 0 \n", + "0 2018 8 8 0 \n", + "1 2019 0 0 0 \n", + "2 2020 1 20 0 \n", + "3 2021 0 15 0 \n", + "4 2022 0 2 0 \n", + "5 2023 0 0 0 \n", "\n", " Exceedances_NAAQS_all Exceedance_Threshold \\\n", "0 0 1 exceedance per year on average over 3 years \n", - "1 3 1 exceedance per year on average over 3 years \n", - "2 5 1 exceedance per year on average over 3 years \n", - "3 0 1 exceedance per year on average over 3 years \n", + "1 0 1 exceedance per year on average over 3 years \n", + "2 3 1 exceedance per year on average over 3 years \n", + "3 5 1 exceedance per year on average over 3 years \n", "4 0 1 exceedance per year on average over 3 years \n", + "5 0 1 exceedance per year on average over 3 years \n", "\n", " Max_PM10_exempt Max_PM10_all Threshold Value CA Threshold Value NAAQS \n", - "0 43.0 43.0 50 150 \n", - "1 55.0 201.0 50 150 \n", - "2 46.0 492.0 50 150 \n", - "3 45.0 136.0 50 150 \n", - "4 39.0 39.0 50 150 \n" + "0 143.0 143.0 50 150 \n", + "1 43.0 43.0 50 150 \n", + "2 55.0 201.0 50 150 \n", + "3 46.0 492.0 50 150 \n", + "4 45.0 136.0 50 150 \n", + "5 39.0 39.0 50 150 \n" ] } ], @@ -2166,7 +2171,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -2175,39 +2180,44 @@ "text": [ "Combined pm2.5 Data:\n", " Year Max_PM2.5_exempt Max_PM2.5_all 3_Year_Mean_98th_Percentile_exempt \\\n", - "0 2019 44.4 44.4 NaN \n", - "1 2020 30.7 124.7 NaN \n", - "2 2021 37.5 421.9 15.558667 \n", - "3 2022 25.0 296.3 17.494000 \n", - "4 2023 19.1 19.1 14.696667 \n", + "0 2018 72.2 72.2 NaN \n", + "1 2019 44.4 44.4 NaN \n", + "2 2020 30.7 124.7 19.604000 \n", + "3 2021 37.5 421.9 15.558667 \n", + "4 2022 25.0 296.3 17.494000 \n", + "5 2023 19.1 19.1 14.696667 \n", "\n", " 3_Year_Mean_98th_Percentile_all 98th_Percentile_PM2.5_exempt \\\n", - "0 NaN 5.144 \n", - "1 NaN 20.864 \n", - "2 63.008000 20.668 \n", - "3 71.474667 10.950 \n", - "4 53.812000 12.472 \n", + "0 NaN 32.804 \n", + "1 NaN 5.144 \n", + "2 34.469333 20.864 \n", + "3 63.008000 20.668 \n", + "4 71.474667 10.950 \n", + "5 53.812000 12.472 \n", "\n", " 98th_Percentile_PM2.5_all Exceedances_NAAQS_exempt Exceedances_NAAQS_all \\\n", - "0 5.144 1 1 \n", - "1 65.460 0 20 \n", - "2 118.420 1 23 \n", - "3 30.544 0 7 \n", - "4 12.472 0 0 \n", + "0 32.804 7 7 \n", + "1 5.144 1 1 \n", + "2 65.460 0 20 \n", + "3 118.420 1 23 \n", + "4 30.544 0 7 \n", + "5 12.472 0 0 \n", "\n", " 98Exceedances_NAAQS_exempt 98Exceedances_NAAQS_all Threshold Value CA \\\n", "0 0 0 None \n", - "1 0 1 None \n", + "1 0 0 None \n", "2 0 1 None \n", - "3 0 0 None \n", + "3 0 1 None \n", "4 0 0 None \n", + "5 0 0 None \n", "\n", " Threshold Value NAAQS \n", "0 35 \n", "1 35 \n", "2 35 \n", "3 35 \n", - "4 35 \n" + "4 35 \n", + "5 35 \n" ] } ], @@ -2216,11 +2226,11 @@ "######### PM 2.5 ###########\n", "#----------------------------------------------\n", "#CALCULATE 3 year average 98th percentil \n", - "#----------------------------------------------\n", - "# Calculate the 98th percentile for each year\n", + "# Calculate the 98th percentile for each year for both TC PM2.5avg and STL PM2.5avg\n", "PM25_98th_percentile_exempt = DailyExemptdf.groupby('Year').agg({'TC PM2.5avg': lambda x: x.quantile(0.98)}).reset_index()\n", "PM25_98th_percentile_all = DailyAir_df.groupby('Year').agg({'TC PM2.5avg': lambda x: x.quantile(0.98)}).reset_index()\n", "\n", + "\n", "# Rename columns for clarity\n", "PM25_98th_percentile_exempt.rename(columns={'TC PM2.5avg': '98th_Percentile_PM2.5'}, inplace=True)\n", "PM25_98th_percentile_all.rename(columns={'TC PM2.5avg': '98th_Percentile_PM2.5'}, inplace=True)\n", @@ -2312,7 +2322,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -2321,11 +2331,12 @@ "text": [ "With Fire Exemption Dates:\n", " Year TC O3max STL O3max IV O3max O3Threshold Value CA \\\n", - "0 2019 72.0 67.081518 67.0 90 \n", - "1 2020 78.0 73.333333 75.0 90 \n", - "2 2021 78.0 63.100000 88.0 90 \n", - "3 2022 86.0 75.853659 78.0 90 \n", - "4 2023 71.0 87.300000 73.0 90 \n", + "0 2018 90.0 71.000000 86.0 90 \n", + "1 2019 72.0 67.081518 67.0 90 \n", + "2 2020 78.0 73.333333 75.0 90 \n", + "3 2021 78.0 63.100000 88.0 90 \n", + "4 2022 86.0 75.853659 78.0 90 \n", + "5 2023 71.0 87.300000 73.0 90 \n", "\n", " O3Threshold Value NAAQS Threshold Value TRPA \n", "0 None 80 \n", @@ -2333,13 +2344,15 @@ "2 None 80 \n", "3 None 80 \n", "4 None 80 \n", + "5 None 80 \n", "All Dates:\n", " Year TC O3max STL O3max IV O3max Threshold Value CA \\\n", - "0 2019 72.0 67.081518 67.0 90 \n", - "1 2020 98.0 85.333333 96.0 90 \n", - "2 2021 98.0 81.200000 93.0 90 \n", - "3 2022 86.0 75.853659 80.0 90 \n", - "4 2023 71.0 87.300000 73.0 90 \n", + "0 2018 90.0 71.000000 86.0 90 \n", + "1 2019 72.0 67.081518 67.0 90 \n", + "2 2020 98.0 85.333333 96.0 90 \n", + "3 2021 98.0 81.200000 93.0 90 \n", + "4 2022 86.0 75.853659 80.0 90 \n", + "5 2023 71.0 87.300000 73.0 90 \n", "\n", " Threshold Value NAAQS Threshold Value TRPA \n", "0 None 80 \n", @@ -2347,34 +2360,39 @@ "2 None 80 \n", "3 None 80 \n", "4 None 80 \n", + "5 None 80 \n", "Combined O3 Data:\n", " Year Exceedances_TRPA_all Exceedances_TRPA_exempt Exceedances_CA_all \\\n", - "0 2019 0 0 0 \n", - "1 2020 9 0 3 \n", - "2 2021 13 1 4 \n", - "3 2022 1 1 0 \n", - "4 2023 1 1 0 \n", + "0 2018 7 7 0 \n", + "1 2019 0 0 0 \n", + "2 2020 9 0 3 \n", + "3 2021 13 1 4 \n", + "4 2022 1 1 0 \n", + "5 2023 1 1 0 \n", "\n", " Exceedances_CA_exempt TC O3max_exempt TC O3max_all STL O3max_exempt \\\n", - "0 0 72.0 72.0 67.081518 \n", - "1 0 78.0 98.0 73.333333 \n", - "2 0 78.0 98.0 63.100000 \n", - "3 0 86.0 86.0 75.853659 \n", - "4 0 71.0 71.0 87.300000 \n", + "0 0 90.0 90.0 71.000000 \n", + "1 0 72.0 72.0 67.081518 \n", + "2 0 78.0 98.0 73.333333 \n", + "3 0 78.0 98.0 63.100000 \n", + "4 0 86.0 86.0 75.853659 \n", + "5 0 71.0 71.0 87.300000 \n", "\n", " STL O3max_all IV O3max_all IV O3max_exempt Threshold Value CA \\\n", - "0 67.081518 67.0 67.0 90 \n", - "1 85.333333 96.0 75.0 90 \n", - "2 81.200000 93.0 88.0 90 \n", - "3 75.853659 80.0 78.0 90 \n", - "4 87.300000 73.0 73.0 90 \n", + "0 71.000000 86.0 86.0 90 \n", + "1 67.081518 67.0 67.0 90 \n", + "2 85.333333 96.0 75.0 90 \n", + "3 81.200000 93.0 88.0 90 \n", + "4 75.853659 80.0 78.0 90 \n", + "5 87.300000 73.0 73.0 90 \n", "\n", " Threshold Value NAAQS Threshold Value TRPA \n", "0 None 80 \n", "1 None 80 \n", "2 None 80 \n", "3 None 80 \n", - "4 None 80 \n" + "4 None 80 \n", + "5 None 80 \n" ] } ], @@ -2445,14 +2463,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "\n", "# Create Excel Doc with a sheet for each of the indicators: 2.5 combined, pm10 combine, and 03 combined\n", "# Create a Pandas Excel writer using XlsxWriter as the engine.\n", - "local_path= r'C:\\Users\\snewsome\\Documents\\Threshold\\AQFireexemptionsexceedances2023.xlsx'\n", + "local_path= r'C:\\Users\\snewsome\\Documents\\Threshold\\AQFireexemptionsexceedances18-2023.xlsx'\n", "with ExcelWriter(local_path) as writer:\n", " # Write each dataframe to a different worksheet.\n", " PM10_combined.to_excel(writer, sheet_name='PM10', index=False)\n", From a50e0ac48648f45b84245b3a86de3177375b1e04 Mon Sep 17 00:00:00 2001 From: TRPA-Sarah Date: Fri, 17 Jan 2025 09:02:28 -0800 Subject: [PATCH 2/2] AQ fire analysis with 2018 data --- ...sholdEvaluation_DataEngineering_2023.ipynb | 202 ++---------------- 1 file changed, 12 insertions(+), 190 deletions(-) diff --git a/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb b/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb index 722d9d4..4f79e7c 100644 --- a/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb +++ b/Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb @@ -1906,22 +1906,9 @@ }, { "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "C:\\Users\\snewsome\\AppData\\Local\\Temp\\ipykernel_24192\\3479171100.py:120: SettingWithCopyWarning: \n", - "A value is trying to be set on a copy of a slice from a DataFrame.\n", - "Try using .loc[row_indexer,col_indexer] = value instead\n", - "\n", - "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", - " DailyExemptdf['Year'] = DailyExemptdf['date'].dt.year\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "#-----------------------#\n", "#Import and Format Data from DRI\n", @@ -2050,56 +2037,9 @@ }, { "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "With Fire Exemption Dates:\n", - " Year Max_PM10 Threshold Value CA Threshold Value NAAQS\n", - "0 2018 143.0 50 150\n", - "1 2019 43.0 50 150\n", - "2 2020 55.0 50 150\n", - "3 2021 46.0 50 150\n", - "4 2022 45.0 50 150\n", - "5 2023 39.0 50 150\n", - "All Dates:\n", - " Year Max_PM10 Threshold Value CA Threshold Value NAAQS\n", - "0 2018 143.0 50 150\n", - "1 2019 43.0 50 150\n", - "2 2020 201.0 50 150\n", - "3 2021 492.0 50 150\n", - "4 2022 136.0 50 150\n", - "5 2023 39.0 50 150\n", - "Combined PM10 Data:\n", - " Year Exceedances_CA_exempt Exceedances_CA_all Exceedances_NAAQS_exempt \\\n", - "0 2018 8 8 0 \n", - "1 2019 0 0 0 \n", - "2 2020 1 20 0 \n", - "3 2021 0 15 0 \n", - "4 2022 0 2 0 \n", - "5 2023 0 0 0 \n", - "\n", - " Exceedances_NAAQS_all Exceedance_Threshold \\\n", - "0 0 1 exceedance per year on average over 3 years \n", - "1 0 1 exceedance per year on average over 3 years \n", - "2 3 1 exceedance per year on average over 3 years \n", - "3 5 1 exceedance per year on average over 3 years \n", - "4 0 1 exceedance per year on average over 3 years \n", - "5 0 1 exceedance per year on average over 3 years \n", - "\n", - " Max_PM10_exempt Max_PM10_all Threshold Value CA Threshold Value NAAQS \n", - "0 143.0 143.0 50 150 \n", - "1 43.0 43.0 50 150 \n", - "2 55.0 201.0 50 150 \n", - "3 46.0 492.0 50 150 \n", - "4 45.0 136.0 50 150 \n", - "5 39.0 39.0 50 150 \n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "\n", "##### PM 10 ###########\n", @@ -2171,56 +2111,9 @@ }, { "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Combined pm2.5 Data:\n", - " Year Max_PM2.5_exempt Max_PM2.5_all 3_Year_Mean_98th_Percentile_exempt \\\n", - "0 2018 72.2 72.2 NaN \n", - "1 2019 44.4 44.4 NaN \n", - "2 2020 30.7 124.7 19.604000 \n", - "3 2021 37.5 421.9 15.558667 \n", - "4 2022 25.0 296.3 17.494000 \n", - "5 2023 19.1 19.1 14.696667 \n", - "\n", - " 3_Year_Mean_98th_Percentile_all 98th_Percentile_PM2.5_exempt \\\n", - "0 NaN 32.804 \n", - "1 NaN 5.144 \n", - "2 34.469333 20.864 \n", - "3 63.008000 20.668 \n", - "4 71.474667 10.950 \n", - "5 53.812000 12.472 \n", - "\n", - " 98th_Percentile_PM2.5_all Exceedances_NAAQS_exempt Exceedances_NAAQS_all \\\n", - "0 32.804 7 7 \n", - "1 5.144 1 1 \n", - "2 65.460 0 20 \n", - "3 118.420 1 23 \n", - "4 30.544 0 7 \n", - "5 12.472 0 0 \n", - "\n", - " 98Exceedances_NAAQS_exempt 98Exceedances_NAAQS_all Threshold Value CA \\\n", - "0 0 0 None \n", - "1 0 0 None \n", - "2 0 1 None \n", - "3 0 1 None \n", - "4 0 0 None \n", - "5 0 0 None \n", - "\n", - " Threshold Value NAAQS \n", - "0 35 \n", - "1 35 \n", - "2 35 \n", - "3 35 \n", - "4 35 \n", - "5 35 \n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "\n", "######### PM 2.5 ###########\n", @@ -2322,80 +2215,9 @@ }, { "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "With Fire Exemption Dates:\n", - " Year TC O3max STL O3max IV O3max O3Threshold Value CA \\\n", - "0 2018 90.0 71.000000 86.0 90 \n", - "1 2019 72.0 67.081518 67.0 90 \n", - "2 2020 78.0 73.333333 75.0 90 \n", - "3 2021 78.0 63.100000 88.0 90 \n", - "4 2022 86.0 75.853659 78.0 90 \n", - "5 2023 71.0 87.300000 73.0 90 \n", - "\n", - " O3Threshold Value NAAQS Threshold Value TRPA \n", - "0 None 80 \n", - "1 None 80 \n", - "2 None 80 \n", - "3 None 80 \n", - "4 None 80 \n", - "5 None 80 \n", - "All Dates:\n", - " Year TC O3max STL O3max IV O3max Threshold Value CA \\\n", - "0 2018 90.0 71.000000 86.0 90 \n", - "1 2019 72.0 67.081518 67.0 90 \n", - "2 2020 98.0 85.333333 96.0 90 \n", - "3 2021 98.0 81.200000 93.0 90 \n", - "4 2022 86.0 75.853659 80.0 90 \n", - "5 2023 71.0 87.300000 73.0 90 \n", - "\n", - " Threshold Value NAAQS Threshold Value TRPA \n", - "0 None 80 \n", - "1 None 80 \n", - "2 None 80 \n", - "3 None 80 \n", - "4 None 80 \n", - "5 None 80 \n", - "Combined O3 Data:\n", - " Year Exceedances_TRPA_all Exceedances_TRPA_exempt Exceedances_CA_all \\\n", - "0 2018 7 7 0 \n", - "1 2019 0 0 0 \n", - "2 2020 9 0 3 \n", - "3 2021 13 1 4 \n", - "4 2022 1 1 0 \n", - "5 2023 1 1 0 \n", - "\n", - " Exceedances_CA_exempt TC O3max_exempt TC O3max_all STL O3max_exempt \\\n", - "0 0 90.0 90.0 71.000000 \n", - "1 0 72.0 72.0 67.081518 \n", - "2 0 78.0 98.0 73.333333 \n", - "3 0 78.0 98.0 63.100000 \n", - "4 0 86.0 86.0 75.853659 \n", - "5 0 71.0 71.0 87.300000 \n", - "\n", - " STL O3max_all IV O3max_all IV O3max_exempt Threshold Value CA \\\n", - "0 71.000000 86.0 86.0 90 \n", - "1 67.081518 67.0 67.0 90 \n", - "2 85.333333 96.0 75.0 90 \n", - "3 81.200000 93.0 88.0 90 \n", - "4 75.853659 80.0 78.0 90 \n", - "5 87.300000 73.0 73.0 90 \n", - "\n", - " Threshold Value NAAQS Threshold Value TRPA \n", - "0 None 80 \n", - "1 None 80 \n", - "2 None 80 \n", - "3 None 80 \n", - "4 None 80 \n", - "5 None 80 \n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "\n", "#######O3 -1HR AVG######\n",