Skip to content

Commit

Permalink
Merge pull request #214 from trpa-agency/Sarah-edits
Browse files Browse the repository at this point in the history 
Sarah edits
  • Loading branch information
@TRPA-Sarah
TRPA-Sarah authored Jan 17, 2025
2 parents a5ff5e4 + a50e0ac commit d36823c
Showing 1 changed file with 19 additions and 179 deletions.
198 changes: 19 additions & 179 deletions Scripts/2023/ThresholdEvaluation_DataEngineering_2023.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1906,22 +1906,9 @@
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"C:\\Users\\snewsome\\AppData\\Local\\Temp\\ipykernel_24192\\322407640.py:120: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" DailyExemptdf['Year'] = DailyExemptdf['date'].dt.year\n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#-----------------------#\n",
"#Import and Format Data from DRI\n",
Expand All @@ -1935,8 +1922,8 @@
" r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ data 2021.xlsx\",\n",
" r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ data 2020.xlsx\",\n",
" r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ data 2019.xlsx\",\n",
" #r\"F:\\\\Research and Analysis\\\\Air Quality\\\\Annual Reports DRI\\\\AQ Data 2013-2018 Daily.xlsx\"]\n",
"]\n",
" r\"F:\\Research and Analysis\\Air Quality\\Annual Reports DRI\\Daily AQ data 2018 from AQ'd data 2018.xlsx\"]\n",
"\n",
"# 2. Read Data from Each File Daily Data\n",
"dfs = [] # List to store DataFrames from each file\n",
"sheet_name = 'daily' # Name of the sheet to read\n",
Expand Down Expand Up @@ -1970,7 +1957,7 @@
"DailyAir_df['STL O3max'] = pd.to_numeric(DailyAir_df['STL O3max'], errors='coerce')\n",
"DailyAir_df['IV O3max'] = pd.to_numeric(DailyAir_df['IV O3max'], errors='coerce')\n",
"DailyAir_df['BSP O3max'] = pd.to_numeric(DailyAir_df['BSP O3max'], errors='coerce')\n",
"#DailyAir_df['LTC O3max'] = pd.to_numeric(DailyAir_df['LTC O3max'], errors='coerce')\n",
"DailyAir_df['LTC O3max'] = pd.to_numeric(DailyAir_df['LTC O3max'], errors='coerce')\n",
"\n",
"#keep data from 2018-2023 --- Excel Table used has 2013-2018 data, before 2018 soo 2017 and earlier the data for STL PM2.5avg is actually PM10avg\n",
"#DailyAir_df = DailyAir_df.loc[DailyAir_df['date'].dt.year >= 2018]\n",
Expand Down Expand Up @@ -2050,51 +2037,9 @@
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"With Fire Exemption Dates:\n",
" Year Max_PM10 Threshold Value CA Threshold Value NAAQS\n",
"0 2019 43.0 50 150\n",
"1 2020 55.0 50 150\n",
"2 2021 46.0 50 150\n",
"3 2022 45.0 50 150\n",
"4 2023 39.0 50 150\n",
"All Dates:\n",
" Year Max_PM10 Threshold Value CA Threshold Value NAAQS\n",
"0 2019 43.0 50 150\n",
"1 2020 201.0 50 150\n",
"2 2021 492.0 50 150\n",
"3 2022 136.0 50 150\n",
"4 2023 39.0 50 150\n",
"Combined PM10 Data:\n",
" Year Exceedances_CA_exempt Exceedances_CA_all Exceedances_NAAQS_exempt \\\n",
"0 2019 0 0 0 \n",
"1 2020 1 20 0 \n",
"2 2021 0 15 0 \n",
"3 2022 0 2 0 \n",
"4 2023 0 0 0 \n",
"\n",
" Exceedances_NAAQS_all Exceedance_Threshold \\\n",
"0 0 1 exceedance per year on average over 3 years \n",
"1 3 1 exceedance per year on average over 3 years \n",
"2 5 1 exceedance per year on average over 3 years \n",
"3 0 1 exceedance per year on average over 3 years \n",
"4 0 1 exceedance per year on average over 3 years \n",
"\n",
" Max_PM10_exempt Max_PM10_all Threshold Value CA Threshold Value NAAQS \n",
"0 43.0 43.0 50 150 \n",
"1 55.0 201.0 50 150 \n",
"2 46.0 492.0 50 150 \n",
"3 45.0 136.0 50 150 \n",
"4 39.0 39.0 50 150 \n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"##### PM 10 ###########\n",
Expand Down Expand Up @@ -2166,61 +2111,19 @@
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Combined pm2.5 Data:\n",
" Year Max_PM2.5_exempt Max_PM2.5_all 3_Year_Mean_98th_Percentile_exempt \\\n",
"0 2019 44.4 44.4 NaN \n",
"1 2020 30.7 124.7 NaN \n",
"2 2021 37.5 421.9 15.558667 \n",
"3 2022 25.0 296.3 17.494000 \n",
"4 2023 19.1 19.1 14.696667 \n",
"\n",
" 3_Year_Mean_98th_Percentile_all 98th_Percentile_PM2.5_exempt \\\n",
"0 NaN 5.144 \n",
"1 NaN 20.864 \n",
"2 63.008000 20.668 \n",
"3 71.474667 10.950 \n",
"4 53.812000 12.472 \n",
"\n",
" 98th_Percentile_PM2.5_all Exceedances_NAAQS_exempt Exceedances_NAAQS_all \\\n",
"0 5.144 1 1 \n",
"1 65.460 0 20 \n",
"2 118.420 1 23 \n",
"3 30.544 0 7 \n",
"4 12.472 0 0 \n",
"\n",
" 98Exceedances_NAAQS_exempt 98Exceedances_NAAQS_all Threshold Value CA \\\n",
"0 0 0 None \n",
"1 0 1 None \n",
"2 0 1 None \n",
"3 0 0 None \n",
"4 0 0 None \n",
"\n",
" Threshold Value NAAQS \n",
"0 35 \n",
"1 35 \n",
"2 35 \n",
"3 35 \n",
"4 35 \n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"######### PM 2.5 ###########\n",
"#----------------------------------------------\n",
"#CALCULATE 3 year average 98th percentil \n",
"#----------------------------------------------\n",
"# Calculate the 98th percentile for each year\n",
"# Calculate the 98th percentile for each year for both TC PM2.5avg and STL PM2.5avg\n",
"PM25_98th_percentile_exempt = DailyExemptdf.groupby('Year').agg({'TC PM2.5avg': lambda x: x.quantile(0.98)}).reset_index()\n",
"PM25_98th_percentile_all = DailyAir_df.groupby('Year').agg({'TC PM2.5avg': lambda x: x.quantile(0.98)}).reset_index()\n",
"\n",
"\n",
"# Rename columns for clarity\n",
"PM25_98th_percentile_exempt.rename(columns={'TC PM2.5avg': '98th_Percentile_PM2.5'}, inplace=True)\n",
"PM25_98th_percentile_all.rename(columns={'TC PM2.5avg': '98th_Percentile_PM2.5'}, inplace=True)\n",
Expand Down Expand Up @@ -2312,72 +2215,9 @@
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"With Fire Exemption Dates:\n",
" Year TC O3max STL O3max IV O3max O3Threshold Value CA \\\n",
"0 2019 72.0 67.081518 67.0 90 \n",
"1 2020 78.0 73.333333 75.0 90 \n",
"2 2021 78.0 63.100000 88.0 90 \n",
"3 2022 86.0 75.853659 78.0 90 \n",
"4 2023 71.0 87.300000 73.0 90 \n",
"\n",
" O3Threshold Value NAAQS Threshold Value TRPA \n",
"0 None 80 \n",
"1 None 80 \n",
"2 None 80 \n",
"3 None 80 \n",
"4 None 80 \n",
"All Dates:\n",
" Year TC O3max STL O3max IV O3max Threshold Value CA \\\n",
"0 2019 72.0 67.081518 67.0 90 \n",
"1 2020 98.0 85.333333 96.0 90 \n",
"2 2021 98.0 81.200000 93.0 90 \n",
"3 2022 86.0 75.853659 80.0 90 \n",
"4 2023 71.0 87.300000 73.0 90 \n",
"\n",
" Threshold Value NAAQS Threshold Value TRPA \n",
"0 None 80 \n",
"1 None 80 \n",
"2 None 80 \n",
"3 None 80 \n",
"4 None 80 \n",
"Combined O3 Data:\n",
" Year Exceedances_TRPA_all Exceedances_TRPA_exempt Exceedances_CA_all \\\n",
"0 2019 0 0 0 \n",
"1 2020 9 0 3 \n",
"2 2021 13 1 4 \n",
"3 2022 1 1 0 \n",
"4 2023 1 1 0 \n",
"\n",
" Exceedances_CA_exempt TC O3max_exempt TC O3max_all STL O3max_exempt \\\n",
"0 0 72.0 72.0 67.081518 \n",
"1 0 78.0 98.0 73.333333 \n",
"2 0 78.0 98.0 63.100000 \n",
"3 0 86.0 86.0 75.853659 \n",
"4 0 71.0 71.0 87.300000 \n",
"\n",
" STL O3max_all IV O3max_all IV O3max_exempt Threshold Value CA \\\n",
"0 67.081518 67.0 67.0 90 \n",
"1 85.333333 96.0 75.0 90 \n",
"2 81.200000 93.0 88.0 90 \n",
"3 75.853659 80.0 78.0 90 \n",
"4 87.300000 73.0 73.0 90 \n",
"\n",
" Threshold Value NAAQS Threshold Value TRPA \n",
"0 None 80 \n",
"1 None 80 \n",
"2 None 80 \n",
"3 None 80 \n",
"4 None 80 \n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"#######O3 -1HR AVG######\n",
Expand Down Expand Up @@ -2445,14 +2285,14 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"\n",
"# Create Excel Doc with a sheet for each of the indicators: 2.5 combined, pm10 combine, and 03 combined\n",
"# Create a Pandas Excel writer using XlsxWriter as the engine.\n",
"local_path= r'C:\\Users\\snewsome\\Documents\\Threshold\\AQFireexemptionsexceedances2023.xlsx'\n",
"local_path= r'C:\\Users\\snewsome\\Documents\\Threshold\\AQFireexemptionsexceedances18-2023.xlsx'\n",
"with ExcelWriter(local_path) as writer:\n",
" # Write each dataframe to a different worksheet.\n",
" PM10_combined.to_excel(writer, sheet_name='PM10', index=False)\n",
Expand Down

0 comments on commit d36823c

Please sign in to comment.