From 09683e98adf743226e8cc142894f083754c9c2fe Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Tue, 2 Jul 2024 14:38:01 +0545 Subject: [PATCH] Fix sentry error on cronjobs add header for icrc fix None error on climate data --- api/management/commands/ingest_icrc.py | 8 ++++- .../commands/ingest_ns_directory.py | 7 +++- api/management/commands/ingest_ns_document.py | 25 ++++++++----- .../commands/ingest_ns_initiatives.py | 19 ++++++---- .../management/commands/ingest_climate.py | 35 ++++++++++++------- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index 0368b44194..bf3f47663d 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -14,7 +14,13 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): logger.info("Strating ICRC data ingest") - response = requests.get(url="https://www.icrc.org/en/where-we-work", headers={"User-Agent": ""}) + HEADERS = { + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36", # noqa + } + response = requests.get( + url="https://www.icrc.org/en/where-we-work", + headers=HEADERS, + ) if response.status_code != 200: text_to_log = "Error querying ICRC feed at https://www.icrc.org/en/where-we-work" logger.error(text_to_log) diff --git a/api/management/commands/ingest_ns_directory.py b/api/management/commands/ingest_ns_directory.py index 0fb3a1cd71..0a5de0bbe8 100644 --- a/api/management/commands/ingest_ns_directory.py +++ b/api/management/commands/ingest_ns_directory.py @@ -15,6 +15,11 @@ class Command(BaseCommand): help = "Add ns contact details" def handle(self, *args, **kwargs): + def postprocessor(path, key, value): + if key == "@i:nil": + return None + return key, value + logger.info("Starting NS Contacts") url = "https://go-api.ifrc.org/" headers = {"accept": "application/xml;q=0.9, */*;q=0.8"} @@ -36,7 +41,7 @@ def handle(self, *args, **kwargs): raise Exception("Error querying NationalSocietiesContacts") added = 0 - dict_data = xmltodict.parse(response.content) + dict_data = xmltodict.parse(response.content, postprocessor=postprocessor) for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]: country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None if country_name is not None: diff --git a/api/management/commands/ingest_ns_document.py b/api/management/commands/ingest_ns_document.py index 92f5fb3c39..fca3d32809 100644 --- a/api/management/commands/ingest_ns_document.py +++ b/api/management/commands/ingest_ns_document.py @@ -103,17 +103,26 @@ def save_documents_to_database(self, result): if country is None: continue - country_key_document, _ = CountryKeyDocument.objects.get_or_create( + country_key_document, created = CountryKeyDocument.objects.get_or_create( country=country, url=document["url"], + defaults={ + "name": document["name"], + "thumbnail": document["thumbnail"], + "document_type": document["document_type"], + "year": document["year"], + "end_year": document["end_year"], + "year_text": document["year_text"], + }, ) - country_key_document.name = document["name"] - country_key_document.thumbnail = document["thumbnail"] - country_key_document.document_type = document["document_type"] - country_key_document.year = document["year"] - country_key_document.end_year = document["end_year"] - country_key_document.year_text = document["year_text"] - country_key_document.save() + if not created: + country_key_document.name = document["name"] + country_key_document.thumbnail = document["thumbnail"] + country_key_document.document_type = document["document_type"] + country_key_document.year = document["year"] + country_key_document.end_year = document["end_year"] + country_key_document.year_text = document["year_text"] + country_key_document.save(update_fields=["name", "thumbnail", "document_type", "year", "end_year", "year_text"]) added += 1 return added diff --git a/api/management/commands/ingest_ns_initiatives.py b/api/management/commands/ingest_ns_initiatives.py index 40227e3e34..47dd005232 100644 --- a/api/management/commands/ingest_ns_initiatives.py +++ b/api/management/commands/ingest_ns_initiatives.py @@ -47,16 +47,23 @@ def handle(self, *args, **kwargs): # TODO: Filter not by society name country = Country.objects.filter(society_name__iexact=data[0]).first() if country: - nsd_initiatives, _ = NSDInitiatives.objects.get_or_create( + nsd_initiatives, created = NSDInitiatives.objects.get_or_create( country=country, year=data[1], fund_type=data[2], + defaults={ + "title": data[3], + "categories": data[4], + "allocation": data[5], + "funding_period": data[6], + }, ) - nsd_initiatives.title = data[3] - nsd_initiatives.allocation = data[5] - nsd_initiatives.funding_period = data[6] - nsd_initiatives.categories = data[4] - nsd_initiatives.save() + if not created: + nsd_initiatives.title = data[3] + nsd_initiatives.allocation = data[5] + nsd_initiatives.funding_period = data[6] + nsd_initiatives.categories = data[4] + nsd_initiatives.save(update_fields=["title", "allocation", "funding_period", "categories"]) added += 1 text_to_log = "%s Ns initiatives added" % added diff --git a/databank/management/commands/ingest_climate.py b/databank/management/commands/ingest_climate.py index 3fd13cd010..bf99e844ea 100644 --- a/databank/management/commands/ingest_climate.py +++ b/databank/management/commands/ingest_climate.py @@ -59,25 +59,34 @@ def handle(self, *args, **options): for value in merged_data.values(): for k, v in value.items(): + if v[0] is None or v[1] is None or v[2] is None or v[3] is None: + continue year_month = k.split("-") - country_key_climate, _ = CountryKeyClimate.objects.get_or_create( + country_key_climate, created = CountryKeyClimate.objects.get_or_create( overview=overview, year=year_month[0], month=year_month[1], + defaults={ + "precipitation": v[0], + "avg_temp": v[1], + "min_temp": v[2], + "max_temp": v[3], + }, ) - country_key_climate.max_temp = v[3] - country_key_climate.min_temp = v[2] - country_key_climate.avg_temp = v[1] - country_key_climate.precipitation = v[0] - # TODO: Use bulk manager - country_key_climate.save( - update_fields=( - "max_temp", - "min_temp", - "avg_temp", - "precipitation", + if not created: + country_key_climate.max_temp = v[3] + country_key_climate.min_temp = v[2] + country_key_climate.avg_temp = v[1] + country_key_climate.precipitation = v[0] + # TODO: Use bulk manager + country_key_climate.save( + update_fields=( + "max_temp", + "min_temp", + "avg_temp", + "precipitation", + ) ) - ) except Exception: logger.error("Error in ingesting climate data", exc_info=True) continue