Update data import script to handle update logic

api/management/commands/ingest_icrc.py

@@ -60,19 +60,19 @@ def handle(self, *args, **kwargs):
                         "Description": description,
                     }
                 )
+
         added = 0
         for data in country_list:
-            country = Country.objects.filter(name__exact=data["Country"])
-            if country.exists():
-                dict_data = {
-                    "country": country.first(),
-                    "icrc_presence": data["ICRC presence"],
-                    "url": data["URL"],
-                    "key_operation": data["Key operation"],
-                    "description": data["Description"],
-                }
+            country = Country.objects.filter(name__exact=data["Country"]).first()
+            if country:
+                country_icrc_presence, _ = CountryICRCPresence.objects.get_or_create(country=country)
+
+                country_icrc_presence.icrc_presence = data["ICRC presence"]
+                country_icrc_presence.url = data["URL"]
+                country_icrc_presence.key_operation = data["Key operation"]
+                country_icrc_presence.description = data["Description"]
+                country_icrc_presence.save()
                 added += 1
-                CountryICRCPresence.objects.create(**dict_data)
 
         text_to_log = "%s ICRC added" % added
         logger.info(text_to_log)

api/management/commands/ingest_ns_directory.py

@@ -57,7 +57,14 @@ def handle(self, *args, **kwargs):
                         "position": data["CON_title"],
                         "country": country,
                     }
-                    CountryDirectory.objects.create(**data)
+                    existing_qs = CountryDirectory.objects.filter(
+                        country=country,
+                        first_name__iexact=data["first_name"],
+                        last_name__iexact=data["last_name"],
+                        position__iexact=data["position"],
+                    )
+                    if not existing_qs.exists():
+                        CountryDirectory.objects.create(**data)
         text_to_log = "%s Ns Directory added" % added
         logger.info(text_to_log)
         body = {"name": "ingest_ns_directory", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL}

api/management/commands/ingest_ns_document.py

@@ -100,19 +100,21 @@ def save_documents_to_database(self, result):
         added = 0
         for document in result:
             country = Country.objects.filter(fdrs=document["country_code"]).first()
-            if country:
-                added += 1
-                data = {
-                    "country": country,
-                    "name": document["name"],
-                    "url": document["url"],
-                    "thumbnail": document["thumbnail"],
-                    "document_type": document["document_type"],
-                    "year": document["year"],
-                    "end_year": document["end_year"],
-                    "year_text": document["year_text"],
-                }
-                CountryKeyDocument.objects.create(**data)
+            if country is None:
+                continue
+
+            country_key_document, _ = CountryKeyDocument.objects.get_or_create(
+                country=country,
+                url=document["url"],
+            )
+            country_key_document.name = document["name"]
+            country_key_document.thumbnail = document["thumbnail"]
+            country_key_document.document_type = document["document_type"]
+            country_key_document.year = document["year"]
+            country_key_document.end_year = document["end_year"]
+            country_key_document.year_text = document["year_text"]
+            country_key_document.save()
+            added += 1
         return added
 
     def sync_cron_success(self, text_to_log, added):

api/management/commands/ingest_ns_initiatives.py

@@ -47,17 +47,18 @@ def handle(self, *args, **kwargs):
             # TODO: Filter not by society name
             country = Country.objects.filter(society_name__iexact=data[0]).first()
             if country:
-                dict_data = {
-                    "country": country,
-                    "title": data[3],
-                    "fund_type": data[2],
-                    "allocation": data[5],
-                    "year": data[1],
-                    "funding_period": data[6],
-                    "categories": data[4],
-                }
+                nsd_initiatives, _ = NSDInitiatives.objects.get_or_create(
+                    country=country,
+                    year=data[1],
+                    fund_type=data[2],
+                )
+                nsd_initiatives.title = data[3]
+                nsd_initiatives.allocation = data[5]
+                nsd_initiatives.funding_period = data[6]
+                nsd_initiatives.categories = data[4]
+                nsd_initiatives.save()
                 added += 1
-                NSDInitiatives.objects.create(**dict_data)
+
         text_to_log = "%s Ns initiatives added" % added
         logger.info(text_to_log)
         body = {"name": "ingest_ns_initiatives", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL}

api/models.py

@@ -349,6 +349,7 @@ class CountryKeyDocument(models.Model):
     end_year = models.DateField(verbose_name=_("End Year"), null=True, blank=True)
     year_text = models.CharField(verbose_name=_("Year Text"), max_length=255, null=True, blank=True)
 
+    # TODO: Add unique_together country, url
     def __str__(self):
         return f"{self.country.name} - {self.name}"
 
@@ -391,6 +392,7 @@ class NSDInitiatives(models.Model):
     funding_period = models.IntegerField(verbose_name=_("Funding Period in Month"))
     categories = ArrayField(models.CharField(max_length=255), verbose_name=_("Funding categories"), default=list, null=True)
 
+    # TODO: Add unique_together country, year, fund_type
     def __str__(self):
         return f"{self.country.name} - {self.title}"
 

databank/management/commands/FDRS_INCOME.py

@@ -47,13 +47,16 @@ def handle(self, *args, **kwargs):
             fdrs_entities = fdrs_entities.json()
             for d in fdrs_entities["data"]:
                 indicator = next(iter(d.values()))
+                fdrs_indicator = map_indicators[fdrs_indicator_enum_data[indicator]]
                 income_list = d["data"][0]["data"]
                 if len(income_list):
                     for income in income_list:
-                        data = {
-                            "date": str(income["year"]) + "-01-01",
-                            "value": income["value"],
-                            "indicator": map_indicators.get(fdrs_indicator_enum_data.get(indicator)),
-                            "overview": overview,
-                        }
-                        FDRSIncome.objects.create(**data)
+                        income_value = income["value"]
+                        fdrs_income, _ = FDRSIncome.objects.get_or_create(
+                            overview=overview,
+                            indicator=fdrs_indicator,
+                            date=str(income["year"]) + "-01-01",
+                        )
+                        fdrs_income.value = income_value
+                        # TODO: Use bulk
+                        fdrs_income.save(update_fields=("value",))

databank/management/commands/fdrs_annual_income.py

@@ -3,6 +3,7 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import models
 from sentry_sdk.crons import monitor
 
 from databank.models import CountryOverview, FDRSAnnualIncome
@@ -16,20 +17,41 @@ class Command(BaseCommand):
     help = "Import FDRS income data"
 
     def handle(self, *args, **kwargs):
-        for overview in CountryOverview.objects.all():
-            country_fdrs_code = overview.country.fdrs
-            fdrs_api = f"https://data-api.ifrc.org/api/data?indicator=KPI_IncomeLC_CHF&KPI_Don_Code={country_fdrs_code}&apiKey={settings.FDRS_APIKEY}"  # noqa: E501
-            fdrs_entities = requests.get(fdrs_api)
+        overview_qs = CountryOverview.objects.annotate(
+            country_fdrd=models.F("country__fdrs"),
+        )
+        fdrs_data_count = 0
+        for overview in overview_qs.all():
+            country_fdrs_code = overview.country_fdrd
+            fdrs_entities = requests.get(
+                "https://data-api.ifrc.org/api/data",
+                params={
+                    "apiKey": settings.FDRS_APIKEY,
+                    "indicator": "KPI_IncomeLC_CHF",
+                    "KPI_Don_Code": country_fdrs_code,
+                },
+            )
             if fdrs_entities.status_code != 200:
-                return
+                continue
+
             fdrs_entities.raise_for_status()
             fdrs_entities = fdrs_entities.json()
-            fdrs_data_count = 0
-            if len(fdrs_entities["data"]):
-                income_list = fdrs_entities["data"][0]["data"][0]["data"]
-                if len(income_list):
-                    for income in income_list:
-                        data = {"date": str(income["year"]) + "-01-01", "value": income["value"], "overview": overview}
-                        fdrs_data_count += 1
-                        FDRSAnnualIncome.objects.get_or_create(**data)
+
+            if len(fdrs_entities["data"]) == 0:
+                continue
+
+            income_list = fdrs_entities["data"][0]["data"][0]["data"]
+            if len(income_list) == 0:
+                continue
+
+            for income in income_list:
+                income_value = income["value"]
+                fdrs_annual_income, _ = FDRSAnnualIncome.objects.get_or_create(
+                    overview=overview,
+                    date=str(income["year"]) + "-01-01",
+                )
+                fdrs_annual_income.value = income_value
+                fdrs_annual_income.save(update_fields=("value",))
+                fdrs_data_count += 1
+
         logger.info(f"Successfully created {fdrs_data_count} country data")

databank/management/commands/ingest_acaps.py

@@ -4,6 +4,7 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import models, transaction
 from sentry_sdk.crons import monitor
 
 from api.logger import logger
@@ -16,31 +17,44 @@
 class Command(BaseCommand):
     help = "Add Acaps seasonal calender data"
 
+    @transaction.atomic
+    def load_country(self, overview):
+        # Remove all existing Seasonal Calendar data for this country
+        AcapsSeasonalCalender.objects.filter(overview=overview).all().delete()
+
+        name = overview.country_name
+        if "," in name:
+            name = name.split(",")[0]
+        response = requests.get(
+            "https://api.acaps.org/api/v1/seasonal-events-calendar/seasonal-calendar/",
+            params={"country": name},
+            headers={"Authorization": "Token %s" % settings.ACAPS_API_TOKEN},
+        )
+        logger.info(f"Importing for country {name}")
+        response_data = response.json()
+        if "results" in response_data and len(response_data["results"]):
+            df = pd.DataFrame.from_records(response_data["results"])
+            for df_data in df.values.tolist():
+                df_country = df_data[2]
+                if name.lower() == df_country[0].lower():
+                    dict_data = {
+                        "overview": overview,
+                        "month": df_data[6],
+                        "event": df_data[7],
+                        "event_type": df_data[8],
+                        "label": df_data[9],
+                        "source": df_data[11],
+                        "source_date": df_data[12],
+                    }
+                    # Use bulk manager
+                    AcapsSeasonalCalender.objects.create(**dict_data)
+        # NOTE: Acaps throttles our requests
+        time.sleep(5)
+
     def handle(self, *args, **kwargs):
         logger.info("Importing Acaps Data")
-        country_name = CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY).values_list(
-            "country__name", flat=True
+        country_overview_qs = CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY).annotate(
+            country_name=models.F("country__name"),
         )
-        for name in country_name:
-            if "," in name:
-                name = name.split(",")[0]
-            SEASONAL_EVENTS_API = f"https://api.acaps.org/api/v1/seasonal-events-calendar/seasonal-calendar/?country={name}"
-            response = requests.get(SEASONAL_EVENTS_API, headers={"Authorization": "Token %s" % settings.ACAPS_API_TOKEN})
-            logger.info(f"Importing for country {name}")
-            response_data = response.json()
-            if "results" in response_data and len(response_data["results"]):
-                df = pd.DataFrame.from_records(response_data["results"])
-                for df_data in df.values.tolist():
-                    df_country = df_data[2]
-                    if name.lower() == df_country[0].lower():
-                        dict_data = {
-                            "overview": CountryOverview.objects.filter(country__name__icontains=name).first(),
-                            "month": df_data[6],
-                            "event": df_data[7],
-                            "event_type": df_data[8],
-                            "label": df_data[9],
-                            "source": df_data[11],
-                            "source_date": df_data[12],
-                        }
-                        AcapsSeasonalCalender.objects.create(**dict_data)
-            time.sleep(5)
+        for overview in country_overview_qs:
+            self.load_country(overview)