Project/phase 2

.gitignore

@@ -32,6 +32,7 @@ MANIFEST
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
+.aider*
 
 # Installer logs
 pip-log.txt

.pre-commit-config.yaml

@@ -31,6 +31,11 @@ repos:
       - id: flake8
 
   - repo: https://github.com/RobertCraigie/pyright-python
-    rev: v1.1.364
+    rev: v1.1.390
     hooks:
       - id: pyright
+
+  # - repo: https://github.com/gruntwork-io/pre-commit
+  #   rev: v0.1.15
+  #   hooks:
+  #     - id: helmlint

Dockerfile

@@ -22,12 +22,13 @@ RUN apt-get update -y \
     && poetry --version \
     # Configure to use system instead of virtualenvs
     && poetry config virtualenvs.create false \
-    && poetry install --no-root \
+    && poetry install --no-root --no-cache --no-interaction \
     # Clean-up
     && pip uninstall -y poetry virtualenv-clone virtualenv \
     && apt-get remove -y gcc libc-dev libproj-dev \
     && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /root/.cache/
 
 
 COPY . /code/

README.md

@@ -1,6 +1,6 @@
 # IFRC/UCL Alert Hub - CAP Aggregator
 
-The CAP Aggregator is an alert aggregation service built for IFRC's Alert Hub. Public alerts use the Common Alerting Protocol (CAP) Version 1.2 standard.
+This repository houses the CAP Aggregator and Alert Manager for IFRC's Alert Hub. These services aggregate and distribute public alerts using the Common Alerting Protocol (CAP) Version 1.2 standard.
 
 This is a Python web app using the Django framework and the Azure Database for PostgreSQL relational database service. The Django app is hosted in a fully managed Azure App Service. Requests to hundreds of publicly available alert feeds are managed by Celery and Redis, which interprets alerts and saves them to the database. The Alert Manager then makes them available to the Alert Hub Website.
 

apps/cap_feed/admin.py

@@ -153,6 +153,7 @@ class FeedAdmin(admin.ModelAdmin):
     list_filter = (
         'format',
         'country__region',
+        'enable_polling',
         AutocompleteFilterFactory('Country', 'country'),
     )
     search_fields = ['url']

apps/cap_feed/data_injector/geo.py

@@ -350,6 +350,46 @@ def get_geojson_data():
         mgr.done()
         self.log_success(str(mgr.summary()))
 
+    def inject_admin1s_geo_codes(self):
+        # TODO: Confirm this
+        fetch_params = {
+            "is_independent": True,
+            "is_deprecated": False,
+            "limit": 1000,
+        }
+
+        go_data = self.handle_pagination(
+            "/api/v2/district/",
+            params=fetch_params,
+            headers={"Accept-Language": "EN"},
+        )
+
+        mgr = BulkUpdateManager(
+            update_fields=[
+                "emma_id",
+                "nuts1",
+                "nuts2",
+                "nuts3",
+                "fips_code",
+            ]
+        )
+
+        for admin1_data in go_data:
+            ifrc_go_id = int(admin1_data.get("id"))
+            admin1 = Admin1.objects.filter(ifrc_go_id=ifrc_go_id).first()
+            if admin1 is None:
+                continue
+
+            admin1.emma_id = admin1_data["emma_id"]
+            admin1.nuts1 = admin1_data["nuts1"]
+            admin1.nuts2 = admin1_data["nuts2"]
+            admin1.nuts3 = admin1_data["nuts3"]
+            admin1.fips_code = admin1_data["fips_code"]
+            mgr.add(admin1)
+
+        mgr.done()
+        self.log_success(str(mgr.summary()))
+
     @transaction.atomic
     def sync(
         self,
@@ -362,4 +402,5 @@ def sync(
         self.inject_countries()
         if not skip_admin1s_sync:
             self.inject_admin1s()
+            self.inject_admin1s_geo_codes()
         # TODO: Show change summary

apps/cap_feed/dataloaders.py

@@ -63,7 +63,23 @@ def load_feed(keys: list[int]) -> list['FeedType']:
     return _load_model(Feed, keys)  # type: ignore[reportGeneralTypeIssues]
 
 
-def load_admin1_by_alert(keys: list[int]) -> list[list['Admin1Type']]:
+def load_admin1_by_admin1s(keys_array: list[tuple[int]]) -> list[list['Admin1Type']]:
+    keys = [key for keys in keys_array for key in keys]
+    qs = Admin1.objects.filter(id__in=keys)
+
+    _map = defaultdict(list)
+    admin1_map = {admin1.pk: admin1 for admin1 in qs.all()}
+
+    for keys in keys_array:
+        for key in keys:
+            if key not in admin1_map:
+                continue
+            _map[keys].append(admin1_map[key])
+
+    return [_map[keys] for keys in keys_array]
+
+
+def load_admin1s_by_alert(keys: list[int]) -> list[list['Admin1Type']]:
     qs = (
         AlertAdmin1.objects.filter(alert__in=keys)
         .order_by()
@@ -228,9 +244,13 @@ def load_continent(self):
     def load_feed(self):
         return DataLoader(load_fn=sync_to_async(load_feed))
 
+    @cached_property
+    def load_admin1_by_admin1s(self):
+        return DataLoader(load_fn=sync_to_async(load_admin1_by_admin1s))
+
     @cached_property
     def load_admin1s_by_alert(self):
-        return DataLoader(load_fn=sync_to_async(load_admin1_by_alert))
+        return DataLoader(load_fn=sync_to_async(load_admin1s_by_alert))
 
     @cached_property
     def load_admin1s_by_country(self):

apps/cap_feed/factories.py

@@ -0,0 +1,71 @@
+import datetime
+
+import factory
+from factory.django import DjangoModelFactory
+
+from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
+
+
+class RegionFactory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Region-{n}')
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Region
+
+
+class CountryFactory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Country-{n}')
+    iso3 = factory.Sequence(lambda n: f"{n:0>3}")
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Country
+
+
+class FeedFactory(DjangoModelFactory):
+    url = factory.Sequence(lambda n: f"https://source-{n}.com/test")
+    format = Feed.Format.RSS
+    polling_interval = Feed.PoolingInterval.I_10m
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Feed
+
+
+class Admin1Factory(DjangoModelFactory):
+    ifrc_go_id = factory.Sequence(lambda n: n)
+    name = factory.Sequence(lambda n: f'Admin1-{n}')
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Admin1
+
+
+class AlertFactory(DjangoModelFactory):
+    url = factory.Sequence(lambda n: f"https://alert-{n}.com/test")
+    identifier = "Identifier-X"
+    sender = "Sender-X"
+    sent = datetime.datetime(year=2024, month=1, day=1)
+    status = Alert.Status.ACTUAL
+    msg_type = Alert.MsgType.ALERT
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = Alert
+
+    @factory.post_generation
+    def admin1s(self, create, extracted, **_):
+        if not create:
+            return
+        if extracted:
+            for author in extracted:
+                self.admin1s.add(author)  # type: ignore[reportAttributeAccessIssue]
+
+
+class AlertInfoFactory(DjangoModelFactory):
+    event = "Event-X"
+    category = AlertInfo.Category.HEALTH
+    urgency = AlertInfo.Urgency.IMMEDIATE
+    severity = AlertInfo.Severity.EXTREME
+    certainty = AlertInfo.Certainty.OBSERVED
+
+    class Meta:  # type: ignore[reportIncompatibleVariableOverride]
+        model = AlertInfo

apps/cap_feed/filters.py

@@ -12,40 +12,15 @@
 from .models import Admin1, Alert, AlertInfo, Country, Feed, Region
 
 
-@strawberry_django.filters.filter(Alert, lookups=True)
-class AlertFilter:
+@strawberry_django.filters.filter(AlertInfo, lookups=True)
+class AlertInfoFilter:
     id: strawberry.auto
-    country: strawberry.auto
-    sent: strawberry.auto
-
-    @strawberry_django.filter_field
-    def region(
-        self,
-        queryset: models.QuerySet,
-        value: strawberry.ID,
-        prefix: str,
-    ) -> tuple[models.QuerySet, models.Q]:
-        return queryset, models.Q(**{f"{prefix}country__region": value})
-
-    @strawberry_django.filter_field
-    def admin1(
-        self,
-        queryset: models.QuerySet,
-        value: strawberry.ID,
-        prefix: str,
-    ) -> tuple[models.QuerySet, models.Q]:
-        return queryset, models.Q(**{f"{prefix}admin1s": value})
 
     def _info_enum_fields(self, field, queryset, value, prefix) -> tuple[models.QuerySet, models.Q]:
         if value:
-            alias_field = f"_infos_{field}_list"
-            queryset = queryset.alias(
-                **{
-                    # NOTE: To avoid duplicate alerts when joining infos
-                    alias_field: ArrayAgg(f"{prefix}infos__{field}"),
-                }
-            )
-            return queryset, models.Q(**{f"{prefix}{alias_field}__overlap": value})
+            # NOTE: With this field, disctinct should be used by the client
+            print(f"{prefix}{field}__in")
+            return queryset, models.Q(**{f"{prefix}{field}__in": value})
         return queryset, models.Q()
 
     @strawberry_django.filter_field
@@ -85,9 +60,30 @@ def category(
         return self._info_enum_fields("category", queryset, value, prefix)
 
 
-@strawberry_django.filters.filter(AlertInfo, lookups=True)
-class AlertInfoFilter:
+@strawberry_django.filters.filter(Alert, lookups=True)
+class AlertFilter:
     id: strawberry.auto
+    country: strawberry.auto
+    sent: strawberry.auto
+    infos: AlertInfoFilter | None
+
+    @strawberry_django.filter_field
+    def region(
+        self,
+        queryset: models.QuerySet,
+        value: strawberry.ID,
+        prefix: str,
+    ) -> tuple[models.QuerySet, models.Q]:
+        return queryset, models.Q(**{f"{prefix}country__region": value})
+
+    @strawberry_django.filter_field
+    def admin1(
+        self,
+        queryset: models.QuerySet,
+        value: strawberry.ID,
+        prefix: str,
+    ) -> tuple[models.QuerySet, models.Q]:
+        return queryset, models.Q(**{f"{prefix}admin1s": value})
 
 
 @strawberry_django.filters.filter(Feed, lookups=True)

apps/cap_feed/fixtures/test_data.json

@@ -18,7 +18,8 @@
       "pk": 1,
       "fields": {
         "name": "test_country",
-        "iso3": "ISO"
+        "iso3": "ISO",
+        "region": 1
       }
     },
     {