Stored Favorite / Membership Counts (#601)

* Linting changes to Management Command

* Black Formatting Changes

* Remove Postgres Files

* Remove Docker Compose Changes

* Add Back Volumes from Docker Compose

* Another Docker Compose Fix

* Bulk Update and Testing

* Merge in New Master

* Re-generate Pipfile.lock

* Revert Pipfile to wildcard Python 3 and re-gen lock file using 3.8

---------

Co-authored-by: Shiva Menta <shivamenta@ist2-07013.apn.wlan.private.upenn.edu>
Co-authored-by: Shiva Menta <shivamenta@ist2-03085.apn.wlan.private.upenn.edu>
Co-authored-by: Julian Weng <julian.weng.us@gmail.com>

backend/clubs/management/commands/update_club_counts.py

@@ -0,0 +1,36 @@
+from django.core.management.base import BaseCommand
+from django.db.models import Count, Q
+
+from clubs.models import Club
+
+
+class Command(BaseCommand):
+    help = "Update stored favorite and membership counts."
+
+    def handle(self, *args, **kwargs):
+        try:
+            queryset = Club.objects.all().annotate(
+                temp_favorite_count=Count("favorite", distinct=True),
+                temp_membership_count=Count(
+                    "membership", distinct=True, filter=Q(active=True)
+                ),
+            )
+
+            for club in queryset:
+                club.favorite_count = club.temp_favorite_count
+                club.membership_count = club.temp_membership_count
+            Club.objects.bulk_update(queryset, ["favorite_count", "membership_count"])
+
+            self.stdout.write(
+                self.style.SUCCESS(
+                    "Successfully updated all club favorite and membership counts!"
+                )
+            )
+        except Exception as e:
+            self.stdout.write(
+                self.style.ERROR(
+                    "An error was encountered while updating"
+                    + "club favorite and membership counts!"
+                )
+            )
+            self.stdout.write(e)

backend/clubs/migrations/0091_auto_20231019_1730.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.2.18 on 2023-10-19 21:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("clubs", "0090_auto_20230106_1443"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="club",
+            name="favorite_count",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="club",
+            name="membership_count",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="historicalclub",
+            name="favorite_count",
+            field=models.IntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name="historicalclub",
+            name="membership_count",
+            field=models.IntegerField(default=0),
+        ),
+    ]

backend/clubs/migrations/0095_merge_20240124_1227.py

@@ -0,0 +1,13 @@
+# Generated by Django 3.2.18 on 2024-01-24 17:27
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("clubs", "0091_auto_20231019_1730"),
+        ("clubs", "0094_applicationcycle_release_date"),
+    ]
+
+    operations = []

backend/clubs/models.py

@@ -322,6 +322,10 @@ class Club(models.Model):
     appointment_needed = models.BooleanField(default=False)
     signature_events = models.TextField(blank=True)  # html
 
+    # cache club aggregation counts
+    favorite_count = models.IntegerField(default=0)
+    membership_count = models.IntegerField(default=0)
+
     # cache club rankings
     rank = models.IntegerField(default=0)
 

backend/clubs/views.py

@@ -1014,13 +1014,7 @@ class ClubViewSet(XLSXFormatterMixin, viewsets.ModelViewSet):
     """
 
     queryset = (
-        Club.objects.all()
-        .annotate(
-            favorite_count=Count("favorite", distinct=True),
-            membership_count=Count("membership", distinct=True, filter=Q(active=True)),
-        )
-        .prefetch_related("tags")
-        .order_by("-favorite_count", "name")
+        Club.objects.all().prefetch_related("tags").order_by("-favorite_count", "name")
     )
     permission_classes = [ClubPermission | IsSuperuser]
     filter_backends = [filters.SearchFilter, ClubsSearchFilter, ClubsOrderingFilter]

backend/docker-compose.yaml

@@ -0,0 +1,14 @@
+version: "3"
+
+services:
+  db:
+    image: postgres
+    command: postgres
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_USER=penn-clubs
+      - POSTGRES_PASSWORD=postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - ./postgres:/var/lib/postgresql/pgdata

backend/tests/clubs/test_commands.py

@@ -650,3 +650,53 @@ def test_wrong_arguments(self):
 
         with self.assertRaises(CommandError):
             call_command("merge_duplicates", "--tag")
+
+
+class StoredFavoritesMembershipsTestCase(TestCase):
+    def setUp(self):
+        self.club1 = Club.objects.create(code="one", name="One", active=True)
+        self.club2 = Club.objects.create(code="two", name="One", active=True)
+
+        self.user1 = get_user_model().objects.create_user(
+            "bfranklin1", "bfranklin1@seas.upenn.edu", "test"
+        )
+        self.user2 = get_user_model().objects.create_user(
+            "bfranklin2", "bfranklin2@seas.upenn.edu", "test"
+        )
+        self.user3 = get_user_model().objects.create_user(
+            "bfranklin3", "bfranklin2@seas.upenn.edu", "test"
+        )
+
+        # Favorites
+        Favorite.objects.create(person=self.user1, club=self.club1)
+        Favorite.objects.create(person=self.user1, club=self.club2)
+        Favorite.objects.create(person=self.user2, club=self.club1)
+        Favorite.objects.create(person=self.user3, club=self.club2)
+
+        # Membership
+        Membership.objects.create(
+            club=self.club1, person=self.user1, role=Membership.ROLE_OWNER
+        )
+        Membership.objects.create(
+            club=self.club1, person=self.user2, role=Membership.ROLE_MEMBER
+        )
+        Membership.objects.create(
+            club=self.club1, person=self.user3, role=Membership.ROLE_MEMBER
+        )
+        Membership.objects.create(
+            club=self.club2, person=self.user2, role=Membership.ROLE_OWNER
+        )
+
+    def test_update_club_counts(self):
+        call_command("update_club_counts")
+
+        club1 = Club.objects.filter(code="one").first()
+        club2 = Club.objects.filter(code="two").first()
+
+        self.assertEqual(club1.favorite_count, 2)
+        self.assertEqual(club2.favorite_count, 2)
+        self.assertEqual(club1.membership_count, 3)
+        self.assertEqual(club2.membership_count, 1)
+        self.assertEqual(
+            Club.objects.all().order_by("-favorite_count", "name").first().code, "one"
+        )

k8s/main.ts

@@ -149,6 +149,13 @@ export class MyChart extends PennLabsChart {
       secret: fyhSecret,
       cmd: ["python", "manage.py", "import_paideia_events"],
     });
+
+    new CronJob(this, 'update-club-aggregation-counts', {
+      schedule: cronTime.everyDayAt(12),
+      image: backendImage,
+      secret: fyhSecret,
+      cmd: ["python", "manage.py", "update_club_counts"],
+    })
   }
 }