Release v1.7

CHANGELOG.rst

@@ -1,6 +1,19 @@
 CHANGELOG
 =========
 
+1.7
+---
+
+Revisions to data exports in preparation for 2.0 dataset publication:
+
+- Address export files renamed `member_addresses` (both csv and json)
+- Creator export files renamed `book_creators` (both csv and json)
+- Address export manage command now does not include start and end dates by default,
+with an optional flag to include them
+- Address export includes care of person id and name, location name, and member names, sort names, and uris
+- Nested information in JSON data exports is now grouped by entity
+- Add support for Plausible analytics, configurable with PLAUSIBLE_ANALYTICS_SCRIPT and PLAUSIBLE_ANALYTICS_404s in django settings
+
 1.6.2
 -----
 

mep/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.6.2"
+__version__ = "1.7.0"
 
 
 # context processor to add version to the template environment

mep/accounts/management/commands/export_addresses.py

@@ -7,7 +7,8 @@
 addresses.
 """
 
-from django.db.models import Prefetch
+import argparse
+
 from mep.common.management.export import BaseExport
 from mep.common.utils import absolutize_url
 from mep.accounts.models import Address
@@ -20,11 +21,14 @@ class Command(BaseExport):
 
     model = Address
 
-    csv_fields = [
+    _csv_fields = [
         "member_ids",  # member slug
+        "member_names",
+        "member_sort_names",
         "member_uris",
         "care_of_person_id",  # c/o person slug
-        "care_of_person",  # c/o person name
+        "care_of_person_name",  # c/o person name
+        "location_name",  # location name if there is one
         "street_address",
         "postal_code",
         "city",
@@ -35,38 +39,61 @@ class Command(BaseExport):
         "longitude",
         "latitude",
     ]
+    include_dates = False
+
+    def add_arguments(self, parser):
+        super().add_arguments(parser)
+        # in addition to base options, add a param to control date inclusion
+        parser.add_argument(
+            "--dates",
+            action=argparse.BooleanOptionalAction,
+            default=self.include_dates,
+            help="Include start and end dates from export?",
+        )
+
+    @property
+    def csv_fields(self):
+        if not self.include_dates:
+            return [f for f in self._csv_fields if not f.endswith("_date")]
+        else:
+            return self._csv_fields
+
+    def handle(self, *args, **kwargs):
+        """Export all model data into a CSV file and JSON file."""
+        self.include_dates = kwargs.get("dates", self.include_dates)
+        super().handle(*args, **kwargs)
 
     def get_queryset(self):
         """
         prefetch account, location and account persons
         """
-        return Address.objects.prefetch_related(
+        # skip any addresses not associated with a member
+        return Address.objects.filter(account__persons__isnull=False).prefetch_related(
             "account",
             "location",
             "account__persons",
         )
 
     def get_base_filename(self):
-        """set export filename to 'addresses.csv'"""
-        return "addresses"
+        """set export filename to 'member_addresses.csv'"""
+        return "member_addresses"
 
-    def get_object_data(self, addr):
+    def get_object_data(self, obj):
         """
         Generate dictionary of data to export for a single
         :class:`~mep.people.models.Person`
         """
-        loc = addr.location
-        persons = addr.account.persons.all()
+        loc = obj.location
 
         # required properties
         data = dict(
             # Member info
-            member=self.member_info(addr),
+            members=self.member_info(obj),
             # Address data
-            start_date=addr.partial_start_date,
-            end_date=addr.partial_end_date,
-            care_of_person_id=addr.care_of_person.slug if addr.care_of_person else None,
-            care_of_person=addr.care_of_person.name if addr.care_of_person else None,
+            start_date=obj.partial_start_date,
+            end_date=obj.partial_end_date,
+            care_of_person_id=obj.care_of_person.slug if obj.care_of_person else None,
+            care_of_person_name=obj.care_of_person.name if obj.care_of_person else None,
             # Location data
             street_address=loc.street_address,
             city=loc.city,
@@ -76,19 +103,23 @@ def get_object_data(self, addr):
             country=loc.country.name if loc.country else None,
             arrondissement=loc.arrondissement(),
         )
+        if not self.include_dates:
+            del data["start_date"]
+            del data["end_date"]
         # filter out unset values so we don't get unnecessary content in json
         return {k: v for k, v in data.items() if v is not None}
 
     def member_info(self, location):
         """Event about member(s) associated with this location"""
-        # adapted from event export logic
-        # NOTE: would be nicer and more logical if each member had their own
-        # dict entry, but that doesn't work with current flatting logic for csv
+        # adapted from event export logic; returns a list of dictionaries
+        # with member id, uri, name and sort name
         members = location.account.persons.all()
-        return dict(
-            ids=[m.slug for m in members],
-            uris=[absolutize_url(m.get_absolute_url()) for m in members],
-            # useful to include or too redundant?
-            # ("names", [m.name for m in members]),
-            # ("sort_names", [m.sort_name for m in members]),
-        )
+        return [
+            dict(
+                id=m.slug,
+                uri=absolutize_url(m.get_absolute_url()),
+                name=m.name,
+                sort_name=m.sort_name,
+            )
+            for m in members
+        ]

mep/accounts/management/commands/export_events.py

@@ -12,7 +12,6 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models.functions import Coalesce
 from django.db.models.query import Prefetch
-from djiffy.models import Manifest
 
 from mep.accounts.models import Event
 from mep.books.models import Creator
@@ -94,18 +93,15 @@ def get_object_data(self, obj):
         """Generate a dictionary of data to export for a single
         :class:`~mep.accounts.models.Event`"""
         event_type = obj.event_type
-        data = OrderedDict(
-            [
-                # use event label instead of type for more detail on some generics
-                ("event_type", obj.event_label),
-                ("start_date", obj.partial_start_date or ""),
-                ("end_date", obj.partial_end_date or ""),
-                ("member", OrderedDict()),
-            ]
+        data = dict(
+            # use event label instead of type for more detail on some generics
+            event_type=obj.event_label,
+            start_date=obj.partial_start_date or "",
+            end_date=obj.partial_end_date or "",
         )
         member_info = self.member_info(obj)
         if member_info:
-            data["member"] = member_info
+            data["members"] = member_info
 
         currency = None
 
@@ -150,14 +146,15 @@ def member_info(self, event):
         if not members:
             return
 
-        return OrderedDict(
-            [
-                ("ids", [m.slug for m in members]),
-                ("uris", [absolutize_url(m.get_absolute_url()) for m in members]),
-                ("names", [m.name for m in members]),
-                ("sort_names", [m.sort_name for m in members]),
-            ]
-        )
+        return [
+            dict(
+                id=m.slug,
+                uri=absolutize_url(m.get_absolute_url()),
+                name=m.name,
+                sort_name=m.sort_name,
+            )
+            for m in members
+        ]
 
     def subscription_info(self, event):
         """subscription details for an event"""
@@ -187,11 +184,9 @@ def subscription_info(self, event):
     def item_info(self, event):
         """associated work details for an event"""
         if event.work:
-            item_info = OrderedDict(
-                [
-                    ("uri", absolutize_url(event.work.get_absolute_url())),
-                    ("title", event.work.title),
-                ]
+            item_info = dict(
+                uri=absolutize_url(event.work.get_absolute_url()),
+                title=event.work.title,
             )
             if event.edition:
                 item_info["volume"] = event.edition.display_text()
@@ -206,11 +201,9 @@ def item_info(self, event):
 
     def source_info(self, footnote):
         """source details from a footnote"""
-        source_info = OrderedDict(
-            [
-                ("type", footnote.bibliography.source_type.name),
-                ("citation", footnote.bibliography.bibliographic_note),
-            ]
+        source_info = dict(
+            type=footnote.bibliography.source_type.name,
+            citation=footnote.bibliography.bibliographic_note,
         )
         if footnote.bibliography.manifest:
             source_info["manifest"] = footnote.bibliography.manifest.uri

mep/accounts/tests/test_accounts_commands.py

@@ -437,24 +437,23 @@ def test_get_data(self):
         assert data.total == Event.objects.count()
         event_data = list(data)
         assert len(event_data) == Event.objects.count()
-        assert isinstance(event_data[0], OrderedDict)
+        assert isinstance(event_data[0], dict)
 
     def test_member_info(self):
         # test single member data
         event = Event.objects.filter(account__persons__name__contains="Brue").first()
         person = event.account.persons.first()
         member_info = self.cmd.member_info(event)
-        assert member_info["sort_names"][0] == person.sort_name
-        assert member_info["names"][0] == person.name
-        assert member_info["uris"][0] == absolutize_url(person.get_absolute_url())
+        assert member_info[0]["sort_name"] == person.sort_name
+        assert member_info[0]["name"] == person.name
+        assert member_info[0]["uri"] == absolutize_url(person.get_absolute_url())
 
         # event with two members; fixture includes Edel joint account
         event = Event.objects.filter(account__persons__name__contains="Edel").first()
 
         member_info = self.cmd.member_info(event)
-        # each field should have two values
-        for field in ("sort_names", "names", "uris"):
-            assert len(member_info[field]) == 2
+        # we should have two members
+        assert len(member_info) == 2
 
         # test event with account but no person
         nomember = Event.objects.filter(account__persons__isnull=True).first()
@@ -578,10 +577,11 @@ def test_get_object_data(self):
             end_date__isnull=False,
             subscription__category__isnull=True,
         ).first()
+
         data = self.cmd.get_object_data(event)
         assert data["event_type"] == event.event_label
         assert data["currency"] == "FRF"
-        assert "member" in data
+        assert "member" not in data  # we don't want an empty member dict
         assert "subscription" in data
 
         # test separate payment event includes subscription info
@@ -638,16 +638,25 @@ def test_get_queryset(self):
         assert address.location == location
         assert member in set(address.account.persons.all())
 
+    def test_csv_fields(self):
+        self.cmd.include_dates = True
+        assert self.cmd.csv_fields == self.cmd._csv_fields
+        self.cmd.include_dates = False
+        assert len(self.cmd.csv_fields) != len(self.cmd._csv_fields)
+        assert "start_date" not in self.cmd.csv_fields
+        assert "end_date" not in self.cmd.csv_fields
+
     def test_get_object_data(self):
         # fetch some example people from fixture & call get_object_data
         address = Address.objects.get(pk=236)
+        # with dates included
+        self.cmd.include_dates = True
         gay_data = self.cmd.get_object_data(address)
-
         # check some basic data
 
         # slug is 'gay' in sample_people, 'gay-francisque' in db
-        assert gay_data["member"]["ids"] == ["gay"]
-        assert gay_data["member"]["uris"] == ["https://example.com/members/gay/"]
+        assert gay_data["members"][0]["id"] == "gay"
+        assert gay_data["members"][0]["uri"] == "https://example.com/members/gay/"
 
         # check addresses & coordinates
         assert "3 Rue Garancière" == gay_data["street_address"]
@@ -661,3 +670,17 @@ def test_get_object_data(self):
         assert gay_data["start_date"] == "1919-01-01"
         assert gay_data["end_date"] == "1930-01-01"
         assert gay_data["care_of_person_id"] == "hemingway"
+
+        # without dates
+        self.cmd.include_dates = False
+        gay_data = self.cmd.get_object_data(address)
+        # doesn't include dates
+        assert "start_date" not in gay_data
+        assert "end_date" not in gay_data
+        # other member data
+        assert gay_data["members"][0]["id"] == "gay"
+        assert gay_data["members"][0]["uri"] == "https://example.com/members/gay/"
+        assert gay_data["members"][0]["name"] == "Francisque Gay"
+        assert gay_data["members"][0]["sort_name"] == "Gay, Francisque"
+        assert gay_data["care_of_person_id"] == "hemingway"
+        assert gay_data["care_of_person_name"] == "Ernest Hemingway"

mep/books/management/commands/export_books.py

@@ -7,7 +7,6 @@
 
 """
 
-from collections import OrderedDict
 from django.db.models import F, Prefetch
 from mep.books.models import CreatorType, Work
 from mep.common.management.export import BaseExport
@@ -76,12 +75,10 @@ def get_object_data(self, work):
         :class:`~mep.books.models.Work`
         """
         # required properties
-        data = OrderedDict(
-            [
-                ("id", work.slug),
-                ("uri", absolutize_url(work.get_absolute_url())),
-                ("title", work.title),
-            ]
+        data = dict(
+            id=work.slug,
+            uri=absolutize_url(work.get_absolute_url()),
+            title=work.title,
         )
         data.update(self.creator_info(work))
         if work.year:
@@ -122,7 +119,7 @@ def get_object_data(self, work):
     def creator_info(self, work):
         """Add information about authors, editors, etc based on creators
         associated with this work."""
-        info = OrderedDict()
+        info = {}
         for creator_type in self.creator_types:
             creators = work.creator_by_type(creator_type)
             if creators: