work adding import functionality using django-import-export

dev-requirements.txt

@@ -6,4 +6,4 @@ django-debug-toolbar
 sphinx
 wheel
 pre-commit
-wagtail-factories
+wagtail-factories

mep/common/admin.py

@@ -2,6 +2,12 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
+from import_export.resources import ModelResource
+from parasolr.django.signals import IndexableSignalHandler
+from django.conf import settings
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class NamedNotableAdmin(admin.ModelAdmin):
@@ -43,5 +49,71 @@
     group_names.short_description = "groups"
 
 
+class ImportExportModelResource(ModelResource):
+    def __init__(self, *x, **y):
+        super().__init__(*x, **y)
+        # list to contain updated objects for batch indexing at end
+        self.objects_to_index = []
+
+    def before_import(self, dataset, *args, **kwargs):
+        # lower and camel_case headers
+        dataset.headers = [x.lower().replace(" ", "_") for x in dataset.headers]
+
+        # turn off indexing temporarily
+        IndexableSignalHandler.disconnect()
+
+        # turn off viaf lookups
+        settings.SKIP_VIAF_LOOKUP = True
+
+    def before_import_row(self, row, **kwargs):
+        """
+        Called on an OrderedDictionary of row attributes.
+        Opportunity to do quick string formatting as a
+        principle of charity to annotators before passing
+        values into django-import-export lookup logic.
+        """
+        pass
+
+    def after_save_instance(self, instance, using_transactions, dry_run):
+        """
+        Called when an instance either was or would be saved (depending on dry_run)
+        """
+        self.objects_to_index.append(instance)
+        return super().after_save_instance(instance, using_transactions, dry_run)
+
+    def after_import(self, dataset, result, using_transactions, dry_run, **kwargs):
+        """
+        Called after importing, twice: once with dry_run==True (preview),
+        once dry_run==False. We report how many objects were updated and need to be indexed.
+        We only do so when dry_run is False.
+        """
+        # run parent method
+        super().after_import(dataset, result, using_transactions, dry_run, **kwargs)
+
+        # report how many need indexing
+        logger.debug(
+            f"indexing {len(self.objects_to_index)} objects, dry_run = {dry_run}"
+        )
+
+        # only continue if not a dry run
+        if not dry_run:
+            # re-enable indexing
+            IndexableSignalHandler.connect()
+
+            # index objects
+            if self.objects_to_index:
+                self.Meta.model.index_items(self.objects_to_index)
+
+        # turn viaf lookups back on
+        settings.SKIP_VIAF_LOOKUP = False
+
+        # make sure indexing disconnected afterward
+        IndexableSignalHandler.disconnect()
+
+    class Meta:
+        skip_unchanged = True
+        report_skipped = True
+
+
 admin.site.unregister(User)
 admin.site.register(User, LocalUserAdmin)

mep/people/admin.py

@@ -1,3 +1,4 @@
+import logging
 from dal import autocomplete
 from django import forms
 from django.conf import settings
@@ -9,14 +10,14 @@
 from django.utils.timezone import now
 from tabular_export.admin import export_to_csv_response
 from viapy.widgets import ViafWidget
-
 from mep.accounts.admin import AddressInline
 from mep.common.admin import (
     CollapsedTabularInline,
     CollapsibleTabularInline,
     NamedNotableAdmin,
 )
 from mep.footnotes.admin import FootnoteInline
+from mep.common.admin import ImportExportModelResource
 
 from .models import (
     Country,
@@ -27,6 +28,41 @@
     Relationship,
     RelationshipType,
 )
+from import_export.admin import (
+    ImportExportModelAdmin,
+)
+from import_export.resources import ModelResource
+from import_export.widgets import ManyToManyWidget, Widget
+from import_export.fields import Field
+from parasolr.django.signals import IndexableSignalHandler
+
+PERSON_IMPORT_COLUMNS = ("slug", "gender", "nationalities")
+
+PERSON_IMPORT_EXPORT_COLUMNS = (
+    "slug",
+    "name",
+    "birth_year",
+    "death_year",
+    "gender",
+    "nationalities",
+    "notes",
+    "start_year",
+    "end_year",
+    "mep_id",
+    "sort_name",
+    "viaf_id",
+    "is_organization",
+    "verified",
+    "title",
+    "profession",
+    "relations",
+    "public_notes",
+    "locations",
+    "updated_at",
+    "id",
+)
+
+logger = logging.getLogger(__name__)
 
 
 class InfoURLInline(CollapsibleTabularInline):
@@ -436,8 +472,55 @@ class Media:
         ]
 
 
+class ExportPersonResource(ModelResource):
+    class Meta:
+        model = Person
+        fields = PERSON_IMPORT_EXPORT_COLUMNS
+        export_order = PERSON_IMPORT_EXPORT_COLUMNS
+
+
+class PersonResource(ImportExportModelResource):
+    def before_import_row(self, row, **kwargs):
+        """
+        Called on an OrderedDictionary of row attributes.
+        Opportunity to do quick string formatting as a
+        principle of charity to annotators before passing
+        values into django-import-export lookup logic.
+        """
+        # gender to one char
+        gstr = str(row.get("gender")).strip()
+        row["gender"] = gstr[0].upper() if gstr else ""
+
+    # only customized fields need specifying here
+    nationalities = Field(
+        column_name="nationalities",
+        attribute="nationalities",
+        widget=ManyToManyWidget(Country, field="name", separator=";"),
+    )
+
+    class Meta:
+        model = Person
+        fields = PERSON_IMPORT_COLUMNS
+        import_id_fields = ("slug",)
+        export_order = PERSON_IMPORT_COLUMNS
+        skip_unchanged = True
+        report_skipped = True
+        store_instance = True
+
+
+class PersonAdminImportExport(PersonAdmin, ImportExportModelAdmin):
+    resource_classes = [PersonResource]
+
+    def get_export_resource_classes(self):
+        """
+        Specifies the resource class to use for exporting,
+        so that separate fields can be exported than those imported
+        """
+        return [ExportPersonResource]
+
+
 # enable default admin to see imported data
-admin.site.register(Person, PersonAdmin)
+admin.site.register(Person, PersonAdminImportExport)
 admin.site.register(Country, CountryAdmin)
 admin.site.register(Location, LocationAdmin)
 admin.site.register(Profession, NamedNotableAdmin)

mep/people/models.py

@@ -1,7 +1,7 @@
 import datetime
 import logging
 from string import punctuation
-
+from django.conf import settings
 from django.apps import apps
 from django.contrib.contenttypes.fields import GenericRelation
 from django.core.exceptions import MultipleObjectsReturned
@@ -533,7 +533,7 @@ def save(self, *args, **kwargs):
         """Adds birth and death dates if they aren't already set
         and there's a viaf id for the record"""
 
-        if self.viaf_id and not self.birth_year and not self.death_year:
+        if not getattr(settings,'SKIP_VIAF_LOOKUP',False) and self.viaf_id and not self.birth_year and not self.death_year:
             self.set_birth_death_years()
 
         # if slug has changed, save the old one as a past slug