Add import and export functionality for App Users. (#4)

* Add import and export functionality for App Users.

* Fix database connections for tests in GitHub Actions.

* Use full_clean to validate AppUserTemplateVariables during import.

* Unit test improvements.

* Show an error message if the import file cannot be read using the selected format.

* style forms; fix auto-selection of format on import

* Use a PostgreSQL service container for the GA workflow.

* Undo temp change in GA workflow.

* Add logging for database changes during imports.

* Add a success message for successful imports.

* Add some comments in our custom import-export classes and methods.

* Make AppUser.central_id nullable.

* Fix AttributeErrors in create_or_update_app_users().

* Test import of a new user with only the name provided.

* Test updating of null AppUser.central_id.

---------

Co-authored-by: Colin Copeland <copelco@caktusgroup.com>

.github/workflows/tests.yaml

@@ -8,6 +8,21 @@ on:
 jobs:
   tests:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        # From:
+        # https://docs.github.com/en/actions/guides/creating-postgresql-service-containers
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -21,4 +36,7 @@ jobs:
           sudo apt-get install -y --no-install-recommends postgresql-client
           pip install -U -q pip-tools
           pip-sync requirements/base/base.txt requirements/dev/dev.txt
-      - run: pytest
+      - name: Run tests
+        run: pytest
+        env:
+          DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres

apps/odk_publish/etl/load.py

@@ -1,6 +1,7 @@
 import structlog
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.db.models import QuerySet
+from django.db.transaction import atomic
 
 from ..models import AppUser, AppUserFormTemplate, CentralServer, FormTemplate, Project
 from .odk.client import ODKPublishClient
@@ -13,9 +14,7 @@ def create_or_update_app_users(form_template: FormTemplate):
     """Create or update app users for the form template."""
     app_user_forms: QuerySet[AppUserFormTemplate] = form_template.app_user_forms.select_related()
 
-    with ODKPublishClient.new_client(
-        base_url=form_template.project.central_server.base_url
-    ) as client:
+    with ODKPublishClient(base_url=form_template.project.central_server.base_url) as client:
         app_users = client.odk_publish.get_or_create_app_users(
             display_names=[app_user_form.app_user.name for app_user_form in app_user_forms],
             project_id=form_template.project.central_id,
@@ -24,9 +23,29 @@ def create_or_update_app_users(form_template: FormTemplate):
         for app_user_form in app_user_forms:
             app_users[app_user_form.app_user.name].xml_form_ids.append(app_user_form.xml_form_id)
         # Create or update the form assignments on the server
-        client.odk_publish.assign_forms(
+        client.odk_publish.assign_app_users_forms(
             app_users=app_users.values(), project_id=form_template.project.central_id
         )
+        # Update AppUsers with null central_id
+        update_app_users_central_id(form_template.project, app_users)
+
+
+@atomic
+def update_app_users_central_id(project: Project, app_users):
+    """Update AppUser.central_id for any user related to `project` that has a
+    null central_id, using the data in `app_users`. `app_users` should be a dict
+    mapping user names to ProjectAppUserAssignment objects, like the dict returned
+    by PublishService.get_or_create_app_users().
+    """
+    for app_user in project.app_users.filter(name__in=app_users, central_id__isnull=True):
+        app_user.central_id = app_users[app_user.name].id
+        app_user.save()
+        logger.info(
+            "Updated AppUser.central_id",
+            id=app_user.id,
+            name=app_user.name,
+            central_id=app_user.central_id,
+        )
 
 
 def generate_and_save_app_user_collect_qrcodes(project: Project):

apps/odk_publish/forms.py

@@ -1,9 +1,11 @@
 from django import forms
+from django.conf import settings
 from django.http import QueryDict
 from django.urls import reverse_lazy
+from import_export import forms as import_export_forms
 
 from apps.patterns.forms import PlatformFormMixin
-from apps.patterns.widgets import Select
+from apps.patterns.widgets import Select, FileInput
 
 from .etl.odk.client import ODKPublishClient
 from .http import HttpRequest
@@ -54,3 +56,75 @@ def set_project_choices(self, base_url: str):
             self.fields["project"].choices = [
                 (project.id, project.name) for project in client.projects.list()
             ]
+
+
+class FileFormatChoiceField(forms.ChoiceField):
+    """Field for selecting a file format for importing and exporting."""
+
+    widget = Select
+
+    def __init__(self, *args, **kwargs):
+        # Load the available file formats from Django settings
+        self.formats = settings.IMPORT_EXPORT_FORMATS
+        choices = [("", "---")] + [
+            (i, format().get_title()) for i, format in enumerate(self.formats)
+        ]
+        super().__init__(choices=choices, *args, **kwargs)
+
+    def clean(self, value):
+        """Return the selected file format instance."""
+        Format = self.formats[int(value)]
+        return Format()
+
+
+class AppUserImportExportFormMixin(PlatformFormMixin, forms.Form):
+    """Base form for importing and exporting AppUsers."""
+
+    format = FileFormatChoiceField()
+
+    def __init__(self, resources, **kwargs):
+        # Formats are handled by the FileFormatChoiceField, so we pass an empty list
+        # to the parent class
+        super().__init__(formats=[], resources=resources, **kwargs)
+
+    def _init_formats(self, formats):
+        # Again, formats are handled by the FileFormatChoiceField, so nothing to do here
+        pass
+
+
+class AppUserExportForm(AppUserImportExportFormMixin, import_export_forms.ImportExportFormBase):
+    """Form for exporting AppUsers to a file."""
+
+    pass
+
+
+class AppUserImportForm(AppUserImportExportFormMixin, import_export_forms.ImportForm):
+    """Form for importing AppUsers from a file."""
+
+    import_file = forms.FileField(label="File to import", widget=FileInput)
+
+    def __init__(self, resources, **kwargs):
+        super().__init__(resources, **kwargs)
+        # Add CSS classes to the import file and format fields so JS can detect them
+        self.fields["import_file"].widget.attrs["class"] = "guess_format"
+        self.fields["format"].widget.attrs["class"] = "guess_format"
+
+    def clean(self):
+        import_format = self.cleaned_data.get("format")
+        import_file = self.cleaned_data.get("import_file")
+        if import_format and import_file:
+            data = import_file.read()
+            if not import_format.is_binary():
+                import_format.encoding = "utf-8-sig"
+            try:
+                self.dataset = import_format.create_dataset(data)
+            except Exception:
+                raise forms.ValidationError(
+                    {
+                        "format": (
+                            "An error was encountered while trying to read the file. "
+                            "Ensure you have chosen the correct format for the file."
+                        )
+                    }
+                )
+        return self.cleaned_data

apps/odk_publish/import_export.py

@@ -0,0 +1,176 @@
+from decimal import InvalidOperation
+from functools import partial
+
+import structlog
+from django.core.exceptions import ValidationError
+from import_export import resources, fields, widgets
+
+from .models import AppUser, AppUserTemplateVariable
+
+logger = structlog.getLogger(__name__)
+
+
+class AppUserTemplateVariableWidget(widgets.ForeignKeyWidget):
+    """Widget for the AppUserTemplateVariable columns in the import/export files."""
+
+    def __init__(self, template_variable, **kwargs):
+        # The TemplateVariable instance for the column
+        self.template_variable = template_variable
+        super().__init__(AppUserTemplateVariable, field="value", **kwargs)
+
+    def get_lookup_kwargs(self, value, row, **kwargs):
+        # A dictionary used to query the AppUserTemplateVariable model to get
+        # the variable for the current row
+        return {"template_variable": self.template_variable, "app_user_id": row["id"] or None}
+
+    def clean(self, value, row=None, **kwargs):
+        """Validate the AppUserTemplateVariable during import."""
+        try:
+            template_variable = super().clean(value, row, **kwargs)
+        except AppUserTemplateVariable.DoesNotExist:
+            template_variable = AppUserTemplateVariable(
+                value=value, **self.get_lookup_kwargs(value, row, **kwargs)
+            )
+        if template_variable is not None:
+            template_variable.value = value
+            try:
+                template_variable.full_clean(exclude=["app_user"])
+            except ValidationError as e:
+                raise ValueError(e.messages[0])
+        return template_variable
+
+
+class AppUserTemplateVariableField(fields.Field):
+    """Field for the AppUserTemplateVariable columns in the import/export files."""
+
+    def save(self, instance, row, is_m2m=False, **kwargs):
+        """Queue up saving or deleting the AppUserTemplateVariable during import.
+        At this point the related AppUser may not exist yet (if "id" column is blank)
+        and/or may not be fully validated so we'll save or delete the
+        AppUserTemplateVariable later in `AppUserResource.do_instance_save()`.
+        """
+        # Get the validated AppUserTemplateVariable. It will be None if the value
+        # is blank in the import file, in which case we need to delete the variable
+        cleaned = self.clean(row, **kwargs)
+        if cleaned is None:
+            instance._template_variables_to_delete.append(self.column_name)
+        else:
+            instance._template_variables_to_save.append(cleaned)
+
+
+class PositiveIntegerWidget(widgets.IntegerWidget):
+    """Widget for the `central_id` column."""
+
+    def clean(self, value, row=None, **kwargs):
+        try:
+            val = super().clean(value, row, **kwargs)
+        except InvalidOperation:
+            # The base class will raise a decimal.InvalidOperation if the value
+            # is not a valid number
+            raise ValueError("Value must be an integer.")
+        if val and val < 0:
+            raise ValueError("Value must be positive.")
+        return val
+
+
+class AppUserResource(resources.ModelResource):
+    """Custom ModelResource for importing/exporting AppUsers."""
+
+    central_id = fields.Field(
+        attribute="central_id", column_name="central_id", widget=PositiveIntegerWidget()
+    )
+
+    class Meta:
+        model = AppUser
+        fields = ("id", "name", "central_id")
+        clean_model_instances = True
+
+    def __init__(self, project):
+        # The project for which we are importing/exporting AppUsers
+        self.project = project
+        super().__init__()
+        # Add columns for each TemplateVariable related to the project passed in
+        for template_variable in project.template_variables.all():
+            self.fields[template_variable.name] = AppUserTemplateVariableField(
+                attribute=template_variable.name,
+                column_name=template_variable.name,
+                # The `dehydrate_method` will be called with an AppUser instance as
+                # the only argument to get the value for the AppUserTemplateVariable
+                # column during export
+                dehydrate_method=partial(self.get_template_variable_value, template_variable.pk),
+                widget=AppUserTemplateVariableWidget(template_variable),
+            )
+
+    def export_resource(self, instance, selected_fields=None, **kwargs):
+        """Called for each AppUser instance during export."""
+        # Create a dictionary that we can use to look up template variables
+        # for the instance, instead of doing a DB query for each variable.
+        # The template variables are prefetched in the queryset passed in to
+        # the `AppUserResource.export()` call in the view
+        instance.template_variables_dict = {
+            i.template_variable_id: i.value for i in instance.app_user_template_variables.all()
+        }
+        return super().export_resource(instance, selected_fields, **kwargs)
+
+    def get_queryset(self):
+        # Queryset used to look up AppUsers during import
+        return self.project.app_users.all()
+
+    def get_instance(self, instance_loader, row):
+        """Called during import to get the current AppUser, by querying the
+        queryset from `get_queryset()` using the id from the current row.
+        """
+        # `instance` will be None if an AppUser with the ID could not be found
+        # in the queryset from `get_queryset()`
+        instance = super().get_instance(instance_loader, row)
+        if row["id"] and not instance:
+            raise ValidationError(
+                {"id": f"An app user with ID {row['id']} does not exist in the current project."}
+            )
+        return instance
+
+    @staticmethod
+    def get_template_variable_value(variable_pk, app_user):
+        """Used to set the `dehydrate_method` argument when instantiating a
+        AppUserTemplateVariableField.
+        """
+        return app_user.template_variables_dict.get(variable_pk)
+
+    def import_instance(self, instance, row, **kwargs):
+        """Called for each AppUser during import."""
+        instance.project = self.project
+        # We'll use these lists to queue up AppUserTemplateVariables for saving
+        # or deleting together in the do_instance_save() method below.
+        instance._template_variables_to_save = []
+        instance._template_variables_to_delete = []
+        super().import_instance(instance, row, **kwargs)
+
+    def do_instance_save(self, instance, is_create):
+        """Save the AppUser and save/delete the AppUserTemplateVariables."""
+        logger.info(
+            "Updating AppUser via file import",
+            new=is_create,
+            id=instance.id,
+            project_id=self.project.id,
+        )
+        instance.save()
+        for template_variable in instance._template_variables_to_save:
+            template_variable.app_user = instance
+            logger.info(
+                "Updating AppUserTemplateVariable via file import",
+                new=not template_variable.pk,
+                id=template_variable.id,
+                variable_name=template_variable.template_variable.name,
+                app_user_id=instance.id,
+                project_id=self.project.id,
+            )
+            template_variable.save()
+        if not is_create:
+            for name in instance._template_variables_to_delete:
+                logger.info(
+                    "Deleting AppUserTemplateVariable via file import",
+                    variable_name=name,
+                    app_user_id=instance.id,
+                    project_id=self.project.id,
+                )
+                instance.app_user_template_variables.filter(template_variable__name=name).delete()

apps/odk_publish/migrations/0002_alter_appuser_central_id.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.5 on 2025-02-07 16:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("odk_publish", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="appuser",
+            name="central_id",
+            field=models.PositiveIntegerField(
+                blank=True,
+                help_text="The ID of this app user in ODK Central.",
+                null=True,
+                verbose_name="app user ID",
+            ),
+        ),
+    ]

apps/odk_publish/models.py

@@ -174,7 +174,10 @@ def __str__(self):
 class AppUser(AbstractBaseModel):
     name = models.CharField(max_length=255)
     central_id = models.PositiveIntegerField(
-        verbose_name="app user ID", help_text="The ID of this app user in ODK Central."
+        verbose_name="app user ID",
+        help_text="The ID of this app user in ODK Central.",
+        blank=True,
+        null=True,
     )
     project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name="app_users")
     qr_code = models.ImageField(

apps/odk_publish/urls.py

@@ -24,6 +24,16 @@
         views.app_user_generate_qr_codes,
         name="app-users-generate-qr-codes",
     ),
+    path(
+        "<int:odk_project_pk>/app-users/export/",
+        views.app_user_export,
+        name="app-users-export",
+    ),
+    path(
+        "<int:odk_project_pk>/app-users/import/",
+        views.app_user_import,
+        name="app-users-import",
+    ),
     path(
         "<int:odk_project_pk>/form-templates/",
         views.form_template_list,