Add import and export functionality for App Users.

.github/workflows/tests.yaml

@@ -21,4 +21,12 @@ jobs:
           sudo apt-get install -y --no-install-recommends postgresql-client
           pip install -U -q pip-tools
           pip-sync requirements/base/base.txt requirements/dev/dev.txt
-      - run: pytest
+      - name: Bring up auxiliary services
+        run: |
+          export COMPOSE_PARALLEL_LIMIT=1
+          while ! docker compose pull; do sleep 1; done
+          docker compose up --detach
+      - name: Run tests
+        run: |
+          export DATABASE_URL=postgresql://postgres@localhost:9062/odk_publish
+          pytest

apps/odk_publish/import_export.py

@@ -0,0 +1,108 @@
+from decimal import InvalidOperation
+from functools import partial
+
+from django.core.exceptions import ValidationError
+from import_export import resources, fields, widgets
+
+from .models import AppUser, AppUserTemplateVariable
+
+
+class AppUserTemplateVariableWidget(widgets.ForeignKeyWidget):
+    def __init__(self, template_variable, **kwargs):
+        self.template_variable = template_variable
+        super().__init__(AppUserTemplateVariable, field="value", **kwargs)
+
+    def get_lookup_kwargs(self, value, row, **kwargs):
+        return {"template_variable": self.template_variable, "app_user_id": row["id"] or None}
+
+    def clean(self, value, row=None, **kwargs):
+        try:
+            template_variable = super().clean(value, row, **kwargs)
+        except AppUserTemplateVariable.DoesNotExist:
+            template_variable = AppUserTemplateVariable(
+                value=value, **self.get_lookup_kwargs(value, row, **kwargs)
+            )
+        if template_variable is not None:
+            template_variable.value = value
+            try:
+                template_variable.full_clean(exclude=["app_user"])
+            except ValidationError as e:
+                raise ValueError(e.messages[0])
+        return template_variable
+
+
+class AppUserTemplateVariableField(fields.Field):
+    def save(self, instance, row, is_m2m=False, **kwargs):
+        cleaned = self.clean(row, **kwargs)
+        if cleaned is None:
+            instance._template_variables_to_delete.append(self.column_name)
+        else:
+            instance._template_variables_to_save.append(cleaned)
+
+
+class PositiveIntegerWidget(widgets.IntegerWidget):
+    def clean(self, value, row=None, **kwargs):
+        try:
+            val = super().clean(value, row, **kwargs)
+        except InvalidOperation:
+            raise ValueError("Value must be an integer.")
+        if val and val < 0:
+            raise ValueError("Value must be positive.")
+        return val
+
+
+class AppUserResource(resources.ModelResource):
+    central_id = fields.Field(
+        attribute="central_id", column_name="central_id", widget=PositiveIntegerWidget()
+    )
+
+    class Meta:
+        model = AppUser
+        fields = ("id", "name", "central_id")
+        clean_model_instances = True
+
+    def __init__(self, project):
+        self.project = project
+        super().__init__()
+        for template_variable in project.template_variables.all():
+            self.fields[template_variable.name] = AppUserTemplateVariableField(
+                attribute=template_variable.name,
+                column_name=template_variable.name,
+                dehydrate_method=partial(self.get_template_variable_value, template_variable.pk),
+                widget=AppUserTemplateVariableWidget(template_variable),
+            )
+
+    def export_resource(self, instance, selected_fields=None, **kwargs):
+        instance.template_variables_dict = {
+            i.template_variable_id: i.value for i in instance.app_user_template_variables.all()
+        }
+        return super().export_resource(instance, selected_fields, **kwargs)
+
+    def get_queryset(self):
+        return self.project.app_users.all()
+
+    def get_instance(self, instance_loader, row):
+        instance = super().get_instance(instance_loader, row)
+        if row["id"] and not instance:
+            raise ValidationError(
+                {"id": f"An app user with ID {row['id']} does not exist in the current project."}
+            )
+        return instance
+
+    @staticmethod
+    def get_template_variable_value(variable_pk, app_user):
+        return app_user.template_variables_dict.get(variable_pk)
+
+    def import_instance(self, instance, row, **kwargs):
+        instance.project = self.project
+        instance._template_variables_to_save = []
+        instance._template_variables_to_delete = []
+        super().import_instance(instance, row, **kwargs)
+
+    def do_instance_save(self, instance, is_create):
+        instance.save()
+        for template_variable in instance._template_variables_to_save:
+            template_variable.app_user = instance
+            template_variable.save()
+        for name in instance._template_variables_to_delete:
+            instance.app_user_template_variables.filter(template_variable__name=name).delete()

apps/odk_publish/urls.py

@@ -24,6 +24,16 @@
         views.app_user_generate_qr_codes,
         name="app-users-generate-qr-codes",
     ),
+    path(
+        "<int:odk_project_pk>/app-users/export/",
+        views.app_user_export,
+        name="app-users-export",
+    ),
+    path(
+        "<int:odk_project_pk>/app-users/import/",
+        views.app_user_import,
+        name="app-users-import",
+    ),
     path(
         "<int:odk_project_pk>/form-templates/",
         views.form_template_list,

apps/odk_publish/views.py

@@ -6,9 +6,13 @@
 from django.http import HttpRequest, HttpResponse
 from django.shortcuts import get_object_or_404, redirect, render
 from django.views.decorators.http import require_http_methods
+from django.utils.timezone import localdate
+from import_export.forms import ImportForm, ExportForm
+from import_export.formats.base_formats import CSV, XLS, XLSX
 
 from .etl.load import generate_and_save_app_user_collect_qrcodes, sync_central_project
 from .forms import ProjectSyncForm
+from .import_export import AppUserResource
 from .models import FormTemplateVersion, FormTemplate
 from .nav import Breadcrumbs
 from .tables import FormTemplateTable
@@ -144,3 +148,87 @@ def form_template_publish_next_version(request: HttpRequest, odk_project_pk, for
     version = form_template.create_next_version(user=request.user)
     messages.add_message(request, messages.SUCCESS, f"{version} published.")
     return HttpResponse(status=204)
+
+
+@login_required
+def app_user_export(request, odk_project_pk):
+    """Exports AppUsers to a CSV or Excel file.
+
+    For each user in the current project, there will be "id", "name", and "central_id"
+    columns. Additionally there will be a column for each TemplateVariable related to
+    the current project.
+    """
+    formats = [CSV, XLS, XLSX]
+    resource = AppUserResource(request.odk_project)
+    if request.method == "POST":
+        form = ExportForm(formats, [resource], data=request.POST)
+        if form.is_valid():
+            export_format = formats[int(form.cleaned_data["format"])]()
+            dataset = resource.export(
+                request.odk_project.app_users.prefetch_related("app_user_template_variables")
+            )
+            data = export_format.export_data(dataset)
+            filename = f"app_users_{odk_project_pk}_{localdate()}.{export_format.get_extension()}"
+            return HttpResponse(
+                data,
+                content_type=export_format.get_content_type(),
+                headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+            )
+    else:
+        form = ExportForm(formats, [resource])
+    context = {
+        "breadcrumbs": Breadcrumbs.from_items(
+            request=request,
+            items=[("App Users", "app-user-list"), ("Export", "app-users-export")],
+        ),
+        "form": form,
+    }
+    return render(request, "odk_publish/app_user_export.html", context)
+
+
+@login_required
+def app_user_import(request, odk_project_pk):
+    """Imports AppUsers from a CSV or Excel file.
+
+    The file is expected to have the same columns as a file exported using the
+    app_user_export view above. New AppUsers will be added if the "id" column is blank.
+    If a AppUserTemplateVariable column is blank and it exists in the database,
+    it will be deleted.
+    """
+    # TODO: Add a preview page after a file is uploaded, similar to imports in Django Admin
+    formats = [CSV, XLS, XLSX]
+    resource = AppUserResource(request.odk_project)
+    if request.POST:
+        form = ImportForm(formats, [resource], data=request.POST, files=request.FILES)
+        if form.is_valid():
+            import_format = formats[int(form.cleaned_data["format"])]()
+            data = form.cleaned_data["import_file"].read()
+            if not import_format.is_binary():
+                data = data.decode()
+            dataset = import_format.create_dataset(data)
+            result = resource.import_data(
+                dataset, use_transactions=True, rollback_on_validation_errors=True
+            )
+            if not (result.has_validation_errors() or result.has_errors()):
+                return redirect("odk_publish:app-user-list", odk_project_pk=odk_project_pk)
+            for row in result.invalid_rows:
+                for field, errors in row.error_dict.items():
+                    if field == "__all__":
+                        field = "id"
+                    for error in errors:
+                        messages.error(request, f"Row {row.number}, Column '{field}': {error}")
+            for row in result.error_rows:
+                for error in row.errors:
+                    messages.error(request, f"Row {row.number}: {error.error!r}")
+            for error in result.base_errors:
+                messages.error(request, repr(error.error))
+    else:
+        form = ImportForm(formats, [resource])
+    context = {
+        "breadcrumbs": Breadcrumbs.from_items(
+            request=request,
+            items=[("App Users", "app-user-list"), ("Import", "app-users-import")],
+        ),
+        "form": form,
+    }
+    return render(request, "odk_publish/app_user_import.html", context)

config/templates/odk_publish/app_user_export.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% block content %}
+    <div class="mb-4 grid gap-4 sm:grid-cols-2 md:mb-8 lg:grid-cols-3 xl:grid-cols-4">
+        {{ form.media }}
+        <form method="post" class="max-w-2xl">
+            {% csrf_token %}
+            <div class="grid gap-4 sm:grid-cols-1 sm:gap-6">
+                <div id="id_format_container">
+                    {{ form.format.label_tag }}
+                    {{ form.format }}
+                    {{ form.format.errors }}
+                </div>
+                <div class="flex items-center space-x-4">
+                    <button type="submit" class="btn btn-outline btn-primary">Submit</button>
+                    <a type="button"
+                       class="btn btn-outline"
+                       href="{% url 'odk_publish:app-user-list' request.odk_project.pk %}">Cancel</a>
+                </div>
+            </div>
+        </form>
+    </div>
+{% endblock content %}

config/templates/odk_publish/app_user_import.html

@@ -0,0 +1,45 @@
+{% extends "base.html" %}
+{% block content %}
+    <div class="mb-4 grid gap-4 sm:grid-cols-2 md:mb-8 lg:grid-cols-3 xl:grid-cols-4">
+        {{ form.media }}
+        <form method="post"
+              class="max-w-2xl"
+              enctype="multipart/form-data"
+              x-data="{ submitting: false }"
+              x-on:submit="submitting = true">
+            {% csrf_token %}
+            <div class="grid gap-4 sm:grid-cols-1 sm:gap-6">
+                {% for field in form.visible_fields %}
+                    <div id="id_import_file_container">
+                        {{ field.label_tag }}
+                        {{ field }}
+                        {{ field.errors }}
+                    </div>
+                {% endfor %}
+                <div class="flex items-center space-x-4">
+                    <button type="submit"
+                            :disabled="submitting"
+                            :class="{'btn-disabled': submitting}"
+                            class="btn btn-outline btn-primary">
+                        Submit
+                        <svg aria-hidden="true"
+                             role="status"
+                             x-cloak
+                             x-show="submitting"
+                             class="inline-flex items-center w-4.5 h-4.5 ml-1.5 -mr-1 -mt-0.5 me-3 text-gray-200 animate-spin dark:text-gray-600"
+                             viewBox="0 0 100 101"
+                             fill="none"
+                             xmlns="http://www.w3.org/2000/svg">
+                            <path d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z" fill="currentColor" />
+                            <path d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z" fill="#1C64F2" />
+                        </svg>
+                    </button>
+                    <a type="button"
+                       class="btn btn-outline"
+                       href="{% url 'odk_publish:app-user-list' request.odk_project.pk %}"
+                       :disabled="submitting">Cancel</a>
+                </div>
+            </div>
+        </form>
+    </div>
+{% endblock content %}

config/templates/odk_publish/app_user_list.html

@@ -36,6 +36,14 @@
                         <a href="{% url 'odk_publish:app-users-generate-qr-codes' request.odk_project.pk %}"
                            class="group inline-flex w-full items-center rounded-md px-3 py-2 text-sm text-gray-500 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-600 dark:hover:text-white">Regenerate QR Codes</a>
                     </li>
+                    <li>
+                        <a href="{% url 'odk_publish:app-users-export' request.odk_project.pk %}"
+                           class="group inline-flex w-full items-center rounded-md px-3 py-2 text-sm text-gray-500 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-600 dark:hover:text-white">Export</a>
+                    </li>
+                    <li>
+                        <a href="{% url 'odk_publish:app-users-import' request.odk_project.pk %}"
+                           class="group inline-flex w-full items-center rounded-md px-3 py-2 text-sm text-gray-500 hover:bg-gray-100 hover:text-gray-900 dark:text-gray-400 dark:hover:bg-gray-600 dark:hover:text-white">Import</a>
+                    </li>
                 </ul>
             </div>
         </div>

requirements/base/base.in

@@ -10,6 +10,7 @@ django-celery-beat
 django-email-bandit @ git+https://github.com/caktus/django-email-bandit@8d84bb5571e531946b46c63301b136f5369e149b
 django-filter
 django-htmx
+django-import-export[xls, xlsx]
 django-storages
 django-structlog[celery]
 django-tables2