Distinguish between the nature of images

closes #1437

CHANGES/1437.feature

@@ -0,0 +1,2 @@
+Incorporated a notion of container images' characteristics. Users can now filter manifests by their
+nature using the ``is_flatpak`` or ``is_bootable`` field on the corresponding Manifest endpoint.

pulp_container/app/migrations/0038_add_manifest_metadata_fields.py

@@ -0,0 +1,148 @@
+# Generated by Django 4.2.10 on 2024-02-29 16:04
+
+import json
+
+from contextlib import suppress
+
+from django.db import migrations, models
+from django.core.paginator import Paginator
+from django.core.exceptions import ObjectDoesNotExist
+
+from pulp_container.constants import MEDIA_TYPE
+
+PAGE_CHUNK_SIZE = 200
+
+
+def populate_annotations_and_labels(apps, schema_editor):
+    Manifest = apps.get_model('container', 'Manifest')
+
+    manifests = Manifest.objects.exclude(
+        media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]
+    ).order_by('pulp_id')
+    update_manifests(manifests)
+
+    manifest_lists = Manifest.objects.filter(
+        media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]
+    ).order_by('pulp_id')
+    update_manifests(manifest_lists)
+
+
+def update_manifests(manifests_qs):
+    paginator = Paginator(manifests_qs, PAGE_CHUNK_SIZE)
+    for page_num in paginator.page_range:
+
+        manifests_to_update = []
+        page = paginator.page(page_num)
+        for obj in page.object_list:
+            if m := update_manifest(obj):
+                manifests_to_update.append(m)
+
+        if manifests_to_update:
+            fields_to_update = ['annotations', 'labels', 'is_bootable', 'is_flatpak']
+            manifests_qs.model.objects.bulk_update(
+                manifests_to_update,
+                fields_to_update,
+            )
+
+
+def update_manifest(manifest):
+    with suppress(ObjectDoesNotExist):
+        artifact = manifest._artifacts.get()
+        # methods for initializing manifest's metadata are not available at the time of
+        # running the migration; therefore, the methods are taken from the Model itself
+        init_metadata(manifest, artifact)
+        return manifest
+
+
+def init_metadata(manifest, manifest_artifact):
+    init_annotations(manifest, manifest_artifact)
+    init_labels(manifest)
+    init_image_nature(manifest)
+
+
+def get_content_data(saved_artifact):
+    raw_data = saved_artifact.file.read()
+    content_data = json.loads(raw_data)
+    saved_artifact.file.close()
+    return content_data
+
+
+def init_annotations(manifest, manifest_artifact):
+    manifest_data = get_content_data(manifest_artifact)
+    manifest.annotations = manifest_data.get('annotations', {})
+
+
+def init_labels(manifest):
+    if manifest.config_blob:
+        config_artifact = manifest.config_blob._artifacts.get()
+        config_data = get_content_data(config_artifact)
+        manifest.labels = config_data.get('config', {}).get('Labels') or {}
+
+
+def init_image_nature(manifest):
+    if manifest.media_type in [MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_LIST]:
+        init_manifest_list_nature(manifest)
+    else:
+        init_manifest_nature(manifest)
+
+
+def init_manifest_list_nature(manifest):
+    for m in manifest.listed_manifests.all():
+        if m.is_bootable:
+            manifest.is_bootable = True
+            break
+        elif m.is_flatpak:
+            manifest.is_flatpak = True
+            break
+
+
+def init_manifest_nature(manifest):
+    if is_bootable_image(manifest.annotations, manifest.labels):
+        manifest.is_bootable = True
+    elif is_flatpak_image(manifest.labels):
+        manifest.is_flatpak = True
+
+
+def is_bootable_image(annotations, labels):
+    if annotations.get('containers.bootc') == '1' or labels.get('containers.bootc') == '1':
+        return True
+    else:
+        return False
+
+
+def is_flatpak_image(labels):
+    return True if labels.get('org.flatpak.ref') else False
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0037_create_pull_through_cache_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='manifest',
+            name='annotations',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='is_bootable',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='is_flatpak',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='labels',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.RunPython(
+            code=populate_annotations_and_labels,
+            reverse_code=migrations.RunPython.noop,
+        )
+    ]

pulp_container/app/models.py

@@ -31,6 +31,7 @@
 
 
 from . import downloaders
+from pulp_container.app.utils import get_content_data
 from pulp_container.constants import MEDIA_TYPE, SIGNATURE_TYPE
 
 
@@ -99,6 +100,12 @@ class Manifest(Content):
         Blob, related_name="config_blob", null=True, on_delete=models.CASCADE
     )
 
+    annotations = models.JSONField(default=dict)
+    labels = models.JSONField(default=dict)
+
+    is_bootable = models.BooleanField(default=False)
+    is_flatpak = models.BooleanField(default=False)
+
     # Order matters for through fields, (source, target)
     listed_manifests = models.ManyToManyField(
         "self",
@@ -107,6 +114,56 @@ class Manifest(Content):
         through_fields=("image_manifest", "manifest_list"),
     )
 
+    def init_metadata(self, manifest_artifact):
+        self.init_annotations(manifest_artifact)
+        self.init_labels()
+        self.init_image_nature()
+
+    def init_annotations(self, manifest_artifact):
+        manifest_data, _ = get_content_data(manifest_artifact)
+        self.annotations = manifest_data.get("annotations", {})
+
+    def init_labels(self):
+        if self.config_blob:
+            config_artifact = self.config_blob._artifacts.get()
+            config_data, _ = get_content_data(config_artifact)
+            self.labels = config_data.get("config", {}).get("Labels") or {}
+
+    def init_image_nature(self):
+        if self.media_type in [MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_LIST]:
+            self.init_manifest_list_nature()
+        else:
+            self.init_manifest_nature()
+
+    def init_manifest_list_nature(self):
+        for manifest in self.listed_manifests.all():
+            # it suffices just to have a single manifest of a specific nature;
+            # there is no case where the manifest is both bootable and flatpak-based
+            if manifest.is_bootable:
+                self.is_bootable = True
+                break
+            elif manifest.is_flatpak:
+                self.is_flatpak = True
+                break
+
+    def init_manifest_nature(self):
+        if self.is_bootable_image():
+            self.is_bootable = True
+        elif self.is_flatpak_image():
+            self.is_flatpak = True
+
+    def is_bootable_image(self):
+        if (
+            self.annotations.get("containers.bootc") == "1"
+            or self.labels.get("containers.bootc") == "1"
+        ):
+            return True
+        else:
+            return False
+
+    def is_flatpak_image(self):
+        return True if self.labels.get("org.flatpak.ref") else False
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("digest",)

pulp_container/app/registry.py

@@ -460,6 +460,9 @@ async def init_pending_content(self, digest, manifest_data, media_type, artifact
             media_type=media_type,
             config_blob=config_blob,
         )
+
+        await sync_to_async(manifest.init_metadata)(artifact)
+
         try:
             await manifest.asave()
         except IntegrityError:

pulp_container/app/registry_api.py

@@ -1181,6 +1181,11 @@ def put(self, request, path, pk=None):
             )
             manifest = manifest_list
 
+            # once relations for listed manifests are established,
+            # it is possible to initialize the metadata, like labels
+            manifest.init_metadata(artifact)
+            manifest.save(update_fields=["annotations", "labels", "is_bootable", "is_flatpak"])
+
             found_blobs = models.Blob.objects.filter(
                 digest__in=found_manifests.values_list("blobs__digest"),
                 pk__in=content_pks,
@@ -1236,6 +1241,8 @@ def put(self, request, path, pk=None):
             manifest = self._save_manifest(
                 artifact, manifest_digest, media_type, found_config_blobs.first()
             )
+            manifest.init_metadata(artifact)
+            manifest.save(update_fields=["annotations", "labels", "is_bootable", "is_flatpak"])
 
             thru = []
             for blob in found_blobs:
@@ -1296,6 +1303,7 @@ def _save_manifest(self, artifact, manifest_digest, content_type, config_blob=No
             media_type=content_type,
             config_blob=config_blob,
         )
+
         try:
             manifest.save()
         except IntegrityError:

pulp_container/app/serializers.py

@@ -3,6 +3,7 @@
 import re
 
 from django.core.validators import URLValidator
+from drf_spectacular.utils import extend_schema_serializer
 from rest_framework import serializers
 
 from pulpcore.plugin.models import (
@@ -84,6 +85,17 @@ class ManifestSerializer(SingleArtifactContentSerializer):
         queryset=models.Blob.objects.all(),
     )
 
+    is_bootable = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=_("A boolean determining whether users can boot from an image or not."),
+    )
+    is_flatpak = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=_("A boolean determining whether the image bundles a Flatpak application"),
+    )
+
     class Meta:
         fields = SingleArtifactContentSerializer.Meta.fields + (
             "digest",
@@ -92,6 +104,31 @@ class Meta:
             "listed_manifests",
             "config_blob",
             "blobs",
+            "is_bootable",
+            "is_flatpak",
+        )
+        model = models.Manifest
+
+
+@extend_schema_serializer(component_name="container.ManifestDetailResponse")
+class ManifestDetailSerializer(ManifestSerializer):
+    """
+    A detailed serializer for displaying manifest's annotations and labels.
+    """
+
+    annotations = serializers.JSONField(
+        read_only=True,
+        help_text=_("Property that contains arbitrary metadata stored inside the image manifest."),
+    )
+    labels = serializers.JSONField(
+        read_only=True,
+        help_text=_("Property describing metadata stored inside the image configuration"),
+    )
+
+    class Meta:
+        fields = ManifestSerializer.Meta.fields + (
+            "annotations",
+            "labels",
         )
         model = models.Manifest
 

pulp_container/app/tasks/download_image_data.py

@@ -64,7 +64,7 @@ async def run(self):
                 self.manifest_list_dcs.append(list_dc)
         else:
             # Simple tagged manifest
-            man_dc = self.create_tagged_manifest(
+            man_dc = await self.create_tagged_manifest(
                 self.tag_name, self.manifest_artifact, content_data, raw_data, media_type
             )
             tag_dc.extra_data["tagged_manifest_dc"] = man_dc