Distinguish between the nature of images

closes #1437

CHANGES/1437.feature

@@ -0,0 +1,4 @@
+Incorporated a notion of container images' characteristics. Users can now filter manifests by their
+nature using the ``is_flatpak`` or ``is_bootable`` field on the corresponding Manifest endpoint.
+In addition to that, manifest's annotations and configuration labels were exposed on the same
+endpoint too.

pulp_container/app/management/commands/init-image-nature.py

@@ -0,0 +1,61 @@
+from json.decoder import JSONDecodeError
+
+from gettext import gettext as _
+
+from contextlib import suppress
+
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.management import BaseCommand
+from django.core.paginator import Paginator
+
+from pulp_container.app.models import Manifest
+
+from pulp_container.constants import MEDIA_TYPE
+
+PAGE_CHUNK_SIZE = 1000
+
+
+class Command(BaseCommand):
+    """
+    A django management command to initialize flags describing the image nature.
+
+    Manifests stored inside Pulp are of various natures. The nature of an image can be determined
+    from JSON-formatted image manifest annotations or image configuration labels. These data are
+    stored inside artifacts.
+
+    This command reads data from the storage backend and populates the 'annotations', 'labels',
+    'is_bootable', and 'is_flatpak' fields on the Manifest model.
+    """
+
+    help = _(__doc__)
+
+    def handle(self, *args, **options):
+        manifests = Manifest.objects.exclude(
+            media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]
+        ).order_by("pulp_id")
+        self.update_manifests(manifests)
+
+        manifest_lists = Manifest.objects.filter(
+            media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI]
+        ).order_by("pulp_id")
+        self.update_manifests(manifest_lists)
+
+    def update_manifests(self, manifests_qs):
+        paginator = Paginator(manifests_qs, PAGE_CHUNK_SIZE)
+        for page_num in paginator.page_range:
+            manifests_to_update = []
+
+            page = paginator.page(page_num)
+            for manifest in page.object_list:
+                # suppress non-existing/already migrated artifacts and corrupted JSON files
+                with suppress(ObjectDoesNotExist, JSONDecodeError):
+                    has_metadata = manifest.init_metadata()
+                    if has_metadata:
+                        manifests_to_update.append(manifest)
+
+            if manifests_to_update:
+                fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak"]
+                manifests_qs.model.objects.bulk_update(
+                    manifests_to_update,
+                    fields_to_update,
+                )

pulp_container/app/migrations/0038_add_manifest_metadata_fields.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.2.10 on 2024-02-29 16:04
+import warnings
+
+from django.db import migrations, models
+
+
+def print_warning_for_initializing_image_nature(apps, schema_editor):
+    warnings.warn(
+        "Run 'pulpcore-manager init-image-nature' to initialize metadata (i.e., annotations "
+        "and labels) for all manifests. The metadata are stored in associated artifacts and "
+        "their contents should be accessible through the Manifest model now."
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0037_create_pull_through_cache_models'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='manifest',
+            name='annotations',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='is_bootable',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='is_flatpak',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='manifest',
+            name='labels',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.RunPython(
+            code=print_warning_for_initializing_image_nature,
+            reverse_code=migrations.RunPython.noop,
+            elidable=True,
+        )
+    ]

pulp_container/app/models.py

@@ -31,6 +31,7 @@
 
 
 from . import downloaders
+from pulp_container.app.utils import get_content_data
 from pulp_container.constants import MEDIA_TYPE, SIGNATURE_TYPE
 
 
@@ -71,6 +72,10 @@ class Manifest(Content):
         digest (models.TextField): The manifest digest.
         schema_version (models.IntegerField): The manifest schema version.
         media_type (models.TextField): The manifest media type.
+        annotations (models.JSONField): Metadata stored inside the image manifest.
+        labels (models.JSONField): Metadata stored inside the image configuration.
+        is_bootable (models.BooleanField): Indicates whether the image is bootable or not.
+        is_flatpak (models.BooleanField): Indicates whether the image is a flatpak package or not.
 
     Relations:
         blobs (models.ManyToManyField): Many-to-many relationship with Blob.
@@ -94,6 +99,12 @@ class Manifest(Content):
     schema_version = models.IntegerField()
     media_type = models.TextField(choices=MANIFEST_CHOICES)
 
+    annotations = models.JSONField(default=dict)
+    labels = models.JSONField(default=dict)
+
+    is_bootable = models.BooleanField(default=False)
+    is_flatpak = models.BooleanField(default=False)
+
     blobs = models.ManyToManyField(Blob, through="BlobManifest")
     config_blob = models.ForeignKey(
         Blob, related_name="config_blob", null=True, on_delete=models.CASCADE
@@ -107,6 +118,71 @@ class Manifest(Content):
         through_fields=("image_manifest", "manifest_list"),
     )
 
+    def init_metadata(self, manifest_data=None):
+        has_annotations = self.init_annotations(manifest_data)
+        has_labels = self.init_labels()
+        has_image_nature = self.init_image_nature()
+        return has_annotations or has_labels or has_image_nature
+
+    def init_annotations(self, manifest_data=None):
+        if manifest_data is None:
+            manifest_artifact = self._artifacts.get()
+            manifest_data, _ = get_content_data(manifest_artifact)
+
+        self.annotations = manifest_data.get("annotations", {})
+
+        return bool(self.annotations)
+
+    def init_labels(self):
+        if self.config_blob:
+            config_artifact = self.config_blob._artifacts.get()
+
+            config_data, _ = get_content_data(config_artifact)
+            self.labels = config_data.get("config", {}).get("Labels") or {}
+
+        return bool(self.labels)
+
+    def init_image_nature(self):
+        if self.media_type in [MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_LIST]:
+            return self.init_manifest_list_nature()
+        else:
+            return self.init_manifest_nature()
+
+    def init_manifest_list_nature(self):
+        for manifest in self.listed_manifests.all():
+            # it suffices just to have a single manifest of a specific nature;
+            # there is no case where the manifest is both bootable and flatpak-based
+            if manifest.is_bootable:
+                self.is_bootable = True
+                return True
+            elif manifest.is_flatpak:
+                self.is_flatpak = True
+                return True
+
+        return False
+
+    def init_manifest_nature(self):
+        if self.is_bootable_image():
+            self.is_bootable = True
+            return True
+        elif self.is_flatpak_image():
+            self.is_flatpak = True
+            return True
+        else:
+            return False
+
+    def is_bootable_image(self):
+        if (
+            self.annotations.get("containers.bootc") == "1"
+            or self.labels.get("containers.bootc") == "1"
+        ):
+            return True
+        else:
+            return False
+
+    def is_flatpak_image(self):
+        return True if self.labels.get("org.flatpak.ref") else False
+
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("digest",)

pulp_container/app/registry.py

@@ -460,6 +460,9 @@ async def init_pending_content(self, digest, manifest_data, media_type, artifact
             media_type=media_type,
             config_blob=config_blob,
         )
+
+        await sync_to_async(manifest.init_metadata)(manifest_data=manifest_data)
+
         try:
             await manifest.asave()
         except IntegrityError:

pulp_container/app/registry_api.py

@@ -1159,7 +1159,8 @@ def put(self, request, path, pk=None):
             if (len(manifests) - found_manifests.count()) != 0:
                 ManifestInvalid(digest=manifest_digest)
 
-            manifest_list = self._save_manifest(artifact, manifest_digest, media_type)
+            manifest_list = self._init_manifest(manifest_digest, media_type)
+            manifest_list = self._save_manifest(manifest_list, artifact)
 
             manifests_to_list = []
             for manifest in found_manifests:
@@ -1181,6 +1182,11 @@ def put(self, request, path, pk=None):
             )
             manifest = manifest_list
 
+            # once relations for listed manifests are established, it is
+            # possible to initialize the nature of the manifest list
+            if manifest.init_manifest_list_nature():
+                manifest.save(update_fields=["is_bootable", "is_flatpak"])
+
             found_blobs = models.Blob.objects.filter(
                 digest__in=found_manifests.values_list("blobs__digest"),
                 pk__in=content_pks,
@@ -1233,9 +1239,11 @@ def put(self, request, path, pk=None):
             if (len(blobs) - found_blobs.count()) != 0:
                 raise ManifestInvalid(digest=manifest_digest)
 
-            manifest = self._save_manifest(
-                artifact, manifest_digest, media_type, found_config_blobs.first()
-            )
+            config_blob = found_config_blobs.first()
+            manifest = self._init_manifest(manifest_digest, media_type, config_blob)
+            manifest.init_metadata(manifest_data=content_data)
+
+            manifest = self._save_manifest(manifest, artifact)
 
             thru = []
             for blob in found_blobs:
@@ -1289,13 +1297,15 @@ def put(self, request, path, pk=None):
             repository.pending_manifests.add(manifest)
             return ManifestResponse(manifest, path, request, status=201)
 
-    def _save_manifest(self, artifact, manifest_digest, content_type, config_blob=None):
-        manifest = models.Manifest(
+    def _init_manifest(self, manifest_digest, media_type, config_blob=None):
+        return models.Manifest(
             digest=manifest_digest,
             schema_version=2,
-            media_type=content_type,
+            media_type=media_type,
             config_blob=config_blob,
         )
+
+    def _save_manifest(self, manifest, artifact):
         try:
             manifest.save()
         except IntegrityError:

pulp_container/app/serializers.py

@@ -84,6 +84,26 @@ class ManifestSerializer(SingleArtifactContentSerializer):
         queryset=models.Blob.objects.all(),
     )
 
+    annotations = serializers.JSONField(
+        read_only=True,
+        help_text=_("Property that contains arbitrary metadata stored inside the image manifest."),
+    )
+    labels = serializers.JSONField(
+        read_only=True,
+        help_text=_("Property describing metadata stored inside the image configuration"),
+    )
+
+    is_bootable = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=_("A boolean determining whether users can boot from an image or not."),
+    )
+    is_flatpak = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text=_("A boolean determining whether the image bundles a Flatpak application"),
+    )
+
     class Meta:
         fields = SingleArtifactContentSerializer.Meta.fields + (
             "digest",
@@ -92,6 +112,10 @@ class Meta:
             "listed_manifests",
             "config_blob",
             "blobs",
+            "annotations",
+            "labels",
+            "is_bootable",
+            "is_flatpak",
         )
         model = models.Manifest