notifications: fetch autoprune tags with multiple policies for image …

…expiry notification (PROJQUAY-8117) (quay#3340)

* notifications: fetch autoprune tags with multiple policies for image expiry notification(PROJQUAY-8117)

* don't fetch notifications if tags expiry is greater than notification days + add tests

data/model/autoprune.py

@@ -603,7 +603,9 @@ def prune_tags(tags, repo, namespace):
             )
 
 
-def fetch_tags_expiring_by_tag_count_policy(repo_id, policy_config, tag_page_limit=100):
+def fetch_tags_expiring_by_tag_count_policy(
+    repo_id, policy_config, tag_page_limit=100, exclude_tags=None
+):
     """
     Fetch tags in the given repository based on the number of tags specified in the policy config.
     """
@@ -625,6 +627,7 @@ def fetch_tags_expiring_by_tag_count_policy(repo_id, policy_config, tag_page_lim
             page,
             policy_config.get("tag_pattern"),
             policy_config.get("tag_pattern_matches"),
+            exclude_tags,
         )
         if len(tags) == 0:
             break
@@ -790,49 +793,50 @@ def execute_namespace_policies(
             break
 
 
-def fetch_tags_for_repo_policies(policies, repo_id):
+def fetch_tags_for_repo_policies(policies, repo_id, notification_config):
     all_tags = []
+    all_tag_names = set()
+    creation_date_tags = []
+
+    # first fetch by CREATION_DATE
     for policy in policies:
-        if policy.method == AutoPruneMethod.NUMBER_OF_TAGS.value:
-            tags = fetch_tags_expiring_by_tag_count_policy(repo_id, policy.config)
-        elif policy.method == AutoPruneMethod.CREATION_DATE.value:
-            tags = fetch_tags_expiring_by_creation_date_policy(repo_id, policy.config)
-        if len(tags):
-            all_tags.extend(tags)
-    return all_tags
+        if policy.method != AutoPruneMethod.CREATION_DATE.value:
+            continue
 
+        # skip policies that have expiry greater that notification's configuration
+        if convert_to_timedelta(policy.config["value"]).days > notification_config["days"]:
+            continue
 
-def fetch_tags_for_namespace_policies(ns_policies, namespace_id):
-    page_token = None
-    all_tags = []
-    while True:
-        repos, page_token = get_paginated_repositories_for_namespace(namespace_id, page_token)
-        for repo in repos:
+        tags = fetch_tags_expiring_by_creation_date_policy(repo_id, policy.config)
+        if len(tags) < 1:
+            continue
 
-            for ns_policy in ns_policies:
-                repo_policies = get_repository_autoprune_policies_by_repo_id(repo.id)
-                repo_tags = fetch_tags_for_repo_policies(repo_policies, repo.id)
-                if len(repo_tags):
-                    all_tags.extend(repo_tags)
+        for tag in tags:
+            if tag.name not in all_tag_names:
+                all_tags.append(tag)
+                all_tag_names.add(tag.name)
+                creation_date_tags.append(tag)
 
-                namespace_tags = fetch_tags_for_repo_policies([ns_policy], repo.id)
-                if len(namespace_tags):
-                    all_tags.extend(namespace_tags)
-        if page_token is None:
-            break
+    # then fetch by NUMBER_OF_TAGS
+    for policy in policies:
+        if policy.method != AutoPruneMethod.NUMBER_OF_TAGS.value:
+            continue
+        tags = fetch_tags_expiring_by_tag_count_policy(
+            repo_id, policy.config, tag_page_limit=100, exclude_tags=creation_date_tags
+        )
+        if len(tags) < 1:
+            continue
+
+        for tag in tags:
+            if tag.name not in all_tag_names:
+                all_tags.append(tag)
+                all_tag_names.add(tag.name)
 
     return all_tags
 
 
-def fetch_tags_expiring_due_to_auto_prune_policies(repo_id, namespace_id):
-    all_tags = []
-    repo_policies = get_repository_autoprune_policies_by_repo_id(repo_id)
-    repo_tags = fetch_tags_for_repo_policies(repo_policies, repo_id)
-    if len(repo_tags):
-        all_tags.extend(repo_tags)
-
-    namespace_policies = get_namespace_autoprune_policies_by_id(namespace_id)
-    namespace_tags = fetch_tags_for_namespace_policies(namespace_policies, namespace_id)
-    if len(namespace_tags):
-        all_tags.extend(namespace_tags)
-    return all_tags
+def fetch_tags_expiring_due_to_auto_prune_policies(repo_id, namespace_id, notification_config):
+    all_policies = []
+    all_policies.extend(get_namespace_autoprune_policies_by_id(namespace_id))
+    all_policies.extend(get_repository_autoprune_policies_by_repo_id(repo_id))
+    return fetch_tags_for_repo_policies(all_policies, repo_id, notification_config)

data/model/oci/tag.py

@@ -781,7 +781,13 @@ def reset_child_manifest_expiration(repository_id, manifest, expiration=None):
 
 
 def fetch_paginated_autoprune_repo_tags_by_number(
-    repo_id, max_tags_allowed: int, items_per_page, page, tag_pattern=None, tag_pattern_matches=True
+    repo_id,
+    max_tags_allowed: int,
+    items_per_page,
+    page,
+    tag_pattern=None,
+    tag_pattern_matches=True,
+    exclude_tags=None,
 ):
     """
     Fetch repository's active tags sorted by creation date & are more than max_tags_allowed
@@ -790,7 +796,7 @@ def fetch_paginated_autoprune_repo_tags_by_number(
         tags_offset = max_tags_allowed + ((page - 1) * items_per_page)
         now_ms = get_epoch_timestamp_ms()
         query = (
-            Tag.select(Tag.name).where(
+            Tag.select(Tag.id, Tag.name).where(
                 Tag.repository_id == repo_id,
                 (Tag.lifetime_end_ms >> None) | (Tag.lifetime_end_ms > now_ms),
                 Tag.hidden == False,
@@ -800,6 +806,9 @@ def fetch_paginated_autoprune_repo_tags_by_number(
             .order_by(Tag.lifetime_start_ms.desc())  # type: ignore[func-returns-value]
         )
 
+        if exclude_tags and len(exclude_tags) > 0:
+            query.where(Tag.name.not_in([tag.name for tag in exclude_tags]))
+
         if tag_pattern is not None:
             query = db_regex_search(
                 Tag.select(query.c.name).from_(query),
@@ -832,7 +841,7 @@ def fetch_paginated_autoprune_repo_tags_older_than_ms(
     try:
         tags_offset = items_per_page * (page - 1)
         now_ms = get_epoch_timestamp_ms()
-        query = Tag.select(Tag.name).where(
+        query = Tag.select(Tag.id, Tag.name).where(
             Tag.repository_id == repo_id,
             (Tag.lifetime_end_ms >> None) | (Tag.lifetime_end_ms > now_ms),
             (now_ms - Tag.lifetime_start_ms) > tag_lifetime_ms,