From 090f1e06d21aaa60b136cb174b4bd5c030ecc306 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20Dutheillet-Lamonth=C3=A9zie?=
 <thomas.dutheillet@ng-analytics.com>
Date: Mon, 30 Sep 2024 10:00:46 +0200
Subject: [PATCH] #2: introduce !extend yaml tag to extend lists in config

---
 ci/config.yaml | 14 +++++++-------
 ci/util.py     | 36 ++++++++++++++++++++++++++----------
 2 files changed, 33 insertions(+), 17 deletions(-)

diff --git a/ci/config.yaml b/ci/config.yaml
index 61149dae..3c8bfaaf 100644
--- a/ci/config.yaml
+++ b/ci/config.yaml
@@ -65,7 +65,7 @@ setup:
       CXX: mpicxx
       GCOV: llvm-gcov
     deps:
-      packages: [ *apk-packages, clang, clang-dev ]
+      packages: !extend [ *apk-packages, clang, clang-dev ]
       mpich: ['3.3.2', '-j4', '', 'clang', 'clang++']
 
   #vt
@@ -77,8 +77,8 @@ setup:
       GCOV: gcov
     # deps and versions to find in vt
     deps:
-      packages: [ *apt-packages, gcc-9, g++-9]
-      cmake: *cmake-args
+      packages: !extend [*apt-packages, gcc-9, g++-9]
+      cmake: *cmake-args # 3.23.4 in vt but seems to be an error with vtk tiff build because error in Cmake CheckSize.
       openmpi: *openmpi-args
       zoltan: *zoltan-args
       mpich: *mpich-args
@@ -110,7 +110,7 @@ setup:
       CXX: g++-13
       GCOV: gcov
     deps:
-      packages: [ *apt-packages, gcc-13, g++-13 ]
+      packages: !extend [ *apt-packages, gcc-13, g++-13 ]
       mesa: ~
       conda: ~
       conda-python-env: *conda-py3-all
@@ -124,7 +124,7 @@ setup:
       CXX: g++-12
       GCOV: gcov
     deps:
-      packages: [ *apt-packages, gcc-12, g++-12 ]
+      packages: !extend [ *apt-packages, gcc-12, g++-12 ]
       mesa: ~
       conda: ~
       conda-python-env: *conda-py3-all
@@ -138,7 +138,7 @@ setup:
       CXX: g++-11
       GCOV: gcov
     deps:
-      packages: [*apt-packages, gcc-11, g++-11 ]
+      packages: !extend [*apt-packages, gcc-11, g++-11 ]
       mesa: ~
       conda: ~
       conda-python-env: *conda-py3-all
@@ -152,7 +152,7 @@ setup:
       CXX: g++-11
       GCOV: gcov
     deps:
-      packages: [*apt-packages, gcc-11, g++-11 ]
+      packages: !extend [*apt-packages, gcc-11, g++-11 ]
       mesa: ~
       conda: ~
       conda-python-env: *conda-py3-all
diff --git a/ci/util.py b/ci/util.py
index a46e293f..645aac66 100644
--- a/ci/util.py
+++ b/ci/util.py
@@ -1,8 +1,34 @@
 """Utility methods
 """
 
+import copy
 from typing import Union
 
+import yaml
+
+
+# < YAML custom tags
+def yaml_list_extend_loader(loader, node):
+    """Loads a YAML sequence node defining a list to extend and the elements to add"""
+    if not isinstance(node, yaml.SequenceNode):
+        raise RuntimeError("!extend only applicable to sequences")
+    params = loader.construct_sequence(node)
+    if not isinstance(params[0], list):
+        raise RuntimeError("!extend first argument must be a list")
+
+    l: list = copy.deepcopy(params[0])
+    if len(params) > 1:
+        for p in params[1:]:
+            if not isinstance(p, list):
+                l.extend([p])
+            else:
+                l.extend(p)
+    return l
+
+# Required for safe_load
+yaml.SafeLoader.add_constructor("!extend", yaml_list_extend_loader)
+# > YAML custom tags
+
 def resolve_conf(config: Union[dict,list]) -> dict:
     """Update configuration to ease its processing:
     - Turn images list into a dict (images indexed by `repo:tag`)
@@ -29,16 +55,6 @@ def resolve_conf(config: Union[dict,list]) -> dict:
             if not isinstance(args, list):
                 setup["deps"][dep_id] = [ args ]
 
-        # flatten package dependencies
-        if setup["deps"].get("packages") is not None:
-            flattened_args = []
-            for (_, arg) in enumerate(setup["deps"].get("packages")):
-                if isinstance(arg, list):
-                    flattened_args.extend(arg)
-                else:
-                    flattened_args.append(arg)
-            setup["deps"]["packages"] = flattened_args   
-
     # Index Docker images by their full name
     config["images"] = dict((image.get("repository") + ":" + image.get("tag"), image)
                 for image in config.get("images"))