Add the host_binaries feature

.devcontainer/devcontainer.json

@@ -29,7 +29,7 @@
     },
     "features": {
         // add in eternal history and other bash features
-        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1.0.0": {}
+        "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1": {}
     },
     // Create the config folder for the bash-config feature
     "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config",
@@ -42,7 +42,11 @@
         "--device=/dev/fuse"
     ],
     "mounts": [
-        "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached"
+        "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached",
+        // these two mounts allow us to pick up cache and config folders that have
+        // been symlinked out of the home directory
+        "source=/dls/science/users/,target=/dls/science/users,type=bind,consistency=cached",
+        "source=/scratch/,target=/scratch,type=bind,consistency=cached"
     ],
     // Mount the parent as /workspaces so we can pip install peers as editable
     "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind",

.github/workflows/_test.yml

@@ -51,7 +51,13 @@ jobs:
           pip-install: ".[dev]"
 
       - name: Run tests
-        run: tox -e tests
+        run: |
+          sudo apt update
+          sudo apt install -y software-properties-common
+          sudo add-apt-repository -y ppa:apptainer/ppa
+          sudo apt update
+          sudo apt install -y apptainer
+          tox -e tests
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

.gitignore

@@ -72,3 +72,6 @@ lockfiles/
 
 # Demo output path for vscode tasks
 demo-output
+
+# vscode workspaces are local only
+*.code-workspace

.vscode/tasks.json

@@ -42,6 +42,15 @@
             "command": "deploy-tools compare ${input:compare-mode} ${input:deploy-folder}",
             "problemMatcher": []
         },
+        {
+            "type": "shell",
+            "label": "Recreate tests sample output from demo_configuration",
+            "command": "tests/generate_samples.sh",
+            "options": {
+                "cwd": "${workspaceRoot}"
+            },
+            "problemMatcher": [],
+        },
     ],
     "inputs": [
         {

src/deploy_tools/app_builder.py

@@ -44,6 +44,9 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
             entrypoint_file = entrypoints_folder / entrypoint.name
 
             mounts = ",".join(chain(global_options.mounts, options.mounts)).strip()
+            host_binaries = " ".join(
+                chain(global_options.host_binaries, options.host_binaries)
+            ).strip()
 
             apptainer_args = f"{global_options.apptainer_args} {options.apptainer_args}"
             apptainer_args = apptainer_args.strip()
@@ -55,6 +58,7 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None:
 
             params = {
                 "mounts": mounts,
+                "host_binaries": host_binaries,
                 "apptainer_args": apptainer_args,
                 "relative_sif_file": relative_sif_file,
                 "command": command,

src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml

@@ -0,0 +1,54 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: edge-containers-cli
+  version: "0.1"
+  description: ec command line tool for kubernetes IOCs
+
+  env_vars:
+    # For host-binaries PR we will use the K8S backend - this will be changed to
+    # ARGOCD when we add in the BinaryApp feature (upcoming PR)
+    - name: EC_CLI_BACKEND
+      value: K8S
+    - name: EC_LOG_URL
+      value: "https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}*"
+    # The default configuration is for p47 for testing
+    - name: EC_TARGET
+      value: p47-beamline # K8S target
+      # value: p47-beamline/p47 # Argocd target
+    - name: EC_SERVICES_REPO
+      value: https://github.com/epics-containers/p47-services
+
+  applications:
+    - app_type: apptainer
+
+      container:
+        path: docker://ghcr.io/epics-containers/edge-containers-cli
+        version: "4.4.1"
+
+      entrypoints:
+        - name: ec
+          command: ec
+        # for debugging enter the container with bash shell
+        - name: ec-bash
+          command: bash
+
+      global_options:
+        mounts:
+          # places to get argocd and kubectl config from
+          - /dls/science/users/:/dls/science/users/
+          - /scratch:/scratch
+          - /dls_sw/apps:/dls_sw/apps
+        host_binaries:
+          - kubectl
+          - helm
+          - kubelogin
+          - klogout
+          - kustomize
+          - kubeseal
+          - kubectl-oidc_login
+
+    - app_type: shell
+      name: ec-login
+      script:
+        - argocd login argocd.diamond.ac.uk --grpc-web --sso

src/deploy_tools/models/apptainer_app.py

@@ -1,13 +1,37 @@
 from collections.abc import Sequence
-from typing import Literal
+from typing import Annotated, Literal
+
+from pydantic import Field
 
 from .parent import ParentModel
 
 
 class EntrypointOptions(ParentModel):
-    apptainer_args: str = ""
-    command_args: str = ""
-    mounts: Sequence[str] = []
+    apptainer_args: Annotated[
+        str,
+        Field(description="Apptainer arguments to pass when launching the container"),
+    ] = ""
+
+    command_args: Annotated[
+        str, Field(description="Arguments to pass to command entrypoint")
+    ] = ""
+
+    mounts: Annotated[
+        Sequence[str],
+        Field(
+            description="A list of mount points to add to the container in the form of "
+            "'host_path:container_path'"
+        ),
+    ] = []
+
+    host_binaries: Annotated[
+        Sequence[str],
+        Field(
+            description="A list of host binaries to mount into the container. "
+            "These are discovered on the host using the current PATH and are "
+            "mounted into the container at /usr/bin/<binary_name>."
+        ),
+    ] = []
 
 
 class Entrypoint(ParentModel):

src/deploy_tools/models/schemas/deployment.json

@@ -1,6 +1,6 @@
 {
   "$defs": {
-    "Apptainer": {
+    "ApptainerApp": {
       "additionalProperties": false,
       "description": "Represents an Apptainer application or set of applications for a single image.\n\nThis uses Apptainer to deploy a portable image of the desired container. Several\nentrypoints can then be specified to allow for multiple commands run on the same\ncontainer image.",
       "properties": {
@@ -24,7 +24,8 @@
           "default": {
             "apptainer_args": "",
             "command_args": "",
-            "mounts": []
+            "mounts": [],
+            "host_binaries": []
           }
         }
       },
@@ -33,7 +34,7 @@
         "container",
         "entrypoints"
       ],
-      "title": "Apptainer",
+      "title": "ApptainerApp",
       "type": "object"
     },
     "ContainerImage": {
@@ -98,7 +99,8 @@
           "default": {
             "apptainer_args": "",
             "command_args": "",
-            "mounts": []
+            "mounts": [],
+            "host_binaries": []
           }
         }
       },
@@ -113,21 +115,33 @@
       "properties": {
         "apptainer_args": {
           "default": "",
+          "description": "Apptainer arguments to pass when launching the container",
           "title": "Apptainer Args",
           "type": "string"
         },
         "command_args": {
           "default": "",
+          "description": "Arguments to pass to command entrypoint",
           "title": "Command Args",
           "type": "string"
         },
         "mounts": {
           "default": [],
+          "description": "A list of mount points to add to the container in the form of 'host_path:container_path'",
           "items": {
             "type": "string"
           },
           "title": "Mounts",
           "type": "array"
+        },
+        "host_binaries": {
+          "default": [],
+          "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/<binary_name>.",
+          "items": {
+            "type": "string"
+          },
+          "title": "Host Binaries",
+          "type": "array"
         }
       },
       "title": "EntrypointOptions",
@@ -197,17 +211,17 @@
           "items": {
             "discriminator": {
               "mapping": {
-                "apptainer": "#/$defs/Apptainer",
-                "shell": "#/$defs/Shell"
+                "apptainer": "#/$defs/ApptainerApp",
+                "shell": "#/$defs/ShellApp"
               },
               "propertyName": "app_type"
             },
             "oneOf": [
               {
-                "$ref": "#/$defs/Apptainer"
+                "$ref": "#/$defs/ApptainerApp"
               },
               {
-                "$ref": "#/$defs/Shell"
+                "$ref": "#/$defs/ShellApp"
               }
             ]
           },
@@ -281,7 +295,7 @@
       },
       "type": "object"
     },
-    "Shell": {
+    "ShellApp": {
       "additionalProperties": false,
       "description": "Represents a Shell application.\n\nThis will run the code specified as a shell script. This currently uses Bash for\nimproved functionality while retaining high compatibility with various Linux\ndistributions.",
       "properties": {
@@ -307,7 +321,7 @@
         "name",
         "script"
       ],
-      "title": "Shell",
+      "title": "ShellApp",
       "type": "object"
     }
   },

src/deploy_tools/models/schemas/module.json

@@ -1,6 +1,6 @@
 {
   "$defs": {
-    "Apptainer": {
+    "ApptainerApp": {
       "additionalProperties": false,
       "description": "Represents an Apptainer application or set of applications for a single image.\n\nThis uses Apptainer to deploy a portable image of the desired container. Several\nentrypoints can then be specified to allow for multiple commands run on the same\ncontainer image.",
       "properties": {
@@ -24,7 +24,8 @@
           "default": {
             "apptainer_args": "",
             "command_args": "",
-            "mounts": []
+            "mounts": [],
+            "host_binaries": []
           }
         }
       },
@@ -33,7 +34,7 @@
         "container",
         "entrypoints"
       ],
-      "title": "Apptainer",
+      "title": "ApptainerApp",
       "type": "object"
     },
     "ContainerImage": {
@@ -80,7 +81,8 @@
           "default": {
             "apptainer_args": "",
             "command_args": "",
-            "mounts": []
+            "mounts": [],
+            "host_binaries": []
           }
         }
       },
@@ -95,21 +97,33 @@
       "properties": {
         "apptainer_args": {
           "default": "",
+          "description": "Apptainer arguments to pass when launching the container",
           "title": "Apptainer Args",
           "type": "string"
         },
         "command_args": {
           "default": "",
+          "description": "Arguments to pass to command entrypoint",
           "title": "Command Args",
           "type": "string"
         },
         "mounts": {
           "default": [],
+          "description": "A list of mount points to add to the container in the form of 'host_path:container_path'",
           "items": {
             "type": "string"
           },
           "title": "Mounts",
           "type": "array"
+        },
+        "host_binaries": {
+          "default": [],
+          "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/<binary_name>.",
+          "items": {
+            "type": "string"
+          },
+          "title": "Host Binaries",
+          "type": "array"
         }
       },
       "title": "EntrypointOptions",
@@ -179,17 +193,17 @@
           "items": {
             "discriminator": {
               "mapping": {
-                "apptainer": "#/$defs/Apptainer",
-                "shell": "#/$defs/Shell"
+                "apptainer": "#/$defs/ApptainerApp",
+                "shell": "#/$defs/ShellApp"
               },
               "propertyName": "app_type"
             },
             "oneOf": [
               {
-                "$ref": "#/$defs/Apptainer"
+                "$ref": "#/$defs/ApptainerApp"
               },
               {
-                "$ref": "#/$defs/Shell"
+                "$ref": "#/$defs/ShellApp"
               }
             ]
           },
@@ -232,7 +246,7 @@
       "title": "ModuleDependency",
       "type": "object"
     },
-    "Shell": {
+    "ShellApp": {
       "additionalProperties": false,
       "description": "Represents a Shell application.\n\nThis will run the code specified as a shell script. This currently uses Bash for\nimproved functionality while retaining high compatibility with various Linux\ndistributions.",
       "properties": {
@@ -258,7 +272,7 @@
         "name",
         "script"
       ],
-      "title": "Shell",
+      "title": "ShellApp",
       "type": "object"
     }
   },