Working on Runtime Python Builder, Update readme and application.yml

README.md

@@ -16,16 +16,16 @@ DH Core offers an API interface that give us the possibility to formalize the en
 3. **Tasks (TASK)** are the logical representations of an action which is performed via a given function, by a runtime, within a framework. As such, they define both the actual scope of the execution and its context, in terms of environment, dependencies, resources etc.
 4. **Runs (RUN)** are the representation of the execution of a given task with the given function, on (a set of) inputs to deliver (a set of) outputs. At a high level, they can be seen both as a summary of the union between a function and a task in a single instance, and as the representation of the actual execution phase, with parameters, status, results etc.
 
-5. Dataitems
-6. Artifacts
-7. Models
-8. Workflows
+5. **Dataitems**
+6. **Artifacts**
+7. **Models**
+8. **Workflows**
 
 The project is essentially divided into the following parts:
 
-1. Core
-2. Modules
-3. Frontend
+1. **Core**
+2. **Modules**
+3. **Frontend**
 
 Core manages all the basic elements that are the foundation of the project.
 Modules include:
@@ -46,7 +46,7 @@ Modules include:
 
 ## Configuration
 
-You can locate the project configuration file in the **application [core]** module, named **_resources/application.yml_**. This file specifies configurations for various sections. 
+You can locate the project configuration file in the **application [core]** module, named **_resources/application.yml_**. This file specifies configurations for various sections.
 
 Each section allows configurations to be written directly in the YAML file or through environment variables (ENV).
 
@@ -231,9 +231,9 @@ To start the database for Core using Docker, execute the following command:
 ```bash
 docker run -d --name <dbname> -p 5434:5432 -e POSTGRES_PASSWORD=<password> postgres
 ```
-Once the database container is started, you can launch your application. 
+Once the database container is started, you can launch your application.
 
-It's important to configure the database parameters either via ENV variables or directly in the application.yml file. You can find those settings in the SpringBoot configuration section  
+It's important to configure the database parameters either via ENV variables or directly in the application.yml file. You can find those settings in the SpringBoot configuration section
 
 **IMPORTANT:** If no database is created locally or through Docker, an H2 database will be automatically created when the application starts.
 
@@ -243,12 +243,12 @@ To run core there are different possibilities:
 1. You can download directly the image from GitHub and run it
 ```bash
 docker pull ghcr.io/scc-digitalhub/digitalhub-core:<version>
-docker run digitalhub-core:<version>
+docker run ghcr.io/scc-digitalhub/digitalhub-core/digitalhub-core:<version>
 
 ```
 2. You can build your own image and run it
 ```bash
-docker build -t ghcr.io/scc-digitalhub/digitalhub-core:<version> .
+docker build -t digitalhub-core:<version> .
 docker run digitalhub-core:<version>
 ```
 
@@ -264,8 +264,8 @@ git sumodule update
 # Run the project
 mvn spring-boot:run -pl application
 ```
-In case of problem once you have updated all submodules you can clean and install all dependencies 
-running the following command 
+In case of problem once you have updated all submodules you can clean and install all dependencies
+running the following command
 ```bash
 mvn clean install -DskipTests 
 ```

application/src/main/resources/application.yml

@@ -67,7 +67,10 @@ runtime:
   kfp:
     image: ${RUNTIME_KFP_IMAGE:ghcr.io/scc-digitalhub/digitalhub-core-wrapper-kfp:latest}
   python:
-    image: ${RUNTIME_PYTHON_IMAGE:ghcr.io/scc-digitalhub/digitalhub-serverless/python-runtime-3.9:latest}
+    images:
+      python-3.9: ${RUNTIME_PYTHON_IMAGE_3_9:ghcr.io/scc-digitalhub/digitalhub-serverless/python-runtime-3.9:latest}
+      python-3.10: ${RUNTIME_PYTHON_IMAGE_3_10:ghcr.io/scc-digitalhub/digitalhub-serverless/python-runtime-3.10:latest}
+
     command: /usr/local/bin/processor
 
 

modules/runtime-container/src/main/java/it/smartcommunitylabdhub/runtime/container/runners/ContainerBuildRunner.java

@@ -50,7 +50,6 @@ public ContainerBuildRunner(FunctionContainerSpec functionContainerSpec, Map<Str
     public K8sKanikoRunnable produce(Run run) {
         RunContainerSpec runSpec = new RunContainerSpec(run.getSpec());
         TaskBuildSpec taskSpec = runSpec.getTaskBuildSpec();
-        StatusFieldAccessor statusFieldAccessor = StatusFieldAccessor.with(run.getStatus());
 
         List<CoreEnv> coreEnvList = new ArrayList<>(
             List.of(new CoreEnv("PROJECT_NAME", run.getProject()), new CoreEnv("RUN_ID", run.getId()))
@@ -110,7 +109,7 @@ public K8sKanikoRunnable produce(Run run) {
             // Task specific
             .dockerFile(dockerfile)
             // specific
-            .backoffLimit(1)
+            .backoffLimit(0)
             .build();
     }
 }

modules/runtime-python/src/main/java/it/smartcommunitylabdhub/runtime/python/PythonRuntime.java

@@ -28,12 +28,17 @@
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sRunnable;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sServeRunnable;
+import it.smartcommunitylabdhub.framework.kaniko.runnables.K8sKanikoRunnable;
+import it.smartcommunitylabdhub.runtime.python.builders.PythonBuildBuilder;
 import it.smartcommunitylabdhub.runtime.python.builders.PythonJobBuilder;
 import it.smartcommunitylabdhub.runtime.python.builders.PythonServeBuilder;
+import it.smartcommunitylabdhub.runtime.python.configs.PythonImage;
+import it.smartcommunitylabdhub.runtime.python.runners.PythonBuildRunner;
 import it.smartcommunitylabdhub.runtime.python.runners.PythonJobRunner;
 import it.smartcommunitylabdhub.runtime.python.runners.PythonServeRunner;
 import it.smartcommunitylabdhub.runtime.python.specs.function.PythonFunctionSpec;
 import it.smartcommunitylabdhub.runtime.python.specs.run.PythonRunSpec;
+import it.smartcommunitylabdhub.runtime.python.specs.task.PythonBuildTaskSpec;
 import it.smartcommunitylabdhub.runtime.python.specs.task.PythonJobTaskSpec;
 import it.smartcommunitylabdhub.runtime.python.specs.task.PythonServeTaskSpec;
 import it.smartcommunitylabdhub.runtime.python.status.PythonRunStatus;
@@ -63,6 +68,7 @@ public class PythonRuntime implements Runtime<PythonFunctionSpec, PythonRunSpec,
 
     private final PythonJobBuilder jobBuilder = new PythonJobBuilder();
     private final PythonServeBuilder serveBuilder = new PythonServeBuilder();
+    private final PythonBuildBuilder buildBuilder = new PythonBuildBuilder();
 
     @Autowired
     private SecretService secretService;
@@ -73,14 +79,17 @@ public class PythonRuntime implements Runtime<PythonFunctionSpec, PythonRunSpec,
     @Autowired(required = false)
     private RunnableStore<K8sServeRunnable> serveRunnableStore;
 
+    @Autowired(required = false)
+    private RunnableStore<K8sKanikoRunnable> buildRunnableStore;
+
     @Autowired
     private FunctionService functionService;
 
     @Autowired
     private LogService logService;
 
-    @Value("${runtime.python.image}")
-    private String image;
+    @Autowired
+    PythonImage images;
 
     @Value("${runtime.python.command}")
     private String command;
@@ -109,6 +118,10 @@ public PythonRunSpec build(@NotNull Executable function, @NotNull Task task, @No
                 PythonServeTaskSpec taskServeSpec = new PythonServeTaskSpec(task.getSpec());
                 return serveBuilder.build(funSpec, taskServeSpec, runSpec);
             }
+            case PythonBuildTaskSpec.KIND -> {
+                PythonBuildTaskSpec taskBuildSpec = new PythonBuildTaskSpec(task.getSpec());
+                return buildBuilder.build(funSpec, taskBuildSpec, runSpec);
+            }
             default -> throw new IllegalArgumentException(
                 "Kind not recognized. Cannot retrieve the right builder or specialize Spec for Run and Task."
             );
@@ -131,19 +144,26 @@ public RunRunnable run(@NotNull Run run) {
 
         return switch (runAccessor.getTask()) {
             case PythonJobTaskSpec.KIND -> new PythonJobRunner(
-                image,
+                images.getImage("python-" + runPythonSpec.getFunctionSpec().getPythonVersion().getVersion()),
                 command,
                 runPythonSpec.getFunctionSpec(),
                 secretService.groupSecrets(run.getProject(), runPythonSpec.getTaskJobSpec().getSecrets())
             )
                 .produce(run);
             case PythonServeTaskSpec.KIND -> new PythonServeRunner(
-                image,
+                images.getImage("python-" + runPythonSpec.getFunctionSpec().getPythonVersion().getVersion()),
                 command,
                 runPythonSpec.getFunctionSpec(),
                 secretService.groupSecrets(run.getProject(), runPythonSpec.getTaskJobSpec().getSecrets())
             )
                 .produce(run);
+            case PythonBuildTaskSpec.KIND -> new PythonBuildRunner(
+                images.getImage("python-" + runPythonSpec.getFunctionSpec().getPythonVersion().getVersion()),
+                command,
+                runPythonSpec.getFunctionSpec(),
+                secretService.groupSecrets(run.getProject(), runPythonSpec.getTaskBuildSpec().getSecrets())
+            )
+                .produce(run);
             default -> throw new IllegalArgumentException("Kind not recognized. Cannot retrieve the right Runner");
         };
     }
@@ -186,6 +206,17 @@ public RunRunnable stop(Run run) throws NoSuchEntityException {
                     }
                     throw new CoreRuntimeException("Store is not available");
                 }
+                case PythonBuildTaskSpec.KIND -> {
+                    if (jobRunnableStore != null) {
+                        K8sKanikoRunnable k8sKanikoRunnable = buildRunnableStore.find(run.getId());
+                        if (k8sKanikoRunnable == null) {
+                            throw new NoSuchEntityException("JobDeployment not found");
+                        }
+                        k8sKanikoRunnable.setState(State.STOP.name());
+                        yield k8sKanikoRunnable;
+                    }
+                    throw new CoreRuntimeException("Build Store is not available");
+                }
                 default -> throw new IllegalArgumentException("Kind not recognized. Cannot retrieve the right Runner");
             };
         } catch (StoreException e) {
@@ -222,6 +253,18 @@ public RunRunnable delete(Run run) throws NoSuchEntityException {
                     }
                     throw new CoreRuntimeException("Job Store is not available");
                 }
+                case PythonBuildTaskSpec.KIND -> {
+                    if (jobRunnableStore != null) {
+                        K8sKanikoRunnable k8sKanikoRunnable = buildRunnableStore.find(run.getId());
+                        if (k8sKanikoRunnable == null) {
+                            //not in store, either not existent or already removed, nothing to do
+                            yield null;
+                        }
+                        k8sKanikoRunnable.setState(State.DELETING.name());
+                        yield k8sKanikoRunnable;
+                    }
+                    throw new CoreRuntimeException("Build Store is not available");
+                }
                 case PythonServeTaskSpec.KIND -> {
                     if (jobRunnableStore != null) {
                         K8sServeRunnable k8sServeRunnable = serveRunnableStore.find(run.getId());

modules/runtime-python/src/main/java/it/smartcommunitylabdhub/runtime/python/builders/PythonBuildBuilder.java

@@ -0,0 +1,26 @@
+package it.smartcommunitylabdhub.runtime.python.builders;
+
+import it.smartcommunitylabdhub.commons.infrastructure.Builder;
+import it.smartcommunitylabdhub.runtime.python.specs.function.PythonFunctionSpec;
+import it.smartcommunitylabdhub.runtime.python.specs.run.PythonRunSpec;
+import it.smartcommunitylabdhub.runtime.python.specs.task.PythonBuildTaskSpec;
+import it.smartcommunitylabdhub.runtime.python.specs.task.PythonJobTaskSpec;
+
+import java.util.Optional;
+
+public class PythonBuildBuilder implements Builder<PythonFunctionSpec, PythonBuildTaskSpec, PythonRunSpec> {
+
+    @Override
+    public PythonRunSpec build(PythonFunctionSpec funSpec, PythonBuildTaskSpec taskSpec, PythonRunSpec runSpec) {
+        PythonRunSpec pythonSpec = new PythonRunSpec(runSpec.toMap());
+        pythonSpec.setTaskBuildSpec(taskSpec);
+        pythonSpec.setFunctionSpec(funSpec);
+
+        //let run override k8s specs
+        Optional
+                .ofNullable(runSpec.getTaskJobSpec())
+                .ifPresent(k8sSpec -> pythonSpec.getTaskJobSpec().configure(k8sSpec.toMap()));
+
+        return pythonSpec;
+    }
+}

modules/runtime-python/src/main/java/it/smartcommunitylabdhub/runtime/python/configs/PythonImage.java

@@ -0,0 +1,27 @@
+package it.smartcommunitylabdhub.runtime.python.configs;
+
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+
+
+@Component
+@ConfigurationProperties(prefix = "runtime.python")
+public class PythonImage {
+
+    private Map<String, String> images;
+
+    public void setImages(Map<String, String> images) {
+        this.images = images;
+    }
+
+    public Map<String, String> getImages() {
+        return images;
+    }
+
+    public String getImage(String version) {
+        return images.get(version);
+    }
+}

modules/runtime-python/src/main/java/it/smartcommunitylabdhub/runtime/python/model/PythonVersion.java

@@ -0,0 +1,23 @@
+package it.smartcommunitylabdhub.runtime.python.model;
+
+
+
+public enum PythonVersion {
+    PYTHON_3_9("3.9"),
+    PYTHON_3_10("3.10"),
+    PYTHON_3_11("3.11"),
+    PYTHON_3_12("3.12"),
+    PYTHON_3_13("3.13");
+
+    private final String version;
+
+    PythonVersion(String version) {
+        this.version = version;
+    }
+
+
+    public String getVersion() {
+        return version;
+    }
+
+}