feat: add k8s runtime/priority classes

application/src/main/java/it/smartcommunitylabdhub/core/components/infrastructure/factories/specs/SpecRegistryImpl.java

@@ -176,4 +176,29 @@ public void validateSpec(Spec spec) throws MethodArgumentNotValidException, Ille
             }
         }
     }
+
+    public void refresh() {
+        //refresh schemas for collected specs
+        specs
+            .entrySet()
+            .forEach(e -> {
+                String kind = e.getKey();
+                Class<? extends Spec> spec = e.getValue();
+                SpecType type = specTypes.get(kind);
+                if (type != null) {
+                    EntityName entity = type.entity();
+                    log.debug("generate schema for spec {}:{} ", entity, kind);
+                    SchemaImplBuilder builder = SchemaImpl
+                        .builder()
+                        .entity(entity)
+                        .kind(kind)
+                        .schema(SchemaUtils.schema(spec));
+                    if (StringUtils.hasText(type.runtime())) {
+                        builder.runtime(type.runtime());
+                    }
+                    SchemaImpl schema = builder.build();
+                    schemas.put(kind, schema);
+                }
+            });
+    }
 }

modules/commons/src/main/java/it/smartcommunitylabdhub/commons/services/SpecRegistry.java

@@ -27,4 +27,6 @@ public interface SpecRegistry {
     Schema getSchema(@NotNull String kind);
     Collection<Schema> getSchemas(@NotNull EntityName name, @NotNull String runtime);
     Collection<Schema> listSchemas(@NotNull EntityName name);
+
+    public void refresh();
 }

modules/framework-k8s/pom.xml

@@ -100,6 +100,11 @@
             <artifactId>log4j-over-slf4j</artifactId>
             <version>${slf4j.version}</version> <!-- or the latest version -->
         </dependency>
+        <dependency>
+            <groupId>net.bytebuddy</groupId>
+            <artifactId>byte-buddy</artifactId>
+            <version>1.14.17</version>
+        </dependency>
     </dependencies>
 
 

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/base/K8sTaskBaseSpec.java

@@ -34,6 +34,10 @@ public class K8sTaskBaseSpec extends TaskBaseSpec {
 
     private List<CoreToleration> tolerations;
 
+    private String runtimeClass;
+
+    private String priorityClass;
+
     @Override
     public void configure(Map<String, Serializable> data) {
         super.configure(data);
@@ -47,5 +51,7 @@ public void configure(Map<String, Serializable> data) {
         this.secrets = spec.getSecrets();
         this.affinity = spec.getAffinity();
         this.tolerations = spec.getTolerations();
+        this.runtimeClass = spec.getRuntimeClass();
+        this.priorityClass = spec.getPriorityClass();
     }
 }

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/base/K8sTaskSpec.java

@@ -4,7 +4,6 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreAffinity;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
-import it.smartcommunitylabdhub.framework.k8s.objects.CoreLabel;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreNodeSelector;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreResource;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreToleration;

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/infrastructure/k8s/K8sCronJobFramework.java

@@ -169,6 +169,8 @@ public V1CronJob build(K8sCronJobRunnable runnable) throws K8sFrameworkException
             .secrets(runnable.getSecrets())
             .task(runnable.getTask())
             .tolerations(runnable.getTolerations())
+            .runtimeClass(runnable.getRuntimeClass())
+            .priorityClass(runnable.getPriorityClass())
             .volumes(runnable.getVolumes())
             .state(State.READY.name())
             .build();

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/infrastructure/k8s/K8sDeploymentFramework.java

@@ -324,6 +324,8 @@ public V1Deployment build(K8sDeploymentRunnable runnable) throws K8sFrameworkExc
             .nodeSelector(buildNodeSelector(runnable))
             .affinity(runnable.getAffinity())
             .tolerations(buildTolerations(runnable))
+            .runtimeClassName(runnable.getRuntimeClass())
+            .priorityClassName(runnable.getPriorityClass())
             .volumes(volumes)
             .restartPolicy("Always")
             .imagePullSecrets(buildImagePullSecrets(runnable));

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/infrastructure/k8s/K8sJobFramework.java

@@ -331,6 +331,8 @@ public V1Job build(K8sJobRunnable runnable) throws K8sFrameworkException {
             .nodeSelector(buildNodeSelector(runnable))
             .affinity(runnable.getAffinity())
             .tolerations(buildTolerations(runnable))
+            .runtimeClassName(runnable.getRuntimeClass())
+            .priorityClassName(runnable.getPriorityClass())
             .volumes(volumes)
             .restartPolicy("Never")
             .imagePullSecrets(buildImagePullSecrets(runnable));

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/infrastructure/k8s/K8sServeFramework.java

@@ -429,6 +429,8 @@ private K8sDeploymentRunnable getDeployment(K8sServeRunnable runnable) {
             .task(runnable.getTask())
             .state(runnable.getState())
             .tolerations(runnable.getTolerations())
+            .runtimeClass(runnable.getRuntimeClass())
+            .priorityClass(runnable.getPriorityClass())
             .volumes(runnable.getVolumes())
             .contextRefs(runnable.getContextRefs())
             .contextSources(runnable.getContextSources())

modules/framework-k8s/src/main/java/it/smartcommunitylabdhub/framework/k8s/runnables/K8sRunnable.java

@@ -62,6 +62,10 @@ public class K8sRunnable implements RunRunnable, SecuredRunnable, CredentialsCon
 
     private List<CoreToleration> tolerations;
 
+    private String runtimeClass;
+
+    private String priorityClass;
+
     private List<CoreLabel> labels;
 
     private String state;

modules/runtime-container/src/main/java/it/smartcommunitylabdhub/runtime/container/runners/ContainerBuildRunner.java

@@ -105,8 +105,9 @@ public K8sKanikoRunnable produce(Run run) {
             .nodeSelector(taskSpec.getNodeSelector())
             .affinity(taskSpec.getAffinity())
             .tolerations(taskSpec.getTolerations())
+            .runtimeClass(taskSpec.getRuntimeClass())
+            .priorityClass(taskSpec.getPriorityClass())
             // Task specific
-
             .dockerFile(dockerfile)
             // specific
             .backoffLimit(1)

modules/runtime-container/src/main/java/it/smartcommunitylabdhub/runtime/container/runners/ContainerDeployRunner.java

@@ -64,6 +64,8 @@ public K8sDeploymentRunnable produce(Run run) {
             .nodeSelector(taskSpec.getNodeSelector())
             .affinity(taskSpec.getAffinity())
             .tolerations(taskSpec.getTolerations())
+            .runtimeClass(taskSpec.getRuntimeClass())
+            .priorityClass(taskSpec.getPriorityClass())
             //specific
             .replicas(taskSpec.getReplicas())
             .build();

modules/runtime-container/src/main/java/it/smartcommunitylabdhub/runtime/container/runners/ContainerJobRunner.java

@@ -67,6 +67,8 @@ public K8sRunnable produce(Run run) {
             .nodeSelector(taskSpec.getNodeSelector())
             .affinity(taskSpec.getAffinity())
             .tolerations(taskSpec.getTolerations())
+            .runtimeClass(taskSpec.getRuntimeClass())
+            .priorityClass(taskSpec.getPriorityClass())
             //specific
             .backoffLimit(taskSpec.getBackoffLimit())
             .build();
@@ -90,6 +92,8 @@ public K8sRunnable produce(Run run) {
                     .nodeSelector(taskSpec.getNodeSelector())
                     .affinity(taskSpec.getAffinity())
                     .tolerations(taskSpec.getTolerations())
+                    .runtimeClass(taskSpec.getRuntimeClass())
+                    .priorityClass(taskSpec.getPriorityClass())
                     //specific
                     .backoffLimit(taskSpec.getBackoffLimit())
                     .schedule(taskSpec.getSchedule())

modules/runtime-container/src/main/java/it/smartcommunitylabdhub/runtime/container/runners/ContainerServeRunner.java

@@ -64,6 +64,8 @@ public K8sServeRunnable produce(Run run) {
             .nodeSelector(taskSpec.getNodeSelector())
             .affinity(taskSpec.getAffinity())
             .tolerations(taskSpec.getTolerations())
+            .runtimeClass(taskSpec.getRuntimeClass())
+            .priorityClass(taskSpec.getPriorityClass())
             //specific
             .servicePorts(taskSpec.getServicePorts())
             .serviceType(taskSpec.getServiceType())

modules/runtime-dbt/src/main/java/it/smartcommunitylabdhub/runtime/dbt/specs/task/TaskTransformSpec.java

@@ -1,15 +1,11 @@
 package it.smartcommunitylabdhub.runtime.dbt.specs.task;
 
 import it.smartcommunitylabdhub.commons.annotations.common.SpecType;
-import it.smartcommunitylabdhub.commons.models.entities.task.TaskBaseSpec;
 import it.smartcommunitylabdhub.commons.models.enums.EntityName;
 import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskBaseSpec;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.runtime.dbt.DbtRuntime;
-
 import java.io.Serializable;
 import java.util.Map;
-
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;

modules/runtime-mlrun/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/runners/MlrunBuildRunner.java

@@ -1,11 +1,9 @@
 package it.smartcommunitylabdhub.runtime.mlrun.runners;
 
-import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor;
 import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.models.entities.run.Run;
 import it.smartcommunitylabdhub.commons.models.enums.State;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.runtime.mlrun.MlrunRuntime;

modules/runtime-mlrun/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/runners/MlrunJobRunner.java

@@ -1,11 +1,9 @@
 package it.smartcommunitylabdhub.runtime.mlrun.runners;
 
-import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor;
 import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.models.entities.run.Run;
 import it.smartcommunitylabdhub.commons.models.enums.State;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.runtime.mlrun.MlrunRuntime;

modules/runtime-mlrun/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/specs/task/TaskMlrunBuildSpec.java

@@ -1,17 +1,13 @@
 package it.smartcommunitylabdhub.runtime.mlrun.specs.task;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import it.smartcommunitylabdhub.commons.annotations.common.SpecType;
-import it.smartcommunitylabdhub.commons.models.entities.task.TaskBaseSpec;
 import it.smartcommunitylabdhub.commons.models.enums.EntityName;
 import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskBaseSpec;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.runtime.mlrun.MlrunRuntime;
 import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -25,8 +21,10 @@ public class TaskMlrunBuildSpec extends K8sTaskBaseSpec {
     public static final String KIND = "mlrun+build";
 
     private List<String> commands;
+
     @JsonProperty("force_build")
     private Boolean forceBuild;
+
     @JsonProperty("target_image")
     private String targetImage;
 

modules/runtime-mlrun/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/specs/task/TaskMlrunJobSpec.java

@@ -1,10 +1,8 @@
 package it.smartcommunitylabdhub.runtime.mlrun.specs.task;
 
 import it.smartcommunitylabdhub.commons.annotations.common.SpecType;
-import it.smartcommunitylabdhub.commons.models.entities.task.TaskBaseSpec;
 import it.smartcommunitylabdhub.commons.models.enums.EntityName;
 import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskBaseSpec;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.runtime.mlrun.MlrunRuntime;
 import java.io.Serializable;
 import java.util.Map;
@@ -20,7 +18,6 @@ public class TaskMlrunJobSpec extends K8sTaskBaseSpec {
 
     public static final String KIND = "mlrun+job";
 
-
     public TaskMlrunJobSpec(Map<String, Serializable> data) {
         configure(data);
     }

modules/runtime-nefertem/src/main/java/it/smartcommunitylabdhub/runtime/nefertem/runners/NefertemInferRunner.java

@@ -1,11 +1,9 @@
 package it.smartcommunitylabdhub.runtime.nefertem.runners;
 
-import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor;
 import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.models.entities.run.Run;
 import it.smartcommunitylabdhub.commons.models.enums.State;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.runtime.nefertem.NefertemRuntime;

modules/runtime-nefertem/src/main/java/it/smartcommunitylabdhub/runtime/nefertem/runners/NefertemMetricRunner.java

@@ -1,11 +1,9 @@
 package it.smartcommunitylabdhub.runtime.nefertem.runners;
 
-import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor;
 import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.models.entities.run.Run;
 import it.smartcommunitylabdhub.commons.models.enums.State;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.runtime.nefertem.NefertemRuntime;

modules/runtime-nefertem/src/main/java/it/smartcommunitylabdhub/runtime/nefertem/runners/NefertemProfileRunner.java

@@ -1,11 +1,9 @@
 package it.smartcommunitylabdhub.runtime.nefertem.runners;
 
-import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor;
 import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.models.entities.run.Run;
 import it.smartcommunitylabdhub.commons.models.enums.State;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.runtime.nefertem.NefertemRuntime;

modules/runtime-nefertem/src/main/java/it/smartcommunitylabdhub/runtime/nefertem/runners/NefertemValidateRunner.java

@@ -1,11 +1,9 @@
 package it.smartcommunitylabdhub.runtime.nefertem.runners;
 
-import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor;
 import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.models.entities.run.Run;
 import it.smartcommunitylabdhub.commons.models.enums.State;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv;
 import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable;
 import it.smartcommunitylabdhub.runtime.nefertem.NefertemRuntime;

modules/runtime-nefertem/src/main/java/it/smartcommunitylabdhub/runtime/nefertem/specs/task/TaskNefertemBaseSpec.java

@@ -1,9 +1,7 @@
 package it.smartcommunitylabdhub.runtime.nefertem.specs.task;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import it.smartcommunitylabdhub.commons.models.entities.task.TaskBaseSpec;
 import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskBaseSpec;
-import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec;
 import java.io.Serializable;
 import java.util.Map;
 import lombok.Getter;
@@ -25,7 +23,6 @@ public class TaskNefertemBaseSpec extends K8sTaskBaseSpec {
     @JsonProperty("num_worker")
     protected Integer numWorker;
 
-
     public TaskNefertemBaseSpec(Map<String, Serializable> data) {
         configure(data);
     }
@@ -40,6 +37,5 @@ public void configure(Map<String, Serializable> data) {
         this.execArgs = spec.getExecArgs();
         this.parallel = spec.getParallel();
         this.numWorker = spec.getNumWorker();
-
     }
 }

modules/runtime-python/src/main/java/it/smartcommunitylabdhub/runtime/python/runners/PythonJobRunner.java

@@ -147,6 +147,8 @@ public K8sRunnable produce(Run run) {
                 .nodeSelector(taskSpec.getNodeSelector())
                 .affinity(taskSpec.getAffinity())
                 .tolerations(taskSpec.getTolerations())
+                .runtimeClass(taskSpec.getRuntimeClass())
+                .priorityClass(taskSpec.getPriorityClass())
                 //specific
                 .backoffLimit(0)
                 .build();

modules/runtime-python/src/main/java/it/smartcommunitylabdhub/runtime/python/runners/PythonServeRunner.java

@@ -1,6 +1,5 @@
 package it.smartcommunitylabdhub.runtime.python.runners;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import it.smartcommunitylabdhub.commons.infrastructure.Runner;
 import it.smartcommunitylabdhub.commons.jackson.JacksonMapper;
@@ -158,6 +157,8 @@ public K8sRunnable produce(Run run) {
             .nodeSelector(taskSpec.getNodeSelector())
             .affinity(taskSpec.getAffinity())
             .tolerations(taskSpec.getTolerations())
+            .runtimeClass(taskSpec.getRuntimeClass())
+            .priorityClass(taskSpec.getPriorityClass())
             //specific
             .replicas(taskSpec.getReplicas())
             .servicePorts(List.of(servicePort))