-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #93 from scc-digitalhub/mlrun-build
Mlrun-build
- Loading branch information
Showing
9 changed files
with
242 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
32 changes: 32 additions & 0 deletions
32
...lrun/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/builders/MlrunBuildBuilder.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| package it.smartcommunitylabdhub.runtime.mlrun.builders; | ||
|
|
||
| import it.smartcommunitylabdhub.commons.infrastructure.Builder; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.specs.function.FunctionMlrunSpec; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.specs.run.RunMlrunSpec; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.specs.task.TaskMlrunBuildSpec; | ||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * MlrunMlrunBuilder | ||
| * <p> | ||
| * You can use this as a simple class or as a registered bean. If you want to retrieve this as bean from BuilderFactory | ||
| * you have to register it using the following annotation: | ||
| * | ||
| * @BuilderComponent(runtime = "mlrun", task = "build") | ||
| */ | ||
|
|
||
| public class MlrunBuildBuilder implements Builder<FunctionMlrunSpec, TaskMlrunBuildSpec, RunMlrunSpec> { | ||
|
|
||
| @Override | ||
| public RunMlrunSpec build(FunctionMlrunSpec funSpec, TaskMlrunBuildSpec taskSpec, RunMlrunSpec runSpec) { | ||
| RunMlrunSpec runMlrunSpec = new RunMlrunSpec(runSpec.toMap()); | ||
| runMlrunSpec.setBuildSpec(taskSpec); | ||
| runMlrunSpec.setFuncSpec(funSpec); | ||
|
|
||
| //let run override k8s specs | ||
| Optional.ofNullable(runSpec.getJobSpec()).ifPresent(k8sSpec -> runSpec.getJobSpec().configure(k8sSpec.toMap())); | ||
|
|
||
| // Return a run spec | ||
| return runMlrunSpec; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
81 changes: 81 additions & 0 deletions
81
...-mlrun/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/runners/MlrunBuildRunner.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package it.smartcommunitylabdhub.runtime.mlrun.runners; | ||
|
|
||
| import it.smartcommunitylabdhub.commons.accessors.fields.StatusFieldAccessor; | ||
| import it.smartcommunitylabdhub.commons.exceptions.CoreRuntimeException; | ||
| import it.smartcommunitylabdhub.commons.infrastructure.Runner; | ||
| import it.smartcommunitylabdhub.commons.models.entities.run.Run; | ||
| import it.smartcommunitylabdhub.commons.models.enums.State; | ||
| import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec; | ||
| import it.smartcommunitylabdhub.framework.k8s.objects.CoreEnv; | ||
| import it.smartcommunitylabdhub.framework.k8s.runnables.K8sJobRunnable; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.MlrunRuntime; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.specs.run.RunMlrunSpec; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.specs.task.TaskMlrunBuildSpec; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * MlrunMlrunRunner | ||
| * <p> | ||
| * You can use this as a simple class or as a registered bean. If you want to retrieve this as bean from RunnerFactory | ||
| * you have to register it using the following annotation: | ||
| * | ||
| * @RunnerComponent(runtime = "mlrun", task = "build") | ||
| */ | ||
| public class MlrunBuildRunner implements Runner<K8sJobRunnable> { | ||
|
|
||
| private static final String TASK = "build"; | ||
| private final String image; | ||
| private final Map<String, Set<String>> groupedSecrets; | ||
|
|
||
| public MlrunBuildRunner(String image, Map<String, Set<String>> groupedSecrets) { | ||
| this.image = image; | ||
| this.groupedSecrets = groupedSecrets; | ||
| } | ||
|
|
||
| @Override | ||
| public K8sJobRunnable produce(Run run) { | ||
| // Retrieve information about RunMlrunSpec | ||
| RunMlrunSpec runSpec = new RunMlrunSpec(run.getSpec()); | ||
| TaskMlrunBuildSpec taskSpec = runSpec.getBuildSpec(); | ||
| if (taskSpec == null) { | ||
| throw new CoreRuntimeException("null or empty task definition"); | ||
| } | ||
|
|
||
| StatusFieldAccessor statusFieldAccessor = StatusFieldAccessor.with(run.getStatus()); | ||
| K8sTaskSpec k8s = taskSpec.getK8s() != null ? taskSpec.getK8s() : new K8sTaskSpec(); | ||
|
|
||
| List<CoreEnv> coreEnvList = new ArrayList<>( | ||
| List.of(new CoreEnv("PROJECT_NAME", run.getProject()), new CoreEnv("RUN_ID", run.getId())) | ||
| ); | ||
|
|
||
| Optional.ofNullable(k8s.getEnvs()).ifPresent(coreEnvList::addAll); | ||
|
|
||
| //TODO: Create runnable using information from Run completed spec. | ||
| K8sJobRunnable k8sJobRunnable = K8sJobRunnable | ||
| .builder() | ||
| .runtime(MlrunRuntime.RUNTIME) | ||
| .task(TASK) | ||
| .image(image) | ||
| .command("python") | ||
| .args(List.of("wrapper.py").toArray(String[]::new)) | ||
| .resources(k8s.getResources()) | ||
| .nodeSelector(k8s.getNodeSelector()) | ||
| .volumes(k8s.getVolumes()) | ||
| .secrets(groupedSecrets) | ||
| .envs(coreEnvList) | ||
| .labels(k8s.getLabels()) | ||
| .affinity(k8s.getAffinity()) | ||
| .tolerations(k8s.getTolerations()) | ||
| .state(State.READY.name()) | ||
| .build(); | ||
|
|
||
| k8sJobRunnable.setId(run.getId()); | ||
| k8sJobRunnable.setProject(run.getProject()); | ||
|
|
||
| return k8sJobRunnable; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
48 changes: 48 additions & 0 deletions
48
...n/src/main/java/it/smartcommunitylabdhub/runtime/mlrun/specs/task/TaskMlrunBuildSpec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package it.smartcommunitylabdhub.runtime.mlrun.specs.task; | ||
|
|
||
| import it.smartcommunitylabdhub.commons.annotations.common.SpecType; | ||
| import it.smartcommunitylabdhub.commons.models.entities.task.TaskBaseSpec; | ||
| import it.smartcommunitylabdhub.commons.models.enums.EntityName; | ||
| import it.smartcommunitylabdhub.framework.k8s.base.K8sTaskSpec; | ||
| import it.smartcommunitylabdhub.runtime.mlrun.MlrunRuntime; | ||
| import java.io.Serializable; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
|
|
||
| import lombok.Getter; | ||
| import lombok.NoArgsConstructor; | ||
| import lombok.Setter; | ||
|
|
||
| @Getter | ||
| @Setter | ||
| @NoArgsConstructor | ||
| @SpecType(runtime = MlrunRuntime.RUNTIME, kind = TaskMlrunBuildSpec.KIND, entity = EntityName.TASK) | ||
| public class TaskMlrunBuildSpec extends TaskBaseSpec { | ||
|
|
||
| public static final String KIND = "mlrun+build"; | ||
|
|
||
| private K8sTaskSpec k8s = new K8sTaskSpec(); | ||
|
|
||
| private List<String> commands; | ||
| @JsonProperty("force_build") | ||
| private Boolean forceBuild; | ||
| @JsonProperty("target_image") | ||
| private String targetImage; | ||
|
|
||
| public TaskMlrunBuildSpec(Map<String, Serializable> data) { | ||
| configure(data); | ||
| } | ||
|
|
||
| @Override | ||
| public void configure(Map<String, Serializable> data) { | ||
| super.configure(data); | ||
|
|
||
| TaskMlrunBuildSpec spec = mapper.convertValue(data, TaskMlrunBuildSpec.class); | ||
| this.k8s = spec.getK8s(); | ||
| this.commands = spec.getCommands(); | ||
| this.forceBuild = spec.getForceBuild(); | ||
| this.targetImage = spec.getTargetImage(); | ||
| } | ||
| } |
20 changes: 20 additions & 0 deletions
20
...ain/java/it/smartcommunitylabdhub/runtime/mlrun/specs/task/TaskMlrunBuildSpecFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| package it.smartcommunitylabdhub.runtime.mlrun.specs.task; | ||
|
|
||
| import it.smartcommunitylabdhub.commons.infrastructure.SpecFactory; | ||
| import java.io.Serializable; | ||
| import java.util.Map; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| @Component | ||
| public class TaskMlrunBuildSpecFactory implements SpecFactory<TaskMlrunBuildSpec> { | ||
|
|
||
| @Override | ||
| public TaskMlrunBuildSpec create() { | ||
| return new TaskMlrunBuildSpec(); | ||
| } | ||
|
|
||
| @Override | ||
| public TaskMlrunBuildSpec create(Map<String, Serializable> data) { | ||
| return new TaskMlrunBuildSpec(data); | ||
| } | ||
| } |