Merge branch 'master' of github.com:armadaproject/armada into f/chris…

…ma/zerlog-minimal

.gitignore

@@ -14,6 +14,10 @@ junit.xml
 docker-compose.dev.yaml
 delve
 
+# VS Code debugging
+__debug_bin
+**/__debug_bin*
+
 # Build artifacts
 dist
 .goreleaser-minimal.yml

.goreleaser.yml

@@ -222,6 +222,7 @@ dockers:
       - internal/lookout/ui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
+      - pkg/api/schedulerobjects/api.swagger.json
     dockerfile: ./build/bundles/lookout/Dockerfile
 
   - id: full-bundle
@@ -255,6 +256,7 @@ dockers:
       - internal/lookout/ui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
+      - pkg/api/schedulerobjects/api.swagger.json
     dockerfile: ./build/bundles/full/Dockerfile
 
   - id: server
@@ -352,6 +354,7 @@ dockers:
       - internal/lookout/ui
       - pkg/api/api.swagger.json
       - pkg/api/binoculars/api.swagger.json
+      - pkg/api/schedulerobjects/api.swagger.json
       - config/lookoutv2/config.yaml
       - config/lookoutingesterv2/config.yaml
     dockerfile: ./build/lookoutv2/Dockerfile

.run/LookoutV2.run.xml

@@ -3,9 +3,11 @@
     <module name="armada" />
     <working_directory value="$PROJECT_DIR$" />
     <envs>
-      <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
+      <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
+      <env name="ARMADA_UICONFIG_ARMADAAPIBASEURL" value="http://localhost:8080" />
+      <env name="ARMADA_UICONFIG_BINOCULARSBASEURLPATTERN" value="http://localhost:8082" />
     </envs>
     <kind value="FILE" />
     <package value="$PROJECT_DIR$/cmd/lookoutv2/main.go" />

.run/Server.run.xml

@@ -4,7 +4,7 @@
     <working_directory value="$PROJECT_DIR$" />
     <parameters value="--config ./developer/config/insecure-armada.yaml" />
     <envs>
-      <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
+      <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
       <env name="ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_EVENTSAPIREDIS_ADDRS" value="localhost:6379" />
       <env name="ARMADA_METRICSPORT" value="9005" />

.run/lookoutv2PostgresMigration.run.xml

@@ -4,7 +4,7 @@
     <working_directory value="$PROJECT_DIR$" />
     <parameters value="--migrateDatabase" />
     <envs>
-      <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
+      <env name="ARMADA_CORSALLOWEDORIGINS" value="&quot;http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089&quot;" />
       <env name="ARMADA_POSTGRES_CONNECTION_HOST" value="localhost" />
       <env name="ARMADA_POSTGRES_CONNECTION_PORT" value="5432" />
     </envs>

build/bundles/full/Dockerfile

@@ -9,6 +9,7 @@ LABEL org.opencontainers.image.url=https://hub.docker.com/r/gresearch/armada-ful
 COPY internal/lookout/ui /project/internal/lookout/ui
 COPY pkg/api/*.swagger.json /project/pkg/api/
 COPY pkg/api/binoculars/*.swagger.json /project/pkg/api/binoculars/
+COPY pkg/api/schedulerobjects/*.swagger.json /project/pkg/api/schedulerobjects/
 RUN ./project/internal/lookout/ui/openapi.sh
 
 FROM ${NODE_BUILD_IMAGE} AS NODE

build/lookoutv2/Dockerfile

@@ -7,6 +7,7 @@ FROM ${OPENAPI_BUILD_IMAGE} AS OPENAPI
 COPY internal/lookout/ui /project/internal/lookout/ui
 COPY pkg/api/*.swagger.json /project/pkg/api/
 COPY pkg/api/binoculars/*.swagger.json /project/pkg/api/binoculars/
+COPY pkg/api/schedulerobjects/*.swagger.json /project/pkg/api/schedulerobjects/
 RUN ./project/internal/lookout/ui/openapi.sh
 
 FROM ${NODE_BUILD_IMAGE} AS NODE

cmd/server/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/armadaproject/armada/internal/server"
 	"github.com/armadaproject/armada/internal/server/configuration"
 	"github.com/armadaproject/armada/pkg/api"
+	"github.com/armadaproject/armada/pkg/api/schedulerobjects"
 )
 
 const CustomConfigLocation string = "config"
@@ -92,6 +93,7 @@ func main() {
 		api.RegisterSubmitHandler,
 		api.RegisterEventHandler,
 		api.RegisterJobsHandler,
+		schedulerobjects.RegisterSchedulerReportingHandler,
 	)
 	defer shutdownGateway()
 

deployment/lookout-v2/templates/cronjob.yaml

@@ -52,8 +52,8 @@ spec:
                 allowPrivilegeEscalation: false
           volumes:
             - name: user-config
-              secret:
-                secretName: {{ include "lookout_v2.config.name" . }}
+              configMap:
+                name: {{ include "lookout_v2.config.name" . }}
             {{- if .Values.additionalVolumes }}
             {{- toYaml .Values.additionalVolumes | nindent 12 }}
             {{- end }}

deployment/scheduler/templates/cronjob.yaml

@@ -65,8 +65,8 @@ spec:
                 allowPrivilegeEscalation: false
           volumes:
             - name: user-config
-              secret:
-                secretName: {{ include "armada-scheduler-pruner.config.name" . }}
+              configMap:
+                name: {{ include "armada-scheduler-pruner.config.name" . }}
             {{- if .Values.scheduler.additionalVolumes }}
             {{- toYaml .Values.scheduler.additionalVolumes | nindent 12 }}
             {{- end }}

developer/env/docker/binoculars.env

@@ -1,2 +1,2 @@
-ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:10000,http://example.com:10000"
+ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:4173,http://localhost:10000,http://example.com:10000"
 ARMADA_HTTP_PORT=8082

developer/env/docker/lookoutv2.env

@@ -1,4 +1,4 @@
 ARMADA_POSTGRES_CONNECTION_PORT=5432
-ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:10000,http://localhost:8082,http://example.com:10000"
+ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:4173,http://localhost:10000,http://localhost:8082,http://example.com:10000"
 ARMADA_UICONFIG_ARMADAAPIBASEURL="http://localhost:8080"
 ARMADA_UICONFIG_BINOCULARSBASEURLPATTERN="http://localhost:8082"

developer/env/docker/server.env

@@ -1,3 +1,3 @@
 ARMADA_QUEUECACHEREFRESHPERIOD="1s"
-ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:10000,http://example.com:10000"
+ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:4173,http://localhost:10000,http://example.com:10000"
 ARMADA_QUERYAPI_POSTGRES_CONNECTION_HOST=postgres

developer/env/local/binoculars.env

@@ -1 +1 @@
-  ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:8089,http://example.com:8089"
+  ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:4173,http://localhost:8089,http://example.com:8089"

developer/env/local/lookoutv2.env

@@ -1,4 +1,4 @@
 ARMADA_POSTGRES_CONNECTION_HOST="localhost"
 ARMADA_POSTGRES_CONNECTION_DBNAME=postgres
 ARMADA_POSTGRES_CONNECTION_PORT=5432
-ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089"
+ARMADA_CORSALLOWEDORIGINS="http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089"

developer/env/local/server.env

@@ -1,5 +1,5 @@
 EXECUTOR_UPDATE_INTERVAL="1s"
-ARMADA_CORSALLOWEDORIGINS=="http://localhost:3000,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089"
+ARMADA_CORSALLOWEDORIGINS=="http://localhost:3000,http://localhost:4173,http://localhost:8089,http://localhost:10000,http://example.com:10000,http://example.com:8089"
 ARMADA_EVENTSAPIREDIS_ADDRS=localhost:6379
 ARMADA_POSTGRES_CONNECTION_HOST=localhost
 ARMADA_PULSAR_URL=pulsar://localhost:6650

docs/helm.md

@@ -65,6 +65,16 @@ applicationConfig:
     armadaUrl: "server.url.com:443"  
 ```
 
+**Note:** The values you enter in this section will be placed into a K8s configmap. For senistive values (e.g. database passwords) we recommend setting them as environmental variables from a non-helm managed secret:
+```yaml
+env:
+- name: ARMADA_POSTGRES_CONNECTION_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      key: password
+      name: lookout-postgres-password
+```
+
 #### Credentials
 
 ##### Open Id

docs/python_airflow_operator.md

@@ -63,7 +63,7 @@ and handles job cancellation if the Airflow task is killed.
     * **reattach_policy** (*Optional**[**str**] **| **Callable**[**[**JobState**, **str**]**, **bool**]*) – 
 
 
-    * **extra_links** (*Optional**[**Dict**[**str**, **str**]**]*) – 
+    * **extra_links** (*Optional**[**Dict**[**str**, **Union**[**str**, **re.Pattern**]**]**]*) – 
 
 
 
@@ -199,74 +199,10 @@ acknowledged by Armada.
 :param reattach_policy: Operator reattach policy to use (defaults to: never)
 :type reattach_policy: Optional[str] | Callable[[JobState, str], bool]
 :param kwargs: Additional keyword arguments to pass to the BaseOperator.
-:param extra_links: Extra links to be shown in addition to Lookout URL.
-:type extra_links: Optional[Dict[str, str]]
+:param extra_links: Extra links to be shown in addition to Lookout URL. Regex patterns will be extracted from container logs (taking first match).
+:type extra_links: Optional[Dict[str, Union[str, re.Pattern]]]
 :param kwargs: Additional keyword arguments to pass to the BaseOperator.
 
-
-### _class_ armada.operators.armada.DynamicLink(name)
-Bases: `BaseOperatorLink`, `LoggingMixin`
-
-
-* **Parameters**
-
-    **name** (*str*) – 
-
-
-
-#### get_link(operator, \*, ti_key)
-Link to external system.
-
-Note: The old signature of this function was `(self, operator, dttm: datetime)`. That is still
-supported at runtime but is deprecated.
-
-
-* **Parameters**
-
-
-    * **operator** (*BaseOperator*) – The Airflow operator object this link is associated to.
-
-
-    * **ti_key** (*TaskInstanceKey*) – TaskInstance ID to return link for.
-
-
-
-* **Returns**
-
-    link to external system
-
-
-
-#### name(_: st_ )
-
-### _class_ armada.operators.armada.LookoutLink()
-Bases: `BaseOperatorLink`
-
-
-#### get_link(operator, \*, ti_key)
-Link to external system.
-
-Note: The old signature of this function was `(self, operator, dttm: datetime)`. That is still
-supported at runtime but is deprecated.
-
-
-* **Parameters**
-
-
-    * **operator** (*BaseOperator*) – The Airflow operator object this link is associated to.
-
-
-    * **ti_key** (*TaskInstanceKey*) – TaskInstance ID to return link for.
-
-
-
-* **Returns**
-
-    link to external system
-
-
-
-#### name(_ = 'Lookout_ )
 ## armada.triggers.armada module
 
 ## armada.auth module

internal/armadactl/scheduling.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/armadaproject/armada/internal/common"
-	"github.com/armadaproject/armada/internal/scheduler/schedulerobjects"
+	"github.com/armadaproject/armada/pkg/api/schedulerobjects"
 	"github.com/armadaproject/armada/pkg/client"
 )
 

internal/lookout/ui/README.md

@@ -76,3 +76,10 @@ yarn build
 
 This builds the app for production to the `build` folder. It correctly bundles
 React in production mode and optimizes the build for the best performance.
+
+You can then run a server to serve this production bundle locally on
+[http://localhost:4173](http://localhost:4173):
+
+```bash
+yarn serve
+```

internal/lookout/ui/openapi.sh

@@ -6,3 +6,7 @@
     -g typescript-fetch \
     -i /project/pkg/api/binoculars/api.swagger.json \
     -o /project/internal/lookout/ui/src/openapi/binoculars
+/usr/local/bin/docker-entrypoint.sh generate \
+    -g typescript-fetch \
+    -i /project/pkg/api/schedulerobjects/api.swagger.json \
+    -o /project/internal/lookout/ui/src/openapi/schedulerobjects

internal/lookout/ui/package.json

@@ -69,6 +69,7 @@
     ]
   },
   "devDependencies": {
+    "@emotion/babel-plugin": "^11.13.5",
     "@eslint/compat": "^1.2.4",
     "@tanstack/eslint-plugin-query": "^5.62.1",
     "@testing-library/dom": "^10.4.0",
@@ -96,7 +97,7 @@
     "prettier": "^3.4.2",
     "typescript": "^4.9.3",
     "vite": "^6.0.3",
-    "vitest": "^2.1.8"
+    "vitest": "^3.0.0-beta.4"
   },
   "resolutions": {
     "@types/react": "^18",