From aa07e003d2717292922a68e00128be59422913aa Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Fri, 25 Oct 2024 10:54:10 +0100 Subject: [PATCH 01/10] update a bunch of libraries (#4017) Signed-off-by: Chris Martin Co-authored-by: Dejan Zele Pejchev --- go.mod | 66 +++++++++++++------------- go.sum | 144 ++++++++++++++++++++++++++++----------------------------- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/go.mod b/go.mod index c5ee29b1f5c..c01b5c50f49 100644 --- a/go.mod +++ b/go.mod @@ -28,27 +28,27 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/golang-lru v1.0.2 github.com/jolestar/go-commons-pool v2.0.0+incompatible - github.com/jstemmer/go-junit-report/v2 v2.0.0 - github.com/mattn/go-zglob v0.0.4 + github.com/jstemmer/go-junit-report/v2 v2.1.0 + github.com/mattn/go-zglob v0.0.6 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 github.com/oklog/ulid v1.3.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_golang v1.20.5 github.com/renstrom/shortuuid v3.0.0+incompatible github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 - github.com/spf13/viper v1.18.2 - github.com/stretchr/testify v1.8.4 + github.com/spf13/viper v1.19.0 + github.com/stretchr/testify v1.9.0 github.com/weaveworks/promrus v1.2.0 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/net v0.22.0 - golang.org/x/oauth2 v0.18.0 - golang.org/x/sync v0.6.0 - google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect - google.golang.org/grpc v1.59.0 + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c + golang.org/x/net v0.30.0 + golang.org/x/oauth2 v0.23.0 + golang.org/x/sync v0.8.0 + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/grpc v1.67.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.26.15 k8s.io/apimachinery v0.26.15 @@ -78,14 +78,14 @@ require ( github.com/magefile/mage v1.14.0 github.com/minio/highwayhash v1.0.2 github.com/openconfig/goyang v1.2.0 - github.com/prometheus/common v0.48.0 + github.com/prometheus/common v0.60.0 github.com/redis/go-redis/extra/redisprometheus/v9 v9.0.5 - github.com/redis/go-redis/v9 v9.5.1 + github.com/redis/go-redis/v9 v9.7.0 github.com/segmentio/fasthash v1.0.3 github.com/xitongsys/parquet-go v1.6.2 - golang.org/x/term v0.18.0 + golang.org/x/term v0.25.0 golang.org/x/time v0.5.0 - google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 gopkg.in/inf.v0 v0.9.1 ) @@ -106,7 +106,7 @@ require ( github.com/bits-and-blooms/bitset v1.4.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/caarlos0/log v0.4.4 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/lipgloss v0.9.1 // indirect github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -122,7 +122,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-errors/errors v1.0.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -152,7 +152,8 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.5 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/linkedin/goavro/v2 v2.9.8 // indirect @@ -170,21 +171,21 @@ require ( github.com/muesli/termenv v0.15.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/pquerna/cachecontrol v0.1.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/pquerna/cachecontrol v0.2.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.2 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cast v1.7.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 // indirect @@ -195,16 +196,15 @@ require ( go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.15.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/square/go-jose.v2 v2.4.1 // indirect + gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/cli-runtime v0.26.15 // indirect k8s.io/klog/v2 v2.100.1 // indirect diff --git a/go.sum b/go.sum index 88c3d7a4b89..f63c61de172 100644 --- a/go.sum +++ b/go.sum @@ -8,14 +8,13 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -86,8 +85,8 @@ github.com/caarlos0/testfs v0.4.4 h1:3PHvzHi5Lt+g332CiShwS8ogTgS3HjrmzZxCm6JCDr8 github.com/caarlos0/testfs v0.4.4/go.mod h1:bRN55zgG4XCUVVHZCeU+/Tz1Q6AxEJOEJTliBy+1DMk= github.com/cenkalti/backoff/v4 v4.0.0/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/glamour v0.7.0 h1:2BtKGZ4iVJCDfMF229EzbeR1QRKLWztO9dMtjmqZSng= github.com/charmbracelet/glamour v0.7.0/go.mod h1:jUMh5MeihljJPQbJ/wf4ldw2+yBP59+ctV36jASy7ps= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= @@ -100,7 +99,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -148,8 +147,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/analysis v0.22.2 h1:ZBmNoP2h5omLKr/srIC9bfqrUGzT6g6gNv03HE9Vpj0= github.com/go-openapi/analysis v0.22.2/go.mod h1:pDF4UbZsQTo/oNuRfAWWd4dAh4yuYf//LYorPTjrpvo= github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= @@ -330,15 +329,15 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jstemmer/go-junit-report/v2 v2.0.0 h1:bMZNO9B16VFn07tKyi4YJFIbZtVmJaa5Xakv9dcwK58= -github.com/jstemmer/go-junit-report/v2 v2.0.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= +github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= +github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= -github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -348,6 +347,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= @@ -371,8 +371,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-zglob v0.0.4 h1:LQi2iOm0/fGgu80AioIJ/1j9w9Oh+9DZ39J4VAGzHQM= -github.com/mattn/go-zglob v0.0.4/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= +github.com/mattn/go-zglob v0.0.6 h1:mP8RnmCgho4oaUYDIDn6GNxYk+qJGUs8fJLn+twYj2A= +github.com/mattn/go-zglob v0.0.6/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= @@ -420,8 +420,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -434,21 +434,21 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8cTqKc= -github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= +github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.0 h1:+V9PAREWNvJMAuJ1x1BaWl9dewMW4YrHZQbx0sJNllA= +github.com/prometheus/common v0.60.0/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/redis/go-redis/extra/redisprometheus/v9 v9.0.5 h1:kvl0LOTQD23VR1R7A9vDti9msfV6mOE2+j6ngYkFsfg= github.com/redis/go-redis/extra/redisprometheus/v9 v9.0.5/go.mod h1:VhyLk7MdSTKbJCx6+wXlj3/ebh49gTq3yBiXymYrG7w= -github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= -github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/renstrom/shortuuid v3.0.0+incompatible h1:F6T1U7bWlI3FTV+JE8HyeR7bkTeYZJntqQLA9ST4HOQ= github.com/renstrom/shortuuid v3.0.0+incompatible/go.mod h1:n18Ycpn8DijG+h/lLBQVnGKv1BCtTeXo8KKSbBOrQ8c= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -460,8 +460,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= -github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= @@ -478,21 +478,22 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= -github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -503,8 +504,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= @@ -558,8 +559,8 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -570,8 +571,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -593,8 +594,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -619,15 +620,15 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -637,8 +638,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -673,12 +674,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -688,8 +689,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -749,8 +750,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -773,12 +772,12 @@ google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg= -google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f/go.mod h1:nWSwAFPb+qfNJXsoeO3Io7zf4tMSfN8EA8RlDA04GhY= -google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY= -google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -790,8 +789,8 @@ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -802,8 +801,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -821,8 +820,9 @@ gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mN gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= +gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From b9a0eb432afbcfee075fbcc0e9a4ce66bd656535 Mon Sep 17 00:00:00 2001 From: Eleanor Pratt <101330560+eleanorpratt@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:28:14 +0000 Subject: [PATCH 02/10] Lookout Ingester DB Load Tester (#264) (#4028) Creates a load test that which using the Lookout Ingester message conversion and database insertion code, allowing us to insert realistic event patterns into the database and measure comparative performance of changes to the database code. Co-authored-by: Eleanor Pratt --- cmd/lookoutingesterv2/dbloadtester/main.go | 91 +++++ internal/common/ingest/batch.go | 3 + .../lookoutingesterv2/dbloadtester/queue.go | 323 ++++++++++++++++++ .../dbloadtester/simulator.go | 229 +++++++++++++ .../dbloadtester/test_data.yaml | 42 +++ 5 files changed, 688 insertions(+) create mode 100644 cmd/lookoutingesterv2/dbloadtester/main.go create mode 100644 internal/lookoutingesterv2/dbloadtester/queue.go create mode 100644 internal/lookoutingesterv2/dbloadtester/simulator.go create mode 100644 internal/lookoutingesterv2/dbloadtester/test_data.yaml diff --git a/cmd/lookoutingesterv2/dbloadtester/main.go b/cmd/lookoutingesterv2/dbloadtester/main.go new file mode 100644 index 00000000000..8daf0960f2f --- /dev/null +++ b/cmd/lookoutingesterv2/dbloadtester/main.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "sigs.k8s.io/yaml" + + "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/internal/common/app" + "github.com/armadaproject/armada/internal/lookoutingesterv2/configuration" + "github.com/armadaproject/armada/internal/lookoutingesterv2/dbloadtester" +) + +func init() { + pflag.StringSlice( + "lookoutIngesterConfig", + []string{}, + "Fully qualified path to application configuration file (for multiple config files repeat this arg or separate paths with commas)", + ) + pflag.Parse() +} + +const ReportTemplate string = ` + Load Test on LookoutIngester at %s + + Configuration: + Total Jobs Simulated: %d + Total Concurrent Jobs Simulated: %d + Maximum Batch of Jobs Per Queue: %d + Queues in Use: %s + LookoutIngester Config: + +%s + + Results: + Total Load Test Duration: %s + Total DB Insertion Duration: %s + Number of Events Processed: %d + Average DB Insertion Time Per Event: %f milliseconds + Events Processed By DB Per Second: %f events +` + +func main() { + common.ConfigureLogging() + common.BindCommandlineArguments() + + var config configuration.LookoutIngesterV2Configuration + userSpecifiedConfigs := viper.GetStringSlice("lookoutIngesterConfig") + common.LoadConfig(&config, "./config/lookoutingesterv2", userSpecifiedConfigs) + + loadtesterConfig := dbloadtester.Config{ + TotalJobs: 500000, + TotalConcurrentJobs: 50000, + QueueSubmitBatchSize: 300, + QueueNames: []string{"queue1", "queue2", "queue3"}, + JobTemplateFile: "internal/lookoutingesterv2/dbloadtester/test_data.yaml", + } + + loadtester := dbloadtester.Setup( + config, + loadtesterConfig, + ) + + results, err := loadtester.Run(app.CreateContextWithShutdown()) + if err != nil { + log.Errorf("Ingestion simulator failed: %v", err) + } + + LIConfig, err := yaml.Marshal(config) + if err != nil { + log.Warn("Failed to marshal lookout ingester config for report output") + } + fmt.Printf( + ReportTemplate, + time.Now().Format("2006-01-02"), + loadtesterConfig.TotalJobs, + loadtesterConfig.TotalConcurrentJobs, + loadtesterConfig.QueueSubmitBatchSize, + loadtesterConfig.QueueNames, + string(LIConfig), + results.TotalTestDuration, + results.TotalDBInsertionDuration, + results.TotalEventsProcessed, + float64(results.TotalDBInsertionDuration.Milliseconds())/float64(results.TotalEventsProcessed), + float64(results.TotalEventsProcessed)/float64(results.TotalDBInsertionDuration.Seconds()), + ) +} diff --git a/internal/common/ingest/batch.go b/internal/common/ingest/batch.go index 2e7c239f3af..02a35f7490c 100644 --- a/internal/common/ingest/batch.go +++ b/internal/common/ingest/batch.go @@ -53,6 +53,9 @@ func (b *Batcher[T]) Run(ctx *armadacontext.Context) { case value, ok := <-b.input: if !ok { // input channel has closed + if totalNumberOfItems > 0 { + b.publish <- b.buffer + } return } b.mutex.Lock() diff --git a/internal/lookoutingesterv2/dbloadtester/queue.go b/internal/lookoutingesterv2/dbloadtester/queue.go new file mode 100644 index 00000000000..6ad9775488f --- /dev/null +++ b/internal/lookoutingesterv2/dbloadtester/queue.go @@ -0,0 +1,323 @@ +package dbloadtester + +import ( + "fmt" + "math/rand" + "time" + + "github.com/apache/pulsar-client-go/pulsar" + "github.com/google/uuid" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/common/database/lookout" + "github.com/armadaproject/armada/internal/common/ingest/utils" + protoutil "github.com/armadaproject/armada/internal/common/proto" + "github.com/armadaproject/armada/internal/common/util" + "github.com/armadaproject/armada/pkg/armadaevents" +) + +var templateAnnotations = map[string]string{ + "armadaproject.io/attempt": "0", + "armadaproject.io/custom_name": "jobcustomnameforreference", + "armadaproject.io/stage": "first_stage_of_complex_workflow", + "armadaproject.io/grade": "Dev", + "armadaproject.io/tracking_guid": "Loremipsumdolorsitametconsecteturadi", + "armadaproject.io/tracking_name": "00000000", + "armadaproject.io/job-request-application": "orchestrate", + "armadaproject.io/namespace": "queueName", + "armadaproject.io/task": "loremipsumdolorsitametconsecteturadipiscingelitseddoeiusmodte", + "armadaproject.io/task_name": "Loremipsumdolors", + "armadaproject.io/workflow": "workflow", + "armadaproject.io/log-path": "loremipsumdolorsi/ametconsecteturadipiscin/elitseddoeiusmodtemporinc/diduntu", + "armadaproject.io/version": "0.00.00", +} + +type QueueEventGenerator struct { + queueName string + jobSetName string + totalJobs int + creationBatchSize int + + jobToJobState map[string]int + jobIdToJobRun map[string]string + newEventTimeToJobs map[int64][]string + + activeJobs int + maxActiveJobs int + completedJobs int + + stepSeconds int64 + startTime int64 + + jobTemplate *v1.PodSpec +} + +func NewQueueEventGenerator(queueName, jobSetName string, totalJobs, maxActiveJobs, creationBatchSize int, timeStep int64, jobTemplate *v1.PodSpec) *QueueEventGenerator { + return &QueueEventGenerator{ + queueName: queueName, + jobSetName: jobSetName, + totalJobs: totalJobs, + creationBatchSize: creationBatchSize, + maxActiveJobs: maxActiveJobs, + jobToJobState: map[string]int{}, + jobIdToJobRun: map[string]string{}, + newEventTimeToJobs: map[int64][]string{}, + activeJobs: 0, + completedJobs: 0, + stepSeconds: timeStep, + jobTemplate: jobTemplate, + } +} + +func (q *QueueEventGenerator) Generate(eventsCh chan<- *utils.EventsWithIds[*armadaevents.EventSequence]) { + totalEventsGenerated := 0 + for i := q.startTime; ; i += q.stepSeconds { + if q.completedJobs >= q.totalJobs { + return + } + events, err := q.generateEventsAtTime(i) + if err != nil { + log.Panicf("failed to generate events %s", err) + } + if len(events) == 0 { + continue + } + eventSequenceWithIds := &utils.EventsWithIds[*armadaevents.EventSequence]{ + Events: []*armadaevents.EventSequence{ + { + Queue: q.queueName, + JobSetName: q.jobSetName, + UserId: q.queueName, + Events: events, + }, + }, + MessageIds: make([]pulsar.MessageID, len(events)), + } + totalEventsGenerated += len(events) + // log.Infof("Queue %s generated %d events so far", q.queueName, totalEventsGenerated) + eventsCh <- eventSequenceWithIds + } +} + +func (q *QueueEventGenerator) generateEventsAtTime(t int64) ([]*armadaevents.EventSequence_Event, error) { + newEventJobs := q.newEventTimeToJobs[t] + var eventsToPublish []*armadaevents.EventSequence_Event + + submitEvents, err := q.generateSubmitEvents(t) + if err != nil { + return nil, err + } + + eventsToPublish = append(eventsToPublish, submitEvents...) + for _, jobId := range newEventJobs { + jobState := q.jobToJobState[jobId] + newJobState := q.getNextState(jobState) + + jobEvents, err := q.getEventsFromTargetState(t, jobId, q.jobIdToJobRun[jobId], newJobState) + if err != nil { + return nil, errors.WithMessage(err, "failed to create events from target state") + } + + eventsToPublish = append(eventsToPublish, jobEvents...) + if isStateTerminal(newJobState) { + delete(q.jobToJobState, jobId) + delete(q.jobIdToJobRun, jobId) + q.activeJobs -= 1 + q.completedJobs += 1 + } else { + timeAtNextEvent := t + q.getStateDuration(newJobState) + q.jobToJobState[jobId] = newJobState + q.newEventTimeToJobs[timeAtNextEvent] = append(q.newEventTimeToJobs[timeAtNextEvent], jobId) + } + } + + // clean up the event generation tracker at time t, as they have all now been processed + delete(q.newEventTimeToJobs, t) + + return eventsToPublish, nil +} + +func (q *QueueEventGenerator) generateSubmitEvents(t int64) ([]*armadaevents.EventSequence_Event, error) { + available := q.maxActiveJobs - q.activeJobs + remainingJobs := q.totalJobs - q.completedJobs - q.activeJobs + if available == 0 || remainingJobs == 0 { + return nil, nil + } + submitEvents := make([]*armadaevents.EventSequence_Event, min(q.creationBatchSize, remainingJobs, available)) + for i := range submitEvents { + jobId := util.NewULID() + jobRunId := uuid.NewString() + + q.jobToJobState[jobId] = lookout.JobQueuedOrdinal + q.jobIdToJobRun[jobId] = jobRunId + nextEventTime := t + q.getStateDuration(lookout.JobQueuedOrdinal) + + q.newEventTimeToJobs[nextEventTime] = append(q.newEventTimeToJobs[nextEventTime], jobId) + + events, err := q.getEventsFromTargetState(t, jobId, jobRunId, lookout.JobQueuedOrdinal) + if err != nil { + return nil, errors.WithMessage(err, "failed to create submit event") + } + submitEvents[i] = events[0] + } + q.activeJobs += len(submitEvents) + + return submitEvents, nil +} + +func (q *QueueEventGenerator) getEventsFromTargetState( + t int64, + jobId string, + jobRunId string, + targetState int, +) ([]*armadaevents.EventSequence_Event, error) { + createdTime := protoutil.ToTimestamp(time.Unix(t, 0)) + switch targetState { + case lookout.JobQueuedOrdinal: + return []*armadaevents.EventSequence_Event{ + { + Event: &armadaevents.EventSequence_Event_SubmitJob{ + SubmitJob: q.getJob(jobId), + }, + Created: createdTime, + }, + }, nil + case lookout.JobLeasedOrdinal: + return []*armadaevents.EventSequence_Event{ + { + Event: &armadaevents.EventSequence_Event_JobRunLeased{ + JobRunLeased: &armadaevents.JobRunLeased{ + JobId: jobId, + RunId: jobRunId, + NodeId: "one_true_node", + }, + }, + Created: createdTime, + }, + }, nil + case lookout.JobPendingOrdinal: + return []*armadaevents.EventSequence_Event{ + { + Event: &armadaevents.EventSequence_Event_JobRunAssigned{ + JobRunAssigned: &armadaevents.JobRunAssigned{ + JobId: jobId, + RunId: jobRunId, + }, + }, + Created: createdTime, + }, + }, nil + case lookout.JobRunningOrdinal: + return []*armadaevents.EventSequence_Event{ + { + Event: &armadaevents.EventSequence_Event_JobRunRunning{ + JobRunRunning: &armadaevents.JobRunRunning{ + JobId: jobId, + RunId: jobRunId, + }, + }, + Created: createdTime, + }, + }, nil + case lookout.JobSucceededOrdinal: + return []*armadaevents.EventSequence_Event{ + { + Event: &armadaevents.EventSequence_Event_JobRunSucceeded{ + JobRunSucceeded: &armadaevents.JobRunSucceeded{ + JobId: jobId, + RunId: jobRunId, + }, + }, + Created: createdTime, + }, + { + Event: &armadaevents.EventSequence_Event_JobSucceeded{ + JobSucceeded: &armadaevents.JobSucceeded{ + JobId: jobId, + }, + }, + Created: createdTime, + }, + }, nil + } + + return nil, fmt.Errorf("unknown target state %d %s", targetState, lookout.JobStateMap[targetState]) +} + +func (q *QueueEventGenerator) getNextState(state int) int { + // submitted to leased, leased to pending, pending to running, + switch state { + case 0: + return lookout.JobQueuedOrdinal + case lookout.JobQueuedOrdinal: + return lookout.JobLeasedOrdinal + case lookout.JobLeasedOrdinal: + return lookout.JobPendingOrdinal + case lookout.JobPendingOrdinal: + return lookout.JobRunningOrdinal + case lookout.JobRunningOrdinal: + return lookout.JobSucceededOrdinal + default: + return lookout.JobSucceededOrdinal + } +} + +func (q *QueueEventGenerator) getStateDuration(state int) int64 { + switch state { + case lookout.JobQueuedOrdinal: + return q.getEventDuration(60 * q.stepSeconds) + case lookout.JobLeasedOrdinal: + return q.getEventDuration(10 * q.stepSeconds) + case lookout.JobPendingOrdinal: + return q.getEventDuration(10 * q.stepSeconds) + case lookout.JobRunningOrdinal: + return q.getEventDuration(120 * q.stepSeconds) + default: + return q.stepSeconds + } +} + +// getEventDuration returns the time before the next event, calculated as a random number of steps between the +// minDuration and max duration of one stepSeconds larger than two times the minDuration. +func (q *QueueEventGenerator) getEventDuration(minDuration int64) int64 { + return minDuration + rand.Int63n((minDuration+q.stepSeconds)/q.stepSeconds+1)*q.stepSeconds +} + +func (q *QueueEventGenerator) getJob(jobId string) *armadaevents.SubmitJob { + submitJob := &armadaevents.SubmitJob{ + JobId: jobId, + Priority: 1000, + ObjectMeta: &armadaevents.ObjectMeta{ + Namespace: fmt.Sprintf("gold-%s", q.queueName), + Annotations: templateAnnotations, + }, + MainObject: &armadaevents.KubernetesMainObject{ + Object: &armadaevents.KubernetesMainObject_PodSpec{ + PodSpec: &armadaevents.PodSpecWithAvoidList{ + PodSpec: q.jobTemplate, + }, + }, + }, + } + + return submitJob +} + +func isStateTerminal(state int) bool { + switch state { + case lookout.JobQueuedOrdinal: + return false + case lookout.JobLeasedOrdinal: + return false + case lookout.JobPendingOrdinal: + return false + case lookout.JobRunningOrdinal: + return false + case lookout.JobSucceededOrdinal: + return true + default: + return true + } +} diff --git a/internal/lookoutingesterv2/dbloadtester/simulator.go b/internal/lookoutingesterv2/dbloadtester/simulator.go new file mode 100644 index 00000000000..c15e31825c9 --- /dev/null +++ b/internal/lookoutingesterv2/dbloadtester/simulator.go @@ -0,0 +1,229 @@ +package dbloadtester + +import ( + "context" + "math" + "regexp" + "sync" + "time" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + + "github.com/armadaproject/armada/internal/common/armadacontext" + "github.com/armadaproject/armada/internal/common/compress" + "github.com/armadaproject/armada/internal/common/database" + "github.com/armadaproject/armada/internal/common/ingest" + "github.com/armadaproject/armada/internal/common/ingest/utils" + "github.com/armadaproject/armada/internal/lookoutingesterv2/configuration" + "github.com/armadaproject/armada/internal/lookoutingesterv2/instructions" + "github.com/armadaproject/armada/internal/lookoutingesterv2/lookoutdb" + "github.com/armadaproject/armada/internal/lookoutingesterv2/metrics" + "github.com/armadaproject/armada/internal/lookoutingesterv2/model" + "github.com/armadaproject/armada/pkg/armadaevents" + clientUtil "github.com/armadaproject/armada/pkg/client/util" +) + +type LoadTester struct { + totalJobs int + queueNames []string + totalConcurrentJobs int + useExistingQueues bool + jobTemplate *v1.PodSpec + batchSize int + batchDuration time.Duration + queueSubmitBatchSize int + + db *lookoutdb.LookoutDb + converter *instructions.InstructionConverter +} + +type Config struct { + TotalJobs int + QueueNames []string + TotalConcurrentJobs int + JobTemplateFile string + QueueSubmitBatchSize int +} + +type Results struct { + TotalTestDuration time.Duration + TotalDBInsertionDuration time.Duration + TotalEventsProcessed int +} + +func Setup(lookoutIngesterConfig configuration.LookoutIngesterV2Configuration, testConfig Config) *LoadTester { + m := metrics.Get() + + db, err := database.OpenPgxPool(lookoutIngesterConfig.Postgres) + if err != nil { + panic(errors.WithMessage(err, "Error opening connection to postgres")) + } + + fatalRegexes := make([]*regexp.Regexp, len(lookoutIngesterConfig.FatalInsertionErrors)) + for i, str := range lookoutIngesterConfig.FatalInsertionErrors { + rgx, err := regexp.Compile(str) + if err != nil { + log.Errorf("Error compiling regex %s", str) + panic(err) + } + fatalRegexes[i] = rgx + } + + lookoutDb := lookoutdb.NewLookoutDb(db, fatalRegexes, m, lookoutIngesterConfig.MaxBackoff) + + // To avoid load testing the compression algorithm, the compressor is configured not to compress. + compressor, err := compress.NewZlibCompressor(math.MaxInt) + if err != nil { + panic(errors.WithMessage(err, "Error creating compressor")) + } + + converter := instructions.NewInstructionConverter(m.Metrics, lookoutIngesterConfig.UserAnnotationPrefix, compressor) + + submitJobTemplate := &v1.PodSpec{} + err = clientUtil.BindJsonOrYaml(testConfig.JobTemplateFile, submitJobTemplate) + if err != nil { + panic(errors.WithMessage(err, "Error reading job template yaml")) + } + + if len(testConfig.QueueNames) > testConfig.TotalConcurrentJobs { + panic("Performance simulator currently requires a minimum of one concurrent job per queue") + } + + return &LoadTester{ + testConfig.TotalJobs, + testConfig.QueueNames, + testConfig.TotalConcurrentJobs, + false, + submitJobTemplate, + lookoutIngesterConfig.BatchSize, + lookoutIngesterConfig.BatchDuration, + testConfig.QueueSubmitBatchSize, + lookoutDb, + converter, + } +} + +// Run performs the load test with the configuration and database provided by the loadtester +func (l *LoadTester) Run(ctx *armadacontext.Context) (*Results, error) { + loadTestStart := time.Now() + + // generates events, simulated to have a realistic pattern + simulatedEvents := make(chan *utils.EventsWithIds[*armadaevents.EventSequence]) + go func() { + l.GenerateEvents(simulatedEvents) + close(simulatedEvents) + }() + + // set up batching, to match expected batching behaviour in non-test code + batchedEventSequences := make(chan []*utils.EventsWithIds[*armadaevents.EventSequence]) + eventCounterFunc := func(seq *utils.EventsWithIds[*armadaevents.EventSequence]) int { + totalEvents := 0 + for _, sequence := range seq.Events { + totalEvents += len(sequence.Events) + } + return totalEvents + } + + // batch the generated events + batcher := ingest.NewBatcher[*utils.EventsWithIds[*armadaevents.EventSequence]](simulatedEvents, l.batchSize, l.batchDuration, eventCounterFunc, batchedEventSequences) + go func() { + batcher.Run(ctx) + close(batchedEventSequences) + }() + + // Merge intermediate event batches + mergedEventBatches := make(chan *utils.EventsWithIds[*armadaevents.EventSequence]) + go func() { + for batch := range batchedEventSequences { + allEvents := &utils.EventsWithIds[*armadaevents.EventSequence]{} + for _, eventsWithIds := range batch { + allEvents.Events = append(allEvents.Events, eventsWithIds.Events...) + allEvents.MessageIds = append(allEvents.MessageIds, eventsWithIds.MessageIds...) + } + mergedEventBatches <- allEvents + } + close(mergedEventBatches) + }() + + // convert the events into the instructionSet taken by the db + instructionSets := make(chan *model.InstructionSet) + go func() { + for msg := range mergedEventBatches { + start := time.Now() + converted := l.converter.Convert(ctx, msg) + taken := time.Now().Sub(start) + log.Infof("Processed %d pulsar messages in %dms", len(msg.MessageIds), taken.Milliseconds()) + instructionSets <- converted + } + close(instructionSets) + }() + + // benchmark the insertion into the db + var totalDBTime time.Duration + var totalMessages int + for msg := range instructionSets { + start := time.Now() + err := l.db.Store(ctx, msg) + totalDBTime += time.Now().Sub(start) + if err != nil { + log.WithError(err).Warn("Error inserting messages") + log.Panic("db err") + } else { + log.Infof("Inserted %d pulsar messages in %dms", len(msg.GetMessageIDs()), totalDBTime.Milliseconds()) + totalMessages += len(msg.GetMessageIDs()) + } + if errors.Is(err, context.DeadlineExceeded) { + // This occurs when we're shutting down- it's a signal to stop processing immediately + break + } + } + loadTestDuration := time.Now().Sub(loadTestStart) + + return &Results{ + TotalTestDuration: loadTestDuration, + TotalDBInsertionDuration: totalDBTime, + TotalEventsProcessed: totalMessages, + }, nil +} + +// GenerateEvents generates EventSequencesWithIds consisting of job and job run events onto the given channel. +func (l *LoadTester) GenerateEvents(eventsCh chan<- *utils.EventsWithIds[*armadaevents.EventSequence]) { + totalPerQueue := l.totalJobs / len(l.queueNames) + additionalJobs := l.totalJobs % len(l.queueNames) + + totalConcurrentPerQueue := l.totalConcurrentJobs / len(l.queueNames) + additionalConcurrentJobs := l.totalConcurrentJobs % len(l.queueNames) + + // create queues + queues := make([]*QueueEventGenerator, len(l.queueNames)) + for i, queueName := range l.queueNames { + if i == len(l.queueNames)-1 { + totalPerQueue += additionalJobs + totalConcurrentPerQueue += additionalConcurrentJobs + } + queue := NewQueueEventGenerator( + queueName, + queueName, + totalPerQueue, + totalConcurrentPerQueue, + l.queueSubmitBatchSize, + 1, + l.jobTemplate, + ) + queues[i] = queue + } + + wg := sync.WaitGroup{} + wg.Add(len(queues)) + for _, queue := range queues { + go func() { + defer wg.Done() + queue.Generate(eventsCh) + }() + } + wg.Wait() + + return +} diff --git a/internal/lookoutingesterv2/dbloadtester/test_data.yaml b/internal/lookoutingesterv2/dbloadtester/test_data.yaml new file mode 100644 index 00000000000..84d29dccb6c --- /dev/null +++ b/internal/lookoutingesterv2/dbloadtester/test_data.yaml @@ -0,0 +1,42 @@ +containers: + - name: maincontainer + image: loremipsumdolo.rsitametconsectetur.adipiscingel/itseddoeius/modtempor.incididu.ntu:LATEST-00000 + command: + - loremip/sumdolo/rsitamet/consectetur/adipisc/ingel.sh + args: + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + resources: + limits: + cpu: '2' + ephemeral-storage: 64Gi + memory: 24Gi + requests: + cpu: '2' + ephemeral-storage: 64Gi + memory: 24Gi + imagePullPolicy: IfNotPresent +restartPolicy: Never +terminationGracePeriodSeconds: 1 +activeDeadlineSeconds: 259200 +serviceAccount: default +shareProcessNamespace: false +priorityClassName: armada-default From e096d6a60eafa285f5698ca5e55bf67187cd3934 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Wed, 30 Oct 2024 09:12:05 +0000 Subject: [PATCH 03/10] Add post processor that will add a gang id label (#265) (#4029) * Add post processor that will add a gang id label * lint * whitespace Co-authored-by: Christopher Martin --- internal/server/configuration/types.go | 2 + .../server/submit/conversion/post_process.go | 19 +++++++ .../submit/conversion/post_process_test.go | 50 +++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/internal/server/configuration/types.go b/internal/server/configuration/types.go index 57c5dde4489..91c26f424cd 100644 --- a/internal/server/configuration/types.go +++ b/internal/server/configuration/types.go @@ -93,6 +93,8 @@ type SubmissionConfig struct { MaxOversubscriptionByResourceRequest map[string]float64 // Enforce that an init containers requestion non-integer cpu. This is due to https://github.com/kubernetes/kubernetes/issues/112228 AssertInitContainersRequestFractionalCpu bool + // Controls whether we add the gang id annotation as a label. + AddGangIdLabel bool } // TODO: we can probably just typedef this to map[string]string diff --git a/internal/server/submit/conversion/post_process.go b/internal/server/submit/conversion/post_process.go index 1c71c8664a5..dfe57ccc3d4 100644 --- a/internal/server/submit/conversion/post_process.go +++ b/internal/server/submit/conversion/post_process.go @@ -23,6 +23,7 @@ var ( msgLevelProcessors = []msgProcessor{ templateMeta, defaultGangNodeUniformityLabel, + addGangIdLabel, } podLevelProcessors = []podProcessor{ defaultActiveDeadlineSeconds, @@ -166,6 +167,24 @@ func defaultGangNodeUniformityLabel(msg *armadaevents.SubmitJob, config configur } } +// Add a gangId label if the gangId annotation is set. We do this because labels are much faster to search on than +// annotations and a gang may want to hit the kubeapi to find its other gang members. +func addGangIdLabel(msg *armadaevents.SubmitJob, config configuration.SubmissionConfig) { + if !config.AddGangIdLabel { + return + } + + gangId := msg.GetObjectMeta().GetAnnotations()[configuration.GangIdAnnotation] + if gangId != "" { + labels := msg.GetObjectMeta().GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[configuration.GangIdAnnotation] = gangId + msg.GetObjectMeta().Labels = labels + } +} + // Templates the JobId in labels and annotations. This allows users to define labels and annotations containing the string // {JobId} and have it populated with the actual id of the job. func templateMeta(msg *armadaevents.SubmitJob, _ configuration.SubmissionConfig) { diff --git a/internal/server/submit/conversion/post_process_test.go b/internal/server/submit/conversion/post_process_test.go index 2573960daa6..1374bb8eb44 100644 --- a/internal/server/submit/conversion/post_process_test.go +++ b/internal/server/submit/conversion/post_process_test.go @@ -584,6 +584,56 @@ func TestDefaultTerminationGracePeriod(t *testing.T) { } } +func TestAddGangIdLabel(t *testing.T) { + tests := map[string]struct { + annotations map[string]string + initialLabels map[string]string + expectedLabels map[string]string + enabled bool + }{ + "Unchanged if no gang id set": { + annotations: map[string]string{}, + enabled: true, + }, + "Label added if gang id set": { + annotations: map[string]string{ + configuration.GangIdAnnotation: "foo", + }, + expectedLabels: map[string]string{ + configuration.GangIdAnnotation: "foo", + }, + enabled: true, + }, + "Doesn't modify existing labels": { + annotations: map[string]string{ + configuration.GangIdAnnotation: "foo", + }, + initialLabels: map[string]string{ + "fish": "chips", + }, + expectedLabels: map[string]string{ + "fish": "chips", + configuration.GangIdAnnotation: "foo", + }, + enabled: true, + }, + "Unchanged if disabled": { + annotations: map[string]string{ + configuration.GangIdAnnotation: "foo", + }, + enabled: false, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + submitMsg := submitMsgFromAnnotations(tc.annotations) + submitMsg.ObjectMeta.Labels = tc.initialLabels + addGangIdLabel(submitMsg, configuration.SubmissionConfig{AddGangIdLabel: tc.enabled}) + assert.Equal(t, tc.expectedLabels, submitMsg.ObjectMeta.Labels) + }) + } +} + func submitMsgFromAnnotations(annotations map[string]string) *armadaevents.SubmitJob { return &armadaevents.SubmitJob{ ObjectMeta: &armadaevents.ObjectMeta{ From af6554b2190908100f497a0b7cb469a7ffe9bdd8 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Wed, 30 Oct 2024 16:06:41 -0700 Subject: [PATCH 04/10] Update scheduler initialization behavior to avoid OOM (#4026) * Update scheduler initialization behavior to avoid OOM Signed-off-by: Jason Parraga * lint Signed-off-by: Jason Parraga * Add unit tests to scheduler Signed-off-by: Jason Parraga * Address some comments Signed-off-by: Jason Parraga * Add assertions for serials Signed-off-by: Jason Parraga * Address comments Signed-off-by: Jason Parraga --------- Signed-off-by: Jason Parraga --- internal/scheduler/database/job_repository.go | 68 +++++++ .../scheduler/database/job_repository_test.go | 174 ++++++++++++++++++ internal/scheduler/database/modelsext.go | 5 + internal/scheduler/database/query.sql.go | 146 +++++++++++++++ internal/scheduler/database/query/query.sql | 6 + internal/scheduler/mocks/job_repository.go | 16 ++ internal/scheduler/scheduler.go | 19 +- internal/scheduler/scheduler_test.go | 148 ++++++++++++++- magefiles/tests.go | 19 +- 9 files changed, 593 insertions(+), 8 deletions(-) diff --git a/internal/scheduler/database/job_repository.go b/internal/scheduler/database/job_repository.go index fabb791d92d..9e84614d329 100644 --- a/internal/scheduler/database/job_repository.go +++ b/internal/scheduler/database/job_repository.go @@ -36,6 +36,9 @@ type JobRunLease struct { // JobRepository is an interface to be implemented by structs which provide job and run information type JobRepository interface { + // FetchInitialJobs returns all non-terminal jobs and their associated job runs. + FetchInitialJobs(ctx *armadacontext.Context) ([]Job, []Run, error) + // FetchJobUpdates returns all jobs and job dbRuns that have been updated after jobSerial and jobRunSerial respectively // These updates are guaranteed to be consistent with each other FetchJobUpdates(ctx *armadacontext.Context, jobSerial int64, jobRunSerial int64) ([]Job, []Run, error) @@ -73,6 +76,71 @@ func NewPostgresJobRepository(db *pgxpool.Pool, batchSize int32) *PostgresJobRep } } +func (r *PostgresJobRepository) FetchInitialJobs(ctx *armadacontext.Context) ([]Job, []Run, error) { + var updatedJobs []Job + var initialRuns []Run + + start := time.Now() + defer func() { + ctx.Infof( + "received %d initial jobs and %d initial job runs from postgres in %s", + len(updatedJobs), len(initialRuns), time.Since(start), + ) + }() + + // Use a RepeatableRead transaction here so that we get consistency between jobs and dbRuns + err := pgx.BeginTxFunc(ctx, r.db, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadOnly, + DeferrableMode: pgx.Deferrable, + }, func(tx pgx.Tx) error { + var err error + queries := New(tx) + + // Fetch jobs + initialJobRows, err := fetch(0, r.batchSize, func(from int64) ([]SelectInitialJobsRow, error) { + return queries.SelectInitialJobs(ctx, SelectInitialJobsParams{Serial: from, Limit: r.batchSize}) + }) + if err != nil { + return err + } + + updatedJobs = make([]Job, len(initialJobRows)) + updatedJobIds := make([]string, len(initialJobRows)) + for i, row := range initialJobRows { + updatedJobIds[i] = row.JobID + updatedJobs[i] = Job{ + JobID: row.JobID, + JobSet: row.JobSet, + Queue: row.Queue, + Priority: row.Priority, + Submitted: row.Submitted, + Validated: row.Validated, + Queued: row.Queued, + QueuedVersion: row.QueuedVersion, + CancelRequested: row.CancelRequested, + Cancelled: row.Cancelled, + CancelByJobsetRequested: row.CancelByJobsetRequested, + Succeeded: row.Succeeded, + Failed: row.Failed, + SchedulingInfo: row.SchedulingInfo, + SchedulingInfoVersion: row.SchedulingInfoVersion, + Serial: row.Serial, + Pools: row.Pools, + } + } + + // Fetch dbRuns + initialRuns, err = fetch(0, r.batchSize, func(from int64) ([]Run, error) { + return queries.SelectInitialRuns(ctx, SelectInitialRunsParams{Serial: from, Limit: r.batchSize, JobIds: updatedJobIds}) + }) + + return err + }) + + return updatedJobs, initialRuns, err +} + // FetchJobRunErrors returns all armadaevents.JobRunErrors for the provided job run ids. The returned map is // keyed by job run id. Any dbRuns which don't have errors wil be absent from the map. func (r *PostgresJobRepository) FetchJobRunErrors(ctx *armadacontext.Context, runIds []string) (map[string]*armadaevents.Error, error) { diff --git a/internal/scheduler/database/job_repository_test.go b/internal/scheduler/database/job_repository_test.go index 7e3706da736..4a29c508337 100644 --- a/internal/scheduler/database/job_repository_test.go +++ b/internal/scheduler/database/job_repository_test.go @@ -24,6 +24,180 @@ import ( const defaultBatchSize = 1 +func TestFetchInitialJobs(t *testing.T) { + leasedJob := Job{ + JobID: util.NewULID(), + JobSet: "test-jobset", + Queue: "test-queue", + QueuedVersion: 1, + SchedulingInfo: []byte{byte(0)}, + SubmitMessage: []byte{}, + } + + expectedLeasedJob := Job{ + JobID: leasedJob.JobID, + JobSet: "test-jobset", + Queue: "test-queue", + QueuedVersion: 1, + SchedulingInfo: []byte{byte(0)}, + Serial: 1, + } + + leasedJobRun := Run{ + RunID: util.NewULID(), + JobID: leasedJob.JobID, + JobSet: "test-jobset", + Executor: "test-executor", + Node: fmt.Sprintf("test-node-%d", 0), + Running: true, + } + + expectedLeasedJobRun := Run{ + RunID: leasedJobRun.RunID, + JobID: leasedJob.JobID, + JobSet: "test-jobset", + Executor: "test-executor", + Node: fmt.Sprintf("test-node-%d", 0), + Running: true, + Serial: 1, + } + + queuedJob := Job{ + JobID: util.NewULID(), + JobSet: "test-jobset", + Queue: "test-queue", + Queued: true, + QueuedVersion: 1, + SchedulingInfo: []byte{byte(0)}, + SubmitMessage: []byte{}, + } + + expectedQueuedJob := Job{ + JobID: queuedJob.JobID, + JobSet: "test-jobset", + Queue: "test-queue", + Queued: true, + QueuedVersion: 1, + SchedulingInfo: []byte{byte(0)}, + Serial: 2, + } + + cancelledJob := Job{ + JobID: util.NewULID(), + JobSet: "test-jobset", + Queue: "test-queue", + QueuedVersion: 1, + Cancelled: true, + SchedulingInfo: []byte{byte(0)}, + SubmitMessage: []byte{}, + } + + cancelledJobRun := Run{ + RunID: util.NewULID(), + JobID: cancelledJob.JobID, + JobSet: "test-jobset", + Executor: "test-executor", + Node: fmt.Sprintf("test-node-%d", 0), + Cancelled: true, + } + + failedJob := Job{ + JobID: util.NewULID(), + JobSet: "test-jobset", + Queue: "test-queue", + QueuedVersion: 1, + Failed: true, + SchedulingInfo: []byte{byte(0)}, + SubmitMessage: []byte{}, + } + + failedJobRun := Run{ + RunID: util.NewULID(), + JobID: failedJob.JobID, + JobSet: "test-jobset", + Executor: "test-executor", + Node: fmt.Sprintf("test-node-%d", 0), + Failed: true, + } + + succeededJob := Job{ + JobID: util.NewULID(), + JobSet: "test-jobset", + Queue: "test-queue", + Queued: false, + QueuedVersion: 1, + Succeeded: true, + SchedulingInfo: []byte{byte(0)}, + SubmitMessage: []byte{}, + } + + succeededJobRun := Run{ + RunID: util.NewULID(), + JobID: succeededJob.JobID, + JobSet: "test-jobset", + Executor: "test-executor", + Node: fmt.Sprintf("test-node-%d", 0), + Succeeded: true, + } + + tests := map[string]struct { + dbJobs []Job + dbRuns []Run + expectedJobs []Job + expectedRuns []Run + }{ + "all jobs and runs": { + dbJobs: []Job{leasedJob, queuedJob, cancelledJob, failedJob, succeededJob}, + expectedJobs: []Job{expectedLeasedJob, expectedQueuedJob}, + dbRuns: []Run{leasedJobRun, cancelledJobRun, failedJobRun, succeededJobRun}, + expectedRuns: []Run{expectedLeasedJobRun}, + }, + "only jobs": { + dbJobs: []Job{leasedJob, queuedJob, cancelledJob, failedJob, succeededJob}, + expectedJobs: []Job{expectedLeasedJob, expectedQueuedJob}, + expectedRuns: []Run{}, + }, + "only runs": { + dbRuns: []Run{leasedJobRun, cancelledJobRun, failedJobRun, succeededJobRun}, + expectedJobs: []Job{}, + expectedRuns: []Run{}, + }, + "empty db": { + expectedJobs: []Job{}, + expectedRuns: []Run{}, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + err := withJobRepository(func(repo *PostgresJobRepository) error { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) + + // Set up db + err := database.UpsertWithTransaction(ctx, repo.db, "jobs", tc.dbJobs) + require.NoError(t, err) + err = database.UpsertWithTransaction(ctx, repo.db, "runs", tc.dbRuns) + require.NoError(t, err) + + // Fetch updates + jobs, runs, err := repo.FetchInitialJobs(ctx) + require.NoError(t, err) + + // Runs will have LastModified filled in- we don't want to compare this + for i := range runs { + runs[i].LastModified = time.Time{} + } + + // Assert results + assert.Equal(t, tc.expectedJobs, jobs) + assert.Equal(t, tc.expectedRuns, runs) + cancel() + return nil + }) + require.NoError(t, err) + }) + } +} + func TestFetchJobUpdates(t *testing.T) { dbJobs, expectedJobs := createTestJobs(10) dbRuns, expectedRuns := createTestRuns(10) diff --git a/internal/scheduler/database/modelsext.go b/internal/scheduler/database/modelsext.go index f5d918b276b..8ccd80aae24 100644 --- a/internal/scheduler/database/modelsext.go +++ b/internal/scheduler/database/modelsext.go @@ -19,3 +19,8 @@ func (run Run) GetSerial() int64 { func (row SelectUpdatedJobsRow) GetSerial() int64 { return row.Serial } + +// GetSerial is needed for the HasSerial interface +func (row SelectInitialJobsRow) GetSerial() int64 { + return row.Serial +} diff --git a/internal/scheduler/database/query.sql.go b/internal/scheduler/database/query.sql.go index af77c1716c6..f4e8f9a95a4 100644 --- a/internal/scheduler/database/query.sql.go +++ b/internal/scheduler/database/query.sql.go @@ -398,6 +398,73 @@ func (q *Queries) SelectJobsForExecutor(ctx context.Context, arg SelectJobsForEx return items, nil } +const selectInitialJobs = `-- name: SelectInitialJobs :many +SELECT job_id, job_set, queue, priority, submitted, queued, queued_version, validated, cancel_requested, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 AND cancelled = 'false' AND succeeded = 'false' and failed = 'false' ORDER BY serial LIMIT $2 +` + +type SelectInitialJobsParams struct { + Serial int64 `db:"serial"` + Limit int32 `db:"limit"` +} + +type SelectInitialJobsRow struct { + JobID string `db:"job_id"` + JobSet string `db:"job_set"` + Queue string `db:"queue"` + Priority int64 `db:"priority"` + Submitted int64 `db:"submitted"` + Queued bool `db:"queued"` + QueuedVersion int32 `db:"queued_version"` + Validated bool `db:"validated"` + CancelRequested bool `db:"cancel_requested"` + CancelByJobsetRequested bool `db:"cancel_by_jobset_requested"` + Cancelled bool `db:"cancelled"` + Succeeded bool `db:"succeeded"` + Failed bool `db:"failed"` + SchedulingInfo []byte `db:"scheduling_info"` + SchedulingInfoVersion int32 `db:"scheduling_info_version"` + Pools []string `db:"pools"` + Serial int64 `db:"serial"` +} + +func (q *Queries) SelectInitialJobs(ctx context.Context, arg SelectInitialJobsParams) ([]SelectInitialJobsRow, error) { + rows, err := q.db.Query(ctx, selectInitialJobs, arg.Serial, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectInitialJobsRow + for rows.Next() { + var i SelectInitialJobsRow + if err := rows.Scan( + &i.JobID, + &i.JobSet, + &i.Queue, + &i.Priority, + &i.Submitted, + &i.Queued, + &i.QueuedVersion, + &i.Validated, + &i.CancelRequested, + &i.CancelByJobsetRequested, + &i.Cancelled, + &i.Succeeded, + &i.Failed, + &i.SchedulingInfo, + &i.SchedulingInfoVersion, + &i.Pools, + &i.Serial, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectNewJobs = `-- name: SelectNewJobs :many SELECT job_id, job_set, queue, user_id, submitted, groups, priority, queued, queued_version, cancel_requested, cancelled, cancel_by_jobset_requested, succeeded, failed, submit_message, scheduling_info, scheduling_info_version, serial, last_modified, validated, pools FROM jobs WHERE serial > $1 ORDER BY serial LIMIT $2 ` @@ -449,6 +516,63 @@ func (q *Queries) SelectNewJobs(ctx context.Context, arg SelectNewJobsParams) ([ return items, nil } +const selectInitialRuns = `-- name: SelectInitialRuns :many +SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool FROM runs WHERE serial > $1 AND job_id = ANY($3::text[]) ORDER BY serial LIMIT $2 +` + +type SelectInitialRunsParams struct { + Serial int64 `db:"serial"` + Limit int32 `db:"limit"` + JobIds []string `db:"job_ids"` +} + +func (q *Queries) SelectInitialRuns(ctx context.Context, arg SelectInitialRunsParams) ([]Run, error) { + rows, err := q.db.Query(ctx, selectInitialRuns, arg.Serial, arg.Limit, arg.JobIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Run + for rows.Next() { + var i Run + if err := rows.Scan( + &i.RunID, + &i.JobID, + &i.Created, + &i.JobSet, + &i.Executor, + &i.Node, + &i.Cancelled, + &i.Running, + &i.Succeeded, + &i.Failed, + &i.Returned, + &i.RunAttempted, + &i.Serial, + &i.LastModified, + &i.LeasedTimestamp, + &i.PendingTimestamp, + &i.RunningTimestamp, + &i.TerminatedTimestamp, + &i.ScheduledAtPriority, + &i.Preempted, + &i.Pending, + &i.PreemptedTimestamp, + &i.PodRequirementsOverlay, + &i.PreemptRequested, + &i.Queue, + &i.Pool, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectNewRuns = `-- name: SelectNewRuns :many SELECT run_id, job_id, created, job_set, executor, node, cancelled, running, succeeded, failed, returned, run_attempted, serial, last_modified, leased_timestamp, pending_timestamp, running_timestamp, terminated_timestamp, scheduled_at_priority, preempted, pending, preempted_timestamp, pod_requirements_overlay, preempt_requested, queue, pool FROM runs WHERE serial > $1 ORDER BY serial LIMIT $2 ` @@ -816,3 +940,25 @@ func (q *Queries) SelectAllExecutorSettings(ctx context.Context) ([]ExecutorSett } return items, nil } + +const selectLatestJobSerial = `-- name: SelectLatestJobSerial :one +SELECT serial FROM jobs ORDER BY serial DESC LIMIT 1 +` + +func (q *Queries) SelectLatestJobSerial(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, selectLatestJobSerial) + var serial int64 + err := row.Scan(&serial) + return serial, err +} + +const selectLatestJobRunSerial = `-- name: SelectLatestJobRunSerial :one +SELECT serial FROM runs ORDER BY serial DESC LIMIT 1 +` + +func (q *Queries) SelectLatestJobRunSerial(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, selectLatestJobRunSerial) + var serial int64 + err := row.Scan(&serial) + return serial, err +} diff --git a/internal/scheduler/database/query/query.sql b/internal/scheduler/database/query/query.sql index 48d70b8b630..eb943fd5e61 100644 --- a/internal/scheduler/database/query/query.sql +++ b/internal/scheduler/database/query/query.sql @@ -4,6 +4,9 @@ SELECT * FROM jobs WHERE serial > $1 ORDER BY serial LIMIT $2; -- name: SelectAllJobIds :many SELECT job_id FROM jobs; +-- name: SelectInitialJobs :many +SELECT job_id, job_set, queue, priority, submitted, queued, queued_version, validated, cancel_requested, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 AND cancelled = 'false' AND succeeded = 'false' and failed = 'false' ORDER BY serial LIMIT $2; + -- name: SelectUpdatedJobs :many SELECT job_id, job_set, queue, priority, submitted, queued, queued_version, validated, cancel_requested, cancel_by_jobset_requested, cancelled, succeeded, failed, scheduling_info, scheduling_info_version, pools, serial FROM jobs WHERE serial > $1 ORDER BY serial LIMIT $2; @@ -28,6 +31,9 @@ UPDATE jobs SET failed = true WHERE job_id = ANY(sqlc.arg(job_ids)::text[]); -- name: UpdateJobPriorityById :exec UPDATE jobs SET priority = $1 WHERE queue = sqlc.arg(queue) and job_set = sqlc.arg(job_set) and job_id = ANY(sqlc.arg(job_ids)::text[]); +-- name: SelectInitialRuns :many +SELECT * FROM runs WHERE serial > $1 AND job_id = ANY(sqlc.arg(job_ids)::text[]) ORDER BY serial LIMIT $2; + -- name: SelectNewRuns :many SELECT * FROM runs WHERE serial > $1 ORDER BY serial LIMIT $2; diff --git a/internal/scheduler/mocks/job_repository.go b/internal/scheduler/mocks/job_repository.go index 898cbebe7f1..21dba092880 100644 --- a/internal/scheduler/mocks/job_repository.go +++ b/internal/scheduler/mocks/job_repository.go @@ -52,6 +52,22 @@ func (mr *MockJobRepositoryMockRecorder) CountReceivedPartitions(arg0, arg1 inte return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountReceivedPartitions", reflect.TypeOf((*MockJobRepository)(nil).CountReceivedPartitions), arg0, arg1) } +// FetchInitialJobs mocks base method. +func (m *MockJobRepository) FetchInitialJobs(arg0 *armadacontext.Context) ([]database.Job, []database.Run, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FetchInitialJobs", arg0) + ret0, _ := ret[0].([]database.Job) + ret1, _ := ret[1].([]database.Run) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// FetchInitialJobs indicates an expected call of FetchInitialJobs. +func (mr *MockJobRepositoryMockRecorder) FetchInitialJobs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchInitialJobs", reflect.TypeOf((*MockJobRepository)(nil).FetchInitialJobs), arg0) +} + // FetchJobRunErrors mocks base method. func (m *MockJobRepository) FetchJobRunErrors(arg0 *armadacontext.Context, arg1 []string) (map[string]*armadaevents.Error, error) { m.ctrl.T.Helper() diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index f60ee6772e8..e5f86f495f5 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -230,7 +230,7 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke // Update job state. ctx.Info("Syncing internal state with database") - updatedJobs, jsts, err := s.syncState(ctx) + updatedJobs, jsts, err := s.syncState(ctx, false) if err != nil { return overallSchedulerResult, err } @@ -357,12 +357,21 @@ func (s *Scheduler) cycle(ctx *armadacontext.Context, updateAll bool, leaderToke } // syncState updates jobs in jobDb to match state in postgres and returns all updated jobs. -func (s *Scheduler) syncState(ctx *armadacontext.Context) ([]*jobdb.Job, []jobdb.JobStateTransitions, error) { +func (s *Scheduler) syncState(ctx *armadacontext.Context, initial bool) ([]*jobdb.Job, []jobdb.JobStateTransitions, error) { txn := s.jobDb.WriteTxn() defer txn.Abort() - // Load new and updated jobs from the jobRepo. - updatedJobs, updatedRuns, err := s.jobRepository.FetchJobUpdates(ctx, s.jobsSerial, s.runsSerial) + var updatedJobs []database.Job + var updatedRuns []database.Run + var err error + + if initial { + // Load initial jobs from the jobRepo. + updatedJobs, updatedRuns, err = s.jobRepository.FetchInitialJobs(ctx) + } else { + // Load new and updated jobs from the jobRepo. + updatedJobs, updatedRuns, err = s.jobRepository.FetchJobUpdates(ctx, s.jobsSerial, s.runsSerial) + } if err != nil { return nil, nil, err } @@ -966,7 +975,7 @@ func (s *Scheduler) initialise(ctx *armadacontext.Context) error { case <-ctx.Done(): return nil default: - if _, _, err := s.syncState(ctx); err != nil { + if _, _, err := s.syncState(ctx, true); err != nil { logging.WithStacktrace(ctx, err).Error("failed to initialise; trying again in 1 second") time.Sleep(1 * time.Second) } else { diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index 177e85a377a..95dbab6fc8e 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -1039,6 +1039,143 @@ func TestRun(t *testing.T) { cancel() } +func TestScheduler_TestSyncInitialState(t *testing.T) { + tests := map[string]struct { + initialJobs []database.Job /// jobs in the jobdb at the start of the cycle + initialJobRuns []database.Run // jobs runs in the jobdb at the start of the cycle + expectedInitialJobs []*jobdb.Job + expectedInitialJobDbIds []string + expectedJobsSerial int64 + expectedRunsSerial int64 + }{ + "no initial jobs": { + initialJobs: []database.Job{}, + initialJobRuns: []database.Run{}, + expectedInitialJobs: []*jobdb.Job{}, + expectedInitialJobDbIds: []string{}, + expectedJobsSerial: -1, + expectedRunsSerial: -1, + }, + "initial jobs are present": { + initialJobs: []database.Job{ + { + JobID: queuedJob.Id(), + JobSet: queuedJob.Jobset(), + Queue: queuedJob.Queue(), + Queued: false, + QueuedVersion: 1, + Priority: int64(queuedJob.Priority()), + SchedulingInfo: schedulingInfoBytes, + Serial: 1, + Validated: true, + Submitted: 1, + }, + }, + initialJobRuns: []database.Run{ + { + RunID: leasedJob.LatestRun().Id(), + JobID: queuedJob.Id(), + JobSet: queuedJob.Jobset(), + Executor: "test-executor", + Node: "test-node", + Created: 123, + ScheduledAtPriority: func() *int32 { + scheduledAtPriority := int32(5) + return &scheduledAtPriority + }(), + Serial: 1, + }, + }, + expectedInitialJobs: []*jobdb.Job{ + queuedJob.WithUpdatedRun( + testfixtures.JobDb.CreateRun( + leasedJob.LatestRun().Id(), + queuedJob.Id(), + 123, + "test-executor", + "test-executor-test-node", + "test-node", + "pool", + pointer.Int32(5), + false, + false, + false, + false, + false, + false, + false, + false, + nil, + nil, + nil, + nil, + nil, + false, + false, + )).WithQueued(false).WithQueuedVersion(1), + }, + expectedInitialJobDbIds: []string{queuedJob.Id()}, + expectedJobsSerial: 1, + expectedRunsSerial: 1, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + ctx, cancel := armadacontext.WithTimeout(armadacontext.Background(), 5*time.Second) + defer cancel() + + // Test objects + jobRepo := &testJobRepository{ + initialJobs: tc.initialJobs, + initialRuns: tc.initialJobRuns, + } + schedulingAlgo := &testSchedulingAlgo{} + publisher := &testPublisher{} + clusterRepo := &testExecutorRepository{} + leaderController := leader.NewStandaloneLeaderController() + sched, err := NewScheduler( + testfixtures.NewJobDb(testfixtures.TestResourceListFactory), + jobRepo, + clusterRepo, + schedulingAlgo, + leaderController, + publisher, + nil, + 1*time.Second, + 5*time.Second, + 1*time.Hour, + maxNumberOfAttempts, + nodeIdLabel, + schedulerMetrics, + ) + require.NoError(t, err) + sched.EnableAssertions() + + // The SchedulingKeyGenerator embedded in the jobDb has some randomness, + // which must be consistent within tests. + sched.jobDb = testfixtures.NewJobDb(testfixtures.TestResourceListFactory) + + initialJobs, _, err := sched.syncState(ctx, true) + require.NoError(t, err) + + expectedJobDb := testfixtures.NewJobDbWithJobs(tc.expectedInitialJobs) + actualJobDb := testfixtures.NewJobDbWithJobs(initialJobs) + assert.NoError(t, expectedJobDb.ReadTxn().AssertEqual(actualJobDb.ReadTxn())) + allDbJobs := sched.jobDb.ReadTxn().GetAll() + + expectedIds := stringSet(tc.expectedInitialJobDbIds) + require.Equal(t, len(tc.expectedInitialJobDbIds), len(allDbJobs)) + for _, job := range allDbJobs { + _, ok := expectedIds[job.Id()] + assert.True(t, ok) + } + + require.Equal(t, tc.expectedJobsSerial, sched.jobsSerial) + require.Equal(t, tc.expectedRunsSerial, sched.runsSerial) + }) + } +} + func TestScheduler_TestSyncState(t *testing.T) { tests := map[string]struct { initialJobs []*jobdb.Job // jobs in the jobdb at the start of the cycle @@ -1232,7 +1369,7 @@ func TestScheduler_TestSyncState(t *testing.T) { require.NoError(t, err) txn.Commit() - updatedJobs, _, err := sched.syncState(ctx) + updatedJobs, _, err := sched.syncState(ctx, false) require.NoError(t, err) expectedJobDb := testfixtures.NewJobDbWithJobs(tc.expectedUpdatedJobs) @@ -1268,6 +1405,8 @@ func (t *testSubmitChecker) Check(_ *armadacontext.Context, jobs []*jobdb.Job) ( // Test implementations of the interfaces needed by the Scheduler type testJobRepository struct { + initialJobs []database.Job + initialRuns []database.Run updatedJobs []database.Job updatedRuns []database.Run errors map[string]*armadaevents.Error @@ -1306,6 +1445,13 @@ func (t *testJobRepository) CountReceivedPartitions(ctx *armadacontext.Context, return t.numReceivedPartitions, nil } +func (t *testJobRepository) FetchInitialJobs(ctx *armadacontext.Context) ([]database.Job, []database.Run, error) { + if t.shouldError { + return nil, nil, errors.New("error fetching job updates") + } + return t.initialJobs, t.initialRuns, nil +} + type testExecutorRepository struct { updateTimes map[string]time.Time shouldError bool diff --git a/magefiles/tests.go b/magefiles/tests.go index a30ba7b51ad..e4460c8eee0 100644 --- a/magefiles/tests.go +++ b/magefiles/tests.go @@ -17,6 +17,11 @@ var Gotestsum string var LocalBin = filepath.Join(os.Getenv("PWD"), "/bin") +var ( + redisImage = "redis:6.2.6" + postgresImage = "postgres:14.2" +) + func makeLocalBin() error { if _, err := os.Stat(LocalBin); os.IsNotExist(err) { err = os.MkdirAll(LocalBin, os.ModePerm) @@ -52,12 +57,22 @@ func Tests() error { return err } - err = dockerRun("run", "-d", "--name=redis", docker_Net, "-p=6379:6379", "redis:6.2.6") + redisArgs := []string{"run", "-d", "--name=redis", "-p=6379:6379"} + if len(docker_Net) > 0 { + redisArgs = append(redisArgs, docker_Net) + } + redisArgs = append(redisArgs, redisImage) + err = dockerRun(redisArgs...) if err != nil { return err } - err = dockerRun("run", "-d", "--name=postgres", docker_Net, "-p", "5432:5432", "-e", "POSTGRES_PASSWORD=psw", "postgres:14.2") + postgresArgs := []string{"run", "-d", "--name=postgres", "-p", "5432:5432", "-e", "POSTGRES_PASSWORD=psw"} + if len(docker_Net) > 0 { + postgresArgs = append(postgresArgs, docker_Net) + } + postgresArgs = append(postgresArgs, postgresImage) + err = dockerRun(postgresArgs...) if err != nil { return err } From 133c490d4bbba50f61a58b1fb2fde5a656b3ffd1 Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Thu, 31 Oct 2024 15:18:24 +0100 Subject: [PATCH 05/10] document TrackedNodeLabels and ToleratedTaints executor config fields (#4030) --- internal/executor/configuration/types.go | 25 +++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/executor/configuration/types.go b/internal/executor/configuration/types.go index 96bd3febf81..42bbd05ed9e 100644 --- a/internal/executor/configuration/types.go +++ b/internal/executor/configuration/types.go @@ -12,6 +12,8 @@ import ( ) type ApplicationConfiguration struct { + // ClusterId is the unique identifier for the cluster that the executor is running on. + // It is used to identify the cluster in the scheduler. ClusterId string Pool string SubmitConcurrencyLimit int @@ -50,18 +52,23 @@ type ClientConfiguration struct { } type KubernetesConfiguration struct { - // Wether to impersonate users when creating Kubernetes objects. + // Whether to impersonate users when creating Kubernetes objects. ImpersonateUsers bool // Max number of Kubernetes API queries per second // and max number of concurrent Kubernetes API queries. - QPS float32 - Burst int - Etcd EtcdConfiguration - NodePoolLabel string - NodeTypeLabel string - NodeIdLabel string - TrackedNodeLabels []string - AvoidNodeLabelsOnRetry []string + QPS float32 + Burst int + Etcd EtcdConfiguration + NodePoolLabel string + NodeTypeLabel string + NodeIdLabel string + // TrackedNodeLabels is a list of node labels that the executor should index and track. + // As nodes can have many labels, taking all of them into consideration can be slow down scheduling. + // Only node labels defined in this list can be referenced in Armada job nodeSelector field. + TrackedNodeLabels []string + AvoidNodeLabelsOnRetry []string + // ToleratedTaints specifies taints which are tolerated by the executor. + // If a node has a taint that is not in this list, the executor will consider it for scheduling Armada jobs. ToleratedTaints []string MinimumPodAge time.Duration StuckTerminatingPodExpiry time.Duration From 77f8a0ad5a7afc35b2a8d7710a59e6443ec6b2a7 Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Thu, 31 Oct 2024 16:21:12 +0000 Subject: [PATCH 06/10] [Simulator] Allow JobTemplates To Be Repeated (#4031) * [Simulator] Allow JobTemplates To Be Repeated (#267) * wip * wip * wip * F/chrisma/simulator repeated submission (#269) * wip * wip * wip * lint * Update simulator.go --------- Co-authored-by: Christopher Martin --- internal/scheduler/simulator/simulator.go | 33 ++ internal/scheduler/simulator/simulator.pb.go | 476 +++++++++++++++---- internal/scheduler/simulator/simulator.proto | 9 + 3 files changed, 420 insertions(+), 98 deletions(-) diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index 2dde5298010..d581450aa00 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/gogo/protobuf/proto" "github.com/pkg/errors" "golang.org/x/exp/maps" "golang.org/x/exp/slices" @@ -143,6 +144,7 @@ func NewSimulator( if err := validateWorkloadSpec(workloadSpec); err != nil { return nil, err } + workloadSpec = expandRepeatingTemplates(workloadSpec) jobDb := jobdb.NewJobDb( schedulingConfig.PriorityClasses, schedulingConfig.DefaultPriorityClassName, @@ -292,6 +294,14 @@ func validateWorkloadSpec(workloadSpec *WorkloadSpec) error { if template.GangCardinality != 0 && int(template.Number)%int(template.GangCardinality) != 0 { return errors.Errorf("template.Number [%d] is not exactly divisible by template.GangCardinality [%d]", template.Number, template.GangCardinality) } + if template.Repeat != nil { + if template.Repeat.Period == nil { + return errors.Errorf("template.Repeat.Period is unset") + } + if template.Repeat.NumTimes < 1 { + return errors.Errorf("template.Repeat.NumTimes must be greater than 0") + } + } } } return nil @@ -1035,3 +1045,26 @@ func (s *Simulator) removeJobFromDemand(job *jobdb.Job) { r.SubV1ResourceList(job.PodRequirements().ResourceRequirements.Requests) } } + +func expandRepeatingTemplates(w *WorkloadSpec) *WorkloadSpec { + workload := proto.Clone(w).(*WorkloadSpec) + for _, q := range workload.GetQueues() { + var templates []*JobTemplate + for _, template := range q.GetJobTemplates() { + if template.Repeat != nil { + period := *template.Repeat.Period + for i := 0; i < int(template.Repeat.NumTimes); i++ { + t := proto.Clone(template).(*JobTemplate) + t.Repeat = nil + t.Id = fmt.Sprintf("%s-repeat-%d", t.Id, i) + t.EarliestSubmitTime = t.EarliestSubmitTime + time.Duration(i)*period + templates = append(templates, t) + } + } else { + templates = append(templates, template) + } + } + q.JobTemplates = templates + } + return workload +} diff --git a/internal/scheduler/simulator/simulator.pb.go b/internal/scheduler/simulator/simulator.pb.go index 472ca2cf906..22f78aea673 100644 --- a/internal/scheduler/simulator/simulator.pb.go +++ b/internal/scheduler/simulator/simulator.pb.go @@ -384,6 +384,8 @@ type JobTemplate struct { GangCardinality uint32 `protobuf:"varint,13,opt,name=gang_cardinality,json=gangCardinality,proto3" json:"gangCardinality,omitempty"` // Node Uniformity label when scheduling gangs. Only applies if gang_cardinality is non-zero. If unset it defaults to armadaproject.io/clusterName GangNodeUniformityLabel string `protobuf:"bytes,14,opt,name=gang_node_uniformity_label,json=gangNodeUniformityLabel,proto3" json:"gangNodeUniformityLabel,omitempty"` + // If set then the template will be repeated at some frequency. If null then the template will be submitted a single time. + Repeat *RepeatDetails `protobuf:"bytes,15,opt,name=repeat,proto3" json:"repeat,omitempty"` } func (m *JobTemplate) Reset() { *m = JobTemplate{} } @@ -517,6 +519,67 @@ func (m *JobTemplate) GetGangNodeUniformityLabel() string { return "" } +func (m *JobTemplate) GetRepeat() *RepeatDetails { + if m != nil { + return m.Repeat + } + return nil +} + +type RepeatDetails struct { + // The number of times that template should be repeated. Must be > 0 + NumTimes uint32 `protobuf:"varint,1,opt,name=num_times,json=numTimes,proto3" json:"numTimes,omitempty"` + // The period between template submissions. May not be null + Period *time.Duration `protobuf:"bytes,2,opt,name=period,proto3,stdduration" json:"period,omitempty"` +} + +func (m *RepeatDetails) Reset() { *m = RepeatDetails{} } +func (m *RepeatDetails) String() string { return proto.CompactTextString(m) } +func (*RepeatDetails) ProtoMessage() {} +func (*RepeatDetails) Descriptor() ([]byte, []int) { + return fileDescriptor_63baccdfe9127510, []int{6} +} +func (m *RepeatDetails) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RepeatDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RepeatDetails.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RepeatDetails) XXX_Merge(src proto.Message) { + xxx_messageInfo_RepeatDetails.Merge(m, src) +} +func (m *RepeatDetails) XXX_Size() int { + return m.Size() +} +func (m *RepeatDetails) XXX_DiscardUnknown() { + xxx_messageInfo_RepeatDetails.DiscardUnknown(m) +} + +var xxx_messageInfo_RepeatDetails proto.InternalMessageInfo + +func (m *RepeatDetails) GetNumTimes() uint32 { + if m != nil { + return m.NumTimes + } + return 0 +} + +func (m *RepeatDetails) GetPeriod() *time.Duration { + if m != nil { + return m.Period + } + return nil +} + type ShiftedExponential struct { Minimum time.Duration `protobuf:"bytes,1,opt,name=minimum,proto3,stdduration" json:"minimum"` TailMean time.Duration `protobuf:"bytes,2,opt,name=tail_mean,json=tailMean,proto3,stdduration" json:"tailMean"` @@ -526,7 +589,7 @@ func (m *ShiftedExponential) Reset() { *m = ShiftedExponential{} } func (m *ShiftedExponential) String() string { return proto.CompactTextString(m) } func (*ShiftedExponential) ProtoMessage() {} func (*ShiftedExponential) Descriptor() ([]byte, []int) { - return fileDescriptor_63baccdfe9127510, []int{6} + return fileDescriptor_63baccdfe9127510, []int{7} } func (m *ShiftedExponential) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -577,6 +640,7 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "simulator.NodeTemplate.LabelsEntry") proto.RegisterType((*Queue)(nil), "simulator.Queue") proto.RegisterType((*JobTemplate)(nil), "simulator.JobTemplate") + proto.RegisterType((*RepeatDetails)(nil), "simulator.RepeatDetails") proto.RegisterType((*ShiftedExponential)(nil), "simulator.ShiftedExponential") } @@ -585,84 +649,89 @@ func init() { } var fileDescriptor_63baccdfe9127510 = []byte{ - // 1228 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x56, 0xcf, 0x6e, 0x13, 0x47, - 0x18, 0xcf, 0xc6, 0xe0, 0xc4, 0x63, 0x27, 0x84, 0x49, 0x14, 0x8c, 0x11, 0x5e, 0x63, 0x54, 0xe4, - 0x56, 0x61, 0x2d, 0xa8, 0x54, 0x51, 0x54, 0x21, 0x75, 0x13, 0x10, 0x42, 0x40, 0xc1, 0xa1, 0x45, - 0x2a, 0x87, 0xd5, 0xec, 0xee, 0x17, 0x67, 0xe2, 0xdd, 0x1d, 0x33, 0x3b, 0x0b, 0xf5, 0xa1, 0xef, - 0xd0, 0x4b, 0x51, 0x1f, 0xa2, 0x97, 0x5e, 0x78, 0x06, 0x8e, 0x1c, 0x7b, 0xda, 0x56, 0x70, 0xdb, - 0x27, 0xe8, 0xb1, 0xda, 0xd9, 0x59, 0x67, 0x8c, 0x03, 0x24, 0x27, 0xfb, 0xfb, 0x7d, 0x7f, 0xf6, - 0x37, 0xdf, 0xbf, 0x19, 0xb4, 0x45, 0x23, 0x01, 0x3c, 0x22, 0x41, 0x3f, 0xf6, 0xf6, 0xc1, 0x4f, - 0x02, 0xe0, 0xfd, 0x98, 0x86, 0x49, 0x40, 0x04, 0xd3, 0xfe, 0x59, 0x63, 0xce, 0x04, 0xc3, 0xb5, - 0x29, 0xd0, 0x6a, 0x0f, 0x19, 0x1b, 0x06, 0xd0, 0x97, 0x0a, 0x37, 0xd9, 0xeb, 0xfb, 0x09, 0x27, - 0x82, 0xb2, 0xa8, 0x30, 0x6d, 0x75, 0x47, 0x37, 0x62, 0x8b, 0xb2, 0x3e, 0x19, 0xd3, 0xbe, 0xc7, - 0x38, 0xf4, 0x5f, 0x5c, 0xeb, 0x0f, 0x21, 0x02, 0x4e, 0x04, 0xf8, 0xca, 0xe6, 0xea, 0x90, 0x8a, - 0xfd, 0xc4, 0xb5, 0x3c, 0x16, 0xf6, 0x87, 0x6c, 0xc8, 0x0e, 0x83, 0xe5, 0x92, 0x14, 0xe4, 0x3f, - 0x65, 0x7e, 0xf3, 0x28, 0xae, 0xe5, 0x3f, 0xe6, 0x1e, 0x80, 0x27, 0xe2, 0x39, 0xa0, 0xf0, 0xed, - 0xfe, 0x5e, 0x41, 0xf5, 0xed, 0x20, 0x89, 0x05, 0xf0, 0xdd, 0x31, 0x78, 0xf8, 0x0a, 0x3a, 0x15, - 0x91, 0x10, 0x9a, 0x46, 0xc7, 0xe8, 0xd5, 0x6c, 0x9c, 0xa5, 0xe6, 0x6a, 0x2e, 0x6f, 0xb1, 0x90, - 0x0a, 0x08, 0xc7, 0x62, 0x32, 0x90, 0x7a, 0x7c, 0x07, 0x2d, 0x7b, 0x85, 0x5b, 0xdc, 0x5c, 0xec, - 0x54, 0x7a, 0xf5, 0xeb, 0xd8, 0x3a, 0xcc, 0x8a, 0x8a, 0x68, 0x6f, 0x66, 0xa9, 0x89, 0x4b, 0x3b, - 0x2d, 0xc6, 0xd4, 0x17, 0xbf, 0x32, 0xd0, 0xe5, 0x97, 0x8c, 0x8f, 0xf6, 0x02, 0xf6, 0xd2, 0x09, - 0x49, 0x44, 0x86, 0xc0, 0x1d, 0x1f, 0x02, 0x32, 0x71, 0x7c, 0x1a, 0x0b, 0x4e, 0xdd, 0x24, 0x4f, - 0x5e, 0xb3, 0xd2, 0x31, 0x7a, 0xf5, 0xeb, 0x17, 0xb5, 0x6f, 0xec, 0xee, 0xd3, 0x3d, 0x01, 0xfe, - 0xed, 0x5f, 0xc6, 0x2c, 0x82, 0x48, 0x50, 0x12, 0xd8, 0xbd, 0x37, 0xa9, 0xb9, 0x90, 0xa5, 0x66, - 0xa7, 0x8c, 0xf8, 0xa0, 0x08, 0xb8, 0x93, 0xc7, 0xdb, 0xd1, 0xc2, 0x0d, 0x3e, 0x6b, 0x81, 0x7f, - 0x45, 0xad, 0x31, 0x44, 0x3e, 0x8d, 0x86, 0x47, 0xd1, 0x39, 0x75, 0x1c, 0x3a, 0x1d, 0x45, 0xa7, - 0xa9, 0x02, 0xcd, 0xd3, 0xf8, 0xa8, 0xa6, 0xfb, 0xda, 0x40, 0x4b, 0x2a, 0x8b, 0xc7, 0xae, 0xc9, - 0x15, 0x74, 0x6a, 0xcc, 0x58, 0x20, 0x73, 0xa5, 0xec, 0x72, 0x59, 0xb7, 0xcb, 0x65, 0xfc, 0x0c, - 0xad, 0x46, 0xcc, 0x07, 0x27, 0x07, 0x03, 0x22, 0xa0, 0xac, 0xe0, 0x39, 0xed, 0x38, 0x0f, 0x99, - 0x0f, 0x4f, 0x94, 0xde, 0xbe, 0x90, 0xa5, 0xe6, 0xb9, 0x48, 0x43, 0xf4, 0x5a, 0xae, 0xcc, 0x28, - 0xba, 0x7f, 0x19, 0xa8, 0xf1, 0x94, 0xf1, 0x51, 0xc0, 0x88, 0x7f, 0xa2, 0x8e, 0xfa, 0x16, 0xd5, - 0x39, 0x89, 0x7c, 0x16, 0x3a, 0x31, 0x80, 0xdf, 0x5c, 0xec, 0x18, 0xbd, 0x8a, 0xdd, 0xcc, 0x52, - 0x73, 0xa3, 0x80, 0x77, 0x01, 0x7c, 0xcd, 0x09, 0x1d, 0xa2, 0xf8, 0x16, 0xaa, 0x3e, 0x4f, 0x20, - 0x81, 0xb8, 0x59, 0x91, 0x07, 0x59, 0xd3, 0x0e, 0xf2, 0x38, 0x57, 0xd8, 0x1b, 0x59, 0x6a, 0xae, - 0x15, 0x36, 0x5a, 0x0c, 0xe5, 0xd5, 0x7d, 0x55, 0x41, 0x0d, 0xfd, 0xc0, 0x78, 0x0b, 0x55, 0xa3, - 0x24, 0x74, 0x81, 0x4b, 0xd6, 0x95, 0xc2, 0xbd, 0x40, 0x74, 0xf7, 0x02, 0xc1, 0xdf, 0xa3, 0xaa, - 0x20, 0x34, 0x12, 0x65, 0x1e, 0xcf, 0x5b, 0xc5, 0x8c, 0x5b, 0x64, 0x4c, 0xad, 0x7c, 0xc6, 0xad, - 0x17, 0xd7, 0xac, 0x27, 0xb9, 0x85, 0xbd, 0xaa, 0x5a, 0x42, 0x39, 0x0c, 0xd4, 0x2f, 0x7e, 0x8c, - 0xaa, 0x01, 0x71, 0x21, 0x28, 0x4f, 0x70, 0xf9, 0x23, 0xa5, 0xb0, 0xee, 0x4b, 0xab, 0xdb, 0x91, - 0xe0, 0x93, 0x82, 0x55, 0xe1, 0xa6, 0xb3, 0x2a, 0x10, 0xec, 0xa0, 0x33, 0x82, 0x09, 0x12, 0x38, - 0x1c, 0x62, 0x96, 0x70, 0x0f, 0x62, 0xd5, 0xb5, 0x6d, 0x6b, 0x6e, 0x17, 0x0c, 0x94, 0xc9, 0x7d, - 0x1a, 0x0b, 0x7b, 0x53, 0x71, 0x5c, 0x95, 0xee, 0xa5, 0x2a, 0x1e, 0x7c, 0x20, 0xb7, 0x08, 0xaa, - 0x6b, 0x6c, 0xf0, 0x65, 0x54, 0x19, 0xc1, 0x44, 0x95, 0xf9, 0x6c, 0x96, 0x9a, 0x2b, 0x23, 0x98, - 0x68, 0xbc, 0x72, 0x2d, 0xfe, 0x12, 0x9d, 0x7e, 0x41, 0x82, 0x04, 0x64, 0x79, 0x6b, 0xf6, 0x7a, - 0x96, 0x9a, 0x67, 0x24, 0xa0, 0x19, 0x16, 0x16, 0x37, 0x17, 0x6f, 0x18, 0xf9, 0x14, 0x9c, 0x96, - 0x05, 0x3c, 0x76, 0x17, 0x6d, 0xa1, 0xea, 0x4b, 0xa0, 0xc3, 0x7d, 0x21, 0xbf, 0x60, 0x14, 0x39, - 0x2a, 0x10, 0x3d, 0x47, 0x05, 0x82, 0x9f, 0xa2, 0x95, 0x03, 0xe6, 0x6a, 0x83, 0x50, 0x64, 0x7f, - 0x53, 0xcb, 0xfe, 0x3d, 0xe6, 0x4e, 0xe7, 0xa0, 0x95, 0xa5, 0xe6, 0xe6, 0xc1, 0x21, 0xa0, 0xa7, - 0xbd, 0xa1, 0xe3, 0xdd, 0xff, 0x96, 0x51, 0x5d, 0xf3, 0x3c, 0x61, 0x43, 0xdd, 0x43, 0x4a, 0xb7, - 0x9b, 0x78, 0x1e, 0xc4, 0xf1, 0x5e, 0x12, 0xa8, 0x79, 0x68, 0x67, 0xa9, 0xd9, 0xfa, 0x50, 0xa7, - 0x45, 0x98, 0xf3, 0xcb, 0x33, 0x2e, 0xbb, 0x5c, 0x6d, 0x05, 0x99, 0x71, 0x09, 0xe8, 0x19, 0x97, - 0x00, 0xee, 0xa0, 0x45, 0xea, 0xcb, 0x26, 0xa9, 0xd9, 0x6b, 0x59, 0x6a, 0x36, 0xa8, 0x3e, 0x70, - 0x8b, 0xd4, 0xc7, 0x57, 0xd1, 0x52, 0x9e, 0xaf, 0x18, 0x44, 0xf3, 0xb4, 0x34, 0x93, 0xe7, 0x38, - 0x60, 0xee, 0x2e, 0xcc, 0xa4, 0xb7, 0x40, 0xb0, 0x8d, 0x56, 0x65, 0x64, 0x67, 0xcc, 0x29, 0xe3, - 0x54, 0x4c, 0x9a, 0xd5, 0x8e, 0xd1, 0x5b, 0x29, 0xf6, 0x89, 0xd4, 0x3c, 0x52, 0x0a, 0x7d, 0x9f, - 0xcc, 0x28, 0xf0, 0x0f, 0x68, 0xbd, 0xf4, 0x76, 0xbc, 0x80, 0xc4, 0xb1, 0x23, 0xfb, 0x60, 0x49, - 0x7e, 0xde, 0xcc, 0x52, 0xf3, 0x42, 0xa9, 0xde, 0xce, 0xb5, 0x0f, 0x67, 0x9b, 0xe2, 0xec, 0x9c, - 0x12, 0x3f, 0x43, 0x0d, 0x0e, 0xcf, 0x13, 0xca, 0x21, 0x84, 0x7c, 0x66, 0x97, 0xe5, 0x50, 0x5c, - 0x9a, 0x1f, 0x8a, 0x47, 0xcc, 0x1f, 0x68, 0x86, 0xf6, 0x86, 0x9a, 0x8b, 0x19, 0xf7, 0xc1, 0x8c, - 0x84, 0x6f, 0xa1, 0x86, 0x0f, 0xf9, 0x52, 0x87, 0xc8, 0xa3, 0x10, 0x37, 0x6b, 0x9d, 0x4a, 0xaf, - 0x56, 0xf4, 0x8d, 0x8e, 0xeb, 0x7d, 0xa3, 0xe3, 0x78, 0x84, 0x36, 0x80, 0xf0, 0x80, 0x42, 0x2c, - 0x9c, 0x38, 0x71, 0x43, 0x2a, 0x1c, 0x41, 0x43, 0x68, 0x22, 0x49, 0xf2, 0xbc, 0x55, 0x3c, 0x2e, - 0xac, 0xf2, 0x3d, 0x60, 0xed, 0xa8, 0xc7, 0x85, 0xdd, 0x56, 0xe4, 0x70, 0xe9, 0xbe, 0x2b, 0xbd, - 0x9f, 0xd0, 0x10, 0xfe, 0xf8, 0xc7, 0x34, 0x06, 0x47, 0xe0, 0xf8, 0xb5, 0x81, 0xfa, 0x47, 0x7d, - 0xcd, 0xd9, 0xe3, 0x2c, 0x74, 0xa6, 0xbc, 0x26, 0x8e, 0xc7, 0xc2, 0x71, 0x00, 0xf2, 0xe2, 0xab, - 0x7f, 0x8e, 0xc8, 0x37, 0x8a, 0xc8, 0x57, 0xf3, 0x1f, 0xbc, 0xc3, 0x59, 0xb8, 0x33, 0x8d, 0xba, - 0x3d, 0x0d, 0x2a, 0x09, 0x9e, 0xc0, 0x1e, 0x87, 0x68, 0x83, 0x27, 0x91, 0xa4, 0x3a, 0x73, 0x2b, - 0x37, 0x8e, 0x73, 0x2b, 0x5f, 0x50, 0x04, 0xd7, 0x55, 0x88, 0x99, 0x0b, 0xf9, 0x28, 0x10, 0xdf, - 0x45, 0x6b, 0x43, 0x12, 0x0d, 0x1d, 0x8f, 0x70, 0x9f, 0x46, 0x24, 0xc8, 0x1b, 0x79, 0x45, 0x36, - 0xf2, 0xc5, 0x2c, 0x35, 0xcf, 0xe7, 0xba, 0xed, 0x43, 0x95, 0x56, 0xdb, 0x33, 0x1f, 0xa8, 0xb0, - 0x8b, 0x5a, 0x32, 0x92, 0xbc, 0x7e, 0x93, 0x88, 0xee, 0x31, 0x1e, 0xe6, 0x8d, 0x2d, 0x57, 0x76, - 0x73, 0x55, 0xf6, 0xf4, 0x17, 0x59, 0x6a, 0x5e, 0xca, 0xad, 0xf2, 0xbd, 0xff, 0xe3, 0xd4, 0x46, - 0xae, 0x5a, 0x2d, 0xf6, 0xb9, 0x8f, 0x98, 0x74, 0xff, 0x34, 0x10, 0x9e, 0x3f, 0x36, 0xbe, 0x8b, - 0x96, 0x42, 0x1a, 0xd1, 0x30, 0x09, 0xe5, 0x0a, 0xfa, 0x64, 0x0d, 0xd7, 0x55, 0x8a, 0x4a, 0x0f, - 0x59, 0xa0, 0x52, 0xc0, 0xf7, 0x51, 0x4d, 0x10, 0x1a, 0x38, 0x21, 0x90, 0x48, 0xae, 0xa5, 0x4f, - 0xc6, 0x2a, 0xa7, 0x66, 0x39, 0xf7, 0x79, 0x00, 0xa4, 0xa8, 0xf6, 0x54, 0xb2, 0x7f, 0x7a, 0xf3, - 0xae, 0x6d, 0xbc, 0x7d, 0xd7, 0x36, 0xfe, 0x7d, 0xd7, 0x36, 0x7e, 0x7b, 0xdf, 0x5e, 0x78, 0xfb, - 0xbe, 0xbd, 0xf0, 0xf7, 0xfb, 0xf6, 0xc2, 0xcf, 0xdf, 0x69, 0xaf, 0x60, 0xc2, 0x43, 0xe2, 0x93, - 0x31, 0x67, 0xf9, 0x68, 0x2a, 0xa9, 0xff, 0xa9, 0x27, 0xba, 0x5b, 0x95, 0x54, 0xbe, 0xfe, 0x3f, - 0x00, 0x00, 0xff, 0xff, 0xb2, 0x34, 0xbe, 0x7a, 0xc9, 0x0b, 0x00, 0x00, + // 1310 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x57, 0x41, 0x6f, 0x13, 0xc7, + 0x17, 0xcf, 0xc6, 0xe0, 0xe0, 0x49, 0x9c, 0x84, 0x49, 0x14, 0x16, 0x23, 0xbc, 0xc6, 0xe8, 0x8f, + 0xfc, 0xaf, 0xc2, 0x5a, 0x80, 0x54, 0x51, 0x54, 0x21, 0x75, 0x13, 0x10, 0xa2, 0x40, 0xc1, 0xa1, + 0x45, 0x2a, 0x87, 0xd5, 0xd8, 0xfb, 0xe2, 0x4c, 0xb2, 0xbb, 0x63, 0x66, 0x67, 0xa1, 0x3e, 0xf4, + 0x3b, 0xb4, 0x87, 0xa2, 0x7e, 0x88, 0x5e, 0x7a, 0xe1, 0xdc, 0x23, 0x47, 0x8e, 0x3d, 0x6d, 0x2b, + 0xb8, 0xed, 0xa7, 0xa8, 0x76, 0x66, 0xd6, 0x19, 0xe3, 0x00, 0xe1, 0x14, 0xcf, 0xef, 0xfd, 0xde, + 0xdb, 0xdf, 0xbc, 0x79, 0xef, 0xcd, 0x04, 0x6d, 0xd2, 0x58, 0x00, 0x8f, 0x49, 0xd8, 0x4d, 0x06, + 0x7b, 0x10, 0xa4, 0x21, 0xf0, 0x6e, 0x42, 0xa3, 0x34, 0x24, 0x82, 0x19, 0xbf, 0xdc, 0x11, 0x67, + 0x82, 0xe1, 0xda, 0x04, 0x68, 0x34, 0x87, 0x8c, 0x0d, 0x43, 0xe8, 0x4a, 0x43, 0x3f, 0xdd, 0xed, + 0x06, 0x29, 0x27, 0x82, 0xb2, 0x58, 0x51, 0x1b, 0xed, 0x83, 0xeb, 0x89, 0x4b, 0x59, 0x97, 0x8c, + 0x68, 0x77, 0xc0, 0x38, 0x74, 0x9f, 0x5f, 0xe9, 0x0e, 0x21, 0x06, 0x4e, 0x04, 0x04, 0x9a, 0x73, + 0x79, 0x48, 0xc5, 0x5e, 0xda, 0x77, 0x07, 0x2c, 0xea, 0x0e, 0xd9, 0x90, 0x1d, 0x06, 0x2b, 0x56, + 0x72, 0x21, 0x7f, 0x69, 0xfa, 0x8d, 0xa3, 0xb4, 0x96, 0xbf, 0x58, 0x7f, 0x1f, 0x06, 0x22, 0x99, + 0x01, 0x94, 0x6f, 0xfb, 0xb7, 0x0a, 0x5a, 0xdc, 0x0a, 0xd3, 0x44, 0x00, 0xdf, 0x19, 0xc1, 0x00, + 0x5f, 0x42, 0x27, 0x62, 0x12, 0x81, 0x6d, 0xb5, 0xac, 0x4e, 0xcd, 0xc3, 0x79, 0xe6, 0x2c, 0x17, + 0xeb, 0x4d, 0x16, 0x51, 0x01, 0xd1, 0x48, 0x8c, 0x7b, 0xd2, 0x8e, 0x6f, 0xa3, 0x53, 0x03, 0xe5, + 0x96, 0xd8, 0xf3, 0xad, 0x4a, 0x67, 0xf1, 0x2a, 0x76, 0x0f, 0xb3, 0xa2, 0x23, 0x7a, 0x1b, 0x79, + 0xe6, 0xe0, 0x92, 0x67, 0xc4, 0x98, 0xf8, 0xe2, 0x97, 0x16, 0xba, 0xf8, 0x82, 0xf1, 0x83, 0xdd, + 0x90, 0xbd, 0xf0, 0x23, 0x12, 0x93, 0x21, 0x70, 0x3f, 0x80, 0x90, 0x8c, 0xfd, 0x80, 0x26, 0x82, + 0xd3, 0x7e, 0x5a, 0x24, 0xcf, 0xae, 0xb4, 0xac, 0xce, 0xe2, 0xd5, 0xf3, 0xc6, 0x37, 0x76, 0xf6, + 0xe8, 0xae, 0x80, 0xe0, 0xd6, 0x4f, 0x23, 0x16, 0x43, 0x2c, 0x28, 0x09, 0xbd, 0xce, 0xeb, 0xcc, + 0x99, 0xcb, 0x33, 0xa7, 0x55, 0x46, 0xbc, 0xaf, 0x02, 0x6e, 0x17, 0xf1, 0xb6, 0x8d, 0x70, 0xbd, + 0x4f, 0x32, 0xf0, 0xcf, 0xa8, 0x31, 0x82, 0x38, 0xa0, 0xf1, 0xf0, 0x28, 0x39, 0x27, 0x8e, 0x23, + 0xa7, 0xa5, 0xe5, 0xd8, 0x3a, 0xd0, 0xac, 0x8c, 0x0f, 0x5a, 0xda, 0xaf, 0x2c, 0xb4, 0xa0, 0xb3, + 0x78, 0xec, 0x33, 0xb9, 0x84, 0x4e, 0x8c, 0x18, 0x0b, 0x65, 0xae, 0x34, 0xaf, 0x58, 0x9b, 0xbc, + 0x62, 0x8d, 0x9f, 0xa2, 0xe5, 0x98, 0x05, 0xe0, 0x17, 0x60, 0x48, 0x04, 0x94, 0x27, 0x78, 0xc6, + 0xd8, 0xce, 0x03, 0x16, 0xc0, 0x63, 0x6d, 0xf7, 0xce, 0xe5, 0x99, 0x73, 0x26, 0x36, 0x10, 0xf3, + 0x2c, 0xeb, 0x53, 0x86, 0xf6, 0x9f, 0x16, 0x5a, 0x7a, 0xc2, 0xf8, 0x41, 0xc8, 0x48, 0xf0, 0x59, + 0x15, 0xf5, 0x15, 0x5a, 0xe4, 0x24, 0x0e, 0x58, 0xe4, 0x27, 0x00, 0x81, 0x3d, 0xdf, 0xb2, 0x3a, + 0x15, 0xcf, 0xce, 0x33, 0x67, 0x5d, 0xc1, 0x3b, 0x00, 0x81, 0xe1, 0x84, 0x0e, 0x51, 0x7c, 0x13, + 0x55, 0x9f, 0xa5, 0x90, 0x42, 0x62, 0x57, 0xe4, 0x46, 0x56, 0x8d, 0x8d, 0x3c, 0x2a, 0x0c, 0xde, + 0x7a, 0x9e, 0x39, 0xab, 0x8a, 0x63, 0xc4, 0xd0, 0x5e, 0xed, 0x97, 0x15, 0xb4, 0x64, 0x6e, 0x18, + 0x6f, 0xa2, 0x6a, 0x9c, 0x46, 0x7d, 0xe0, 0x52, 0x75, 0x45, 0xb9, 0x2b, 0xc4, 0x74, 0x57, 0x08, + 0xfe, 0x06, 0x55, 0x05, 0xa1, 0xb1, 0x28, 0xf3, 0x78, 0xd6, 0x55, 0x3d, 0xee, 0x92, 0x11, 0x75, + 0x8b, 0x1e, 0x77, 0x9f, 0x5f, 0x71, 0x1f, 0x17, 0x0c, 0x6f, 0x59, 0x97, 0x84, 0x76, 0xe8, 0xe9, + 0xbf, 0xf8, 0x11, 0xaa, 0x86, 0xa4, 0x0f, 0x61, 0xb9, 0x83, 0x8b, 0x1f, 0x38, 0x0a, 0xf7, 0x9e, + 0x64, 0xdd, 0x8a, 0x05, 0x1f, 0x2b, 0x55, 0xca, 0xcd, 0x54, 0xa5, 0x10, 0xec, 0xa3, 0x15, 0xc1, + 0x04, 0x09, 0x7d, 0x0e, 0x09, 0x4b, 0xf9, 0x00, 0x12, 0x5d, 0xb5, 0x4d, 0x77, 0x66, 0x16, 0xf4, + 0x34, 0xe5, 0x1e, 0x4d, 0x84, 0xb7, 0xa1, 0x35, 0x2e, 0x4b, 0xf7, 0xd2, 0x94, 0xf4, 0xde, 0x5b, + 0x37, 0x08, 0x5a, 0x34, 0xd4, 0xe0, 0x8b, 0xa8, 0x72, 0x00, 0x63, 0x7d, 0xcc, 0xa7, 0xf3, 0xcc, + 0xa9, 0x1f, 0xc0, 0xd8, 0xd0, 0x55, 0x58, 0xf1, 0xff, 0xd1, 0xc9, 0xe7, 0x24, 0x4c, 0x41, 0x1e, + 0x6f, 0xcd, 0x5b, 0xcb, 0x33, 0x67, 0x45, 0x02, 0x06, 0x51, 0x31, 0x6e, 0xcc, 0x5f, 0xb7, 0x8a, + 0x2e, 0x38, 0x29, 0x0f, 0xf0, 0xd8, 0x55, 0xb4, 0x89, 0xaa, 0x2f, 0x80, 0x0e, 0xf7, 0x84, 0xfc, + 0x82, 0xa5, 0x72, 0xa4, 0x10, 0x33, 0x47, 0x0a, 0xc1, 0x4f, 0x50, 0x7d, 0x9f, 0xf5, 0x8d, 0x46, + 0x50, 0xd9, 0xdf, 0x30, 0xb2, 0x7f, 0x97, 0xf5, 0x27, 0x7d, 0xd0, 0xc8, 0x33, 0x67, 0x63, 0xff, + 0x10, 0x30, 0xd3, 0xbe, 0x64, 0xe2, 0xed, 0xbf, 0x6a, 0x68, 0xd1, 0xf0, 0xfc, 0xcc, 0x82, 0xba, + 0x8b, 0xb4, 0x6d, 0x27, 0x1d, 0x0c, 0x20, 0x49, 0x76, 0xd3, 0x50, 0xf7, 0x43, 0x33, 0xcf, 0x9c, + 0xc6, 0xfb, 0x36, 0x23, 0xc2, 0x8c, 0x5f, 0x91, 0x71, 0x59, 0xe5, 0x7a, 0x2a, 0xc8, 0x8c, 0x4b, + 0xc0, 0xcc, 0xb8, 0x04, 0x70, 0x0b, 0xcd, 0xd3, 0x40, 0x16, 0x49, 0xcd, 0x5b, 0xcd, 0x33, 0x67, + 0x89, 0x9a, 0x0d, 0x37, 0x4f, 0x03, 0x7c, 0x19, 0x2d, 0x14, 0xf9, 0x4a, 0x40, 0xd8, 0x27, 0x25, + 0x4d, 0xee, 0x63, 0x9f, 0xf5, 0x77, 0x60, 0x2a, 0xbd, 0x0a, 0xc1, 0x1e, 0x5a, 0x96, 0x91, 0xfd, + 0x11, 0xa7, 0x8c, 0x53, 0x31, 0xb6, 0xab, 0x2d, 0xab, 0x53, 0x57, 0xf3, 0x44, 0x5a, 0x1e, 0x6a, + 0x83, 0x39, 0x4f, 0xa6, 0x0c, 0xf8, 0x3b, 0xb4, 0x56, 0x7a, 0xfb, 0x83, 0x90, 0x24, 0x89, 0x2f, + 0xeb, 0x60, 0x41, 0x7e, 0xde, 0xc9, 0x33, 0xe7, 0x5c, 0x69, 0xde, 0x2a, 0xac, 0x0f, 0xa6, 0x8b, + 0xe2, 0xf4, 0x8c, 0x11, 0x3f, 0x45, 0x4b, 0x1c, 0x9e, 0xa5, 0x94, 0x43, 0x04, 0x45, 0xcf, 0x9e, + 0x92, 0x4d, 0x71, 0x61, 0xb6, 0x29, 0x1e, 0xb2, 0xa0, 0x67, 0x10, 0xbd, 0x75, 0xdd, 0x17, 0x53, + 0xee, 0xbd, 0xa9, 0x15, 0xbe, 0x89, 0x96, 0x02, 0x28, 0x86, 0x3a, 0xc4, 0x03, 0x0a, 0x89, 0x5d, + 0x6b, 0x55, 0x3a, 0x35, 0x55, 0x37, 0x26, 0x6e, 0xd6, 0x8d, 0x89, 0xe3, 0x03, 0xb4, 0x0e, 0x84, + 0x87, 0x14, 0x12, 0xe1, 0x27, 0x69, 0x3f, 0xa2, 0xc2, 0x17, 0x34, 0x02, 0x1b, 0x49, 0x91, 0x67, + 0x5d, 0xf5, 0xb8, 0x70, 0xcb, 0xf7, 0x80, 0xbb, 0xad, 0x1f, 0x17, 0x5e, 0x53, 0x8b, 0xc3, 0xa5, + 0xfb, 0x8e, 0xf4, 0x7e, 0x4c, 0x23, 0xf8, 0xfd, 0x1f, 0xc7, 0xea, 0x1d, 0x81, 0xe3, 0x57, 0x16, + 0xea, 0x1e, 0xf5, 0x35, 0x7f, 0x97, 0xb3, 0xc8, 0x9f, 0xe8, 0x1a, 0xfb, 0x03, 0x16, 0x8d, 0x42, + 0x90, 0x17, 0xdf, 0xe2, 0xa7, 0x84, 0x7c, 0xa9, 0x85, 0x7c, 0x31, 0xfb, 0xc1, 0xdb, 0x9c, 0x45, + 0xdb, 0x93, 0xa8, 0x5b, 0x93, 0xa0, 0x52, 0xe0, 0x67, 0xf0, 0x71, 0x84, 0xd6, 0x79, 0x1a, 0x4b, + 0xa9, 0x53, 0xb7, 0xf2, 0xd2, 0x71, 0x6e, 0xe5, 0x73, 0x5a, 0xe0, 0x9a, 0x0e, 0x31, 0x75, 0x21, + 0x1f, 0x05, 0xe2, 0x3b, 0x68, 0x75, 0x48, 0xe2, 0xa1, 0x3f, 0x20, 0x3c, 0xa0, 0x31, 0x09, 0x8b, + 0x42, 0xae, 0xcb, 0x42, 0x3e, 0x9f, 0x67, 0xce, 0xd9, 0xc2, 0xb6, 0x75, 0x68, 0x32, 0xce, 0x76, + 0xe5, 0x3d, 0x13, 0xee, 0xa3, 0x86, 0x8c, 0x24, 0xaf, 0xdf, 0x34, 0xa6, 0xbb, 0x8c, 0x47, 0x45, + 0x61, 0xcb, 0x91, 0x6d, 0x2f, 0xcb, 0x9a, 0xfe, 0x5f, 0x9e, 0x39, 0x17, 0x0a, 0x56, 0x31, 0xf7, + 0xbf, 0x9f, 0x70, 0xe4, 0xa8, 0x35, 0x62, 0x9f, 0xf9, 0x00, 0x05, 0xdf, 0x46, 0x55, 0x0e, 0x23, + 0x20, 0xc2, 0x5e, 0x91, 0xe9, 0xb0, 0x8d, 0x74, 0xf4, 0xa4, 0x61, 0x1b, 0x04, 0xa1, 0x61, 0xa2, + 0x9a, 0x57, 0x71, 0xcd, 0xe6, 0x55, 0x48, 0xfb, 0x57, 0x0b, 0xd5, 0xa7, 0xf8, 0xf8, 0x1a, 0xaa, + 0xc5, 0x69, 0x24, 0x4b, 0x24, 0x91, 0x73, 0xac, 0xae, 0x1e, 0x78, 0x71, 0x1a, 0x15, 0xc7, 0x35, + 0xf5, 0xc0, 0x2b, 0x31, 0xfc, 0x2d, 0xaa, 0x8e, 0x80, 0x53, 0xa6, 0x6e, 0xf4, 0x8f, 0x96, 0x4e, + 0x71, 0xd9, 0xaf, 0x2a, 0xf2, 0x61, 0x28, 0x59, 0x1c, 0x3a, 0x44, 0xfb, 0x0f, 0x0b, 0xe1, 0xd9, + 0x23, 0xc5, 0x77, 0xd0, 0x42, 0x44, 0x63, 0x1a, 0xa5, 0x91, 0x94, 0xf5, 0xd1, 0x8f, 0xac, 0xe9, + 0xe3, 0x2f, 0x3d, 0x64, 0xfc, 0x72, 0x81, 0xef, 0xa1, 0x5a, 0xb1, 0x57, 0x3f, 0x02, 0x12, 0x7f, + 0x5a, 0x70, 0x39, 0x11, 0x4e, 0x15, 0x3e, 0xf7, 0x81, 0xa8, 0x4a, 0x9e, 0xac, 0xbc, 0x1f, 0x5e, + 0xbf, 0x6d, 0x5a, 0x6f, 0xde, 0x36, 0xad, 0x7f, 0xdf, 0x36, 0xad, 0x5f, 0xde, 0x35, 0xe7, 0xde, + 0xbc, 0x6b, 0xce, 0xfd, 0xfd, 0xae, 0x39, 0xf7, 0xe3, 0xd7, 0xc6, 0x0b, 0x9f, 0xf0, 0x88, 0x04, + 0x64, 0xc4, 0x59, 0x31, 0x76, 0xf4, 0xaa, 0xfb, 0xb1, 0x7f, 0x3f, 0xfa, 0x55, 0x29, 0xe5, 0xda, + 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb1, 0x00, 0x01, 0x6c, 0xa5, 0x0c, 0x00, 0x00, } func (m *ClusterSpec) Marshal() (dAtA []byte, err error) { @@ -970,6 +1039,18 @@ func (m *JobTemplate) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Repeat != nil { + { + size, err := m.Repeat.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintSimulator(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x7a + } if len(m.GangNodeUniformityLabel) > 0 { i -= len(m.GangNodeUniformityLabel) copy(dAtA[i:], m.GangNodeUniformityLabel) @@ -992,21 +1073,21 @@ func (m *JobTemplate) MarshalToSizedBuffer(dAtA []byte) (int, error) { } i-- dAtA[i] = 0x62 - n5, err5 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.EarliestSubmitTimeFromDependencyCompletion, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.EarliestSubmitTimeFromDependencyCompletion):]) - if err5 != nil { - return 0, err5 - } - i -= n5 - i = encodeVarintSimulator(dAtA, i, uint64(n5)) - i-- - dAtA[i] = 0x5a - n6, err6 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.EarliestSubmitTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.EarliestSubmitTime):]) + n6, err6 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.EarliestSubmitTimeFromDependencyCompletion, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.EarliestSubmitTimeFromDependencyCompletion):]) if err6 != nil { return 0, err6 } i -= n6 i = encodeVarintSimulator(dAtA, i, uint64(n6)) i-- + dAtA[i] = 0x5a + n7, err7 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.EarliestSubmitTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.EarliestSubmitTime):]) + if err7 != nil { + return 0, err7 + } + i -= n7 + i = encodeVarintSimulator(dAtA, i, uint64(n7)) + i-- dAtA[i] = 0x52 if len(m.Dependencies) > 0 { for iNdEx := len(m.Dependencies) - 1; iNdEx >= 0; iNdEx-- { @@ -1073,6 +1154,44 @@ func (m *JobTemplate) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *RepeatDetails) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RepeatDetails) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RepeatDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.Period != nil { + n9, err9 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.Period, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Period):]) + if err9 != nil { + return 0, err9 + } + i -= n9 + i = encodeVarintSimulator(dAtA, i, uint64(n9)) + i-- + dAtA[i] = 0x12 + } + if m.NumTimes != 0 { + i = encodeVarintSimulator(dAtA, i, uint64(m.NumTimes)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *ShiftedExponential) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1093,20 +1212,20 @@ func (m *ShiftedExponential) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - n8, err8 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.TailMean, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.TailMean):]) - if err8 != nil { - return 0, err8 + n10, err10 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.TailMean, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.TailMean):]) + if err10 != nil { + return 0, err10 } - i -= n8 - i = encodeVarintSimulator(dAtA, i, uint64(n8)) + i -= n10 + i = encodeVarintSimulator(dAtA, i, uint64(n10)) i-- dAtA[i] = 0x12 - n9, err9 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.Minimum, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.Minimum):]) - if err9 != nil { - return 0, err9 + n11, err11 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.Minimum, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.Minimum):]) + if err11 != nil { + return 0, err11 } - i -= n9 - i = encodeVarintSimulator(dAtA, i, uint64(n9)) + i -= n11 + i = encodeVarintSimulator(dAtA, i, uint64(n11)) i-- dAtA[i] = 0xa return len(dAtA) - i, nil @@ -1293,6 +1412,26 @@ func (m *JobTemplate) Size() (n int) { if l > 0 { n += 1 + l + sovSimulator(uint64(l)) } + if m.Repeat != nil { + l = m.Repeat.Size() + n += 1 + l + sovSimulator(uint64(l)) + } + return n +} + +func (m *RepeatDetails) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.NumTimes != 0 { + n += 1 + sovSimulator(uint64(m.NumTimes)) + } + if m.Period != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.Period) + n += 1 + l + sovSimulator(uint64(l)) + } return n } @@ -2599,6 +2738,147 @@ func (m *JobTemplate) Unmarshal(dAtA []byte) error { } m.GangNodeUniformityLabel = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 15: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Repeat", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSimulator + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSimulator + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSimulator + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Repeat == nil { + m.Repeat = &RepeatDetails{} + } + if err := m.Repeat.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipSimulator(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthSimulator + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RepeatDetails) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSimulator + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RepeatDetails: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RepeatDetails: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NumTimes", wireType) + } + m.NumTimes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSimulator + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NumTimes |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Period", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSimulator + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSimulator + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthSimulator + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Period == nil { + m.Period = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.Period, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSimulator(dAtA[iNdEx:]) diff --git a/internal/scheduler/simulator/simulator.proto b/internal/scheduler/simulator/simulator.proto index 7aea2c46321..cc71e807855 100644 --- a/internal/scheduler/simulator/simulator.proto +++ b/internal/scheduler/simulator/simulator.proto @@ -79,6 +79,15 @@ message JobTemplate { uint32 gang_cardinality = 13; // Node Uniformity label when scheduling gangs. Only applies if gang_cardinality is non-zero. If unset it defaults to armadaproject.io/clusterName string gang_node_uniformity_label = 14; + // If set then the template will be repeated at some frequency. If null then the template will be submitted a single time. + RepeatDetails repeat = 15; +} + +message RepeatDetails { + // The number of times that template should be repeated. Must be > 0 + uint32 num_times = 1; + // The period between template submissions. May not be null + google.protobuf.Duration period = 2 [(gogoproto.stdduration) = true]; } message ShiftedExponential { From d14a539ccf8614e71363e0c66100ebee25ba14fc Mon Sep 17 00:00:00 2001 From: robertdavidsmith <34475852+robertdavidsmith@users.noreply.github.com> Date: Mon, 4 Nov 2024 13:09:55 +0000 Subject: [PATCH 07/10] Scheduler: refactor, add ResourceFractionList (#4032) Signed-off-by: Robert Smith --- .../internaltypes/resource_fraction_list.go | 31 +++++++++++ .../resource_fraction_list_test.go | 33 ++++++++++++ .../scheduler/internaltypes/resource_list.go | 52 ++++++++++++++----- .../internaltypes/resource_list_factory.go | 13 +++++ .../resource_list_factory_test.go | 22 ++++++++ .../internaltypes/resource_list_test.go | 45 ++++++++++++++++ .../scheduling/constraints/constraints.go | 9 +--- .../scheduling/preempting_queue_scheduler.go | 12 ++++- .../preempting_queue_scheduler_test.go | 3 ++ .../scheduler/scheduling/queue_scheduler.go | 3 +- .../scheduling/queue_scheduler_test.go | 2 +- .../scheduler/scheduling/scheduling_algo.go | 1 + internal/scheduler/simulator/simulator.go | 1 + 13 files changed, 203 insertions(+), 24 deletions(-) create mode 100644 internal/scheduler/internaltypes/resource_fraction_list.go create mode 100644 internal/scheduler/internaltypes/resource_fraction_list_test.go diff --git a/internal/scheduler/internaltypes/resource_fraction_list.go b/internal/scheduler/internaltypes/resource_fraction_list.go new file mode 100644 index 00000000000..233b4f6e5e6 --- /dev/null +++ b/internal/scheduler/internaltypes/resource_fraction_list.go @@ -0,0 +1,31 @@ +package internaltypes + +import ( + "fmt" +) + +type ResourceFractionList struct { + fractions []float64 // immutable, do not change this, return a new struct instead! + factory *ResourceListFactory // immutable, do not change this! +} + +func (rfl ResourceFractionList) IsEmpty() bool { + return rfl.factory == nil +} + +func (rfl ResourceFractionList) GetByName(name string) (float64, error) { + if rfl.IsEmpty() { + return 0, fmt.Errorf("resource type %s not found as resource fraction list is empty", name) + } + index, ok := rfl.factory.nameToIndex[name] + if !ok { + return 0, fmt.Errorf("resource type %s not found", name) + } + return rfl.fractions[index], nil +} + +func (rfl ResourceFractionList) assertSameResourceListFactory(other ResourceList) { + if rfl.factory != nil && other.factory != nil && rfl.factory != other.factory { + panic("mismatched ResourceListFactory") + } +} diff --git a/internal/scheduler/internaltypes/resource_fraction_list_test.go b/internal/scheduler/internaltypes/resource_fraction_list_test.go new file mode 100644 index 00000000000..5990282f4bf --- /dev/null +++ b/internal/scheduler/internaltypes/resource_fraction_list_test.go @@ -0,0 +1,33 @@ +package internaltypes + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResourceFractionList_IsEmpty(t *testing.T) { + factory := testFactory() + + assert.True(t, ResourceFractionList{}.IsEmpty()) + assert.False(t, testResourceFractionList(factory, 0, 0).IsEmpty()) + assert.False(t, testResourceFractionList(factory, 1, 1).IsEmpty()) +} + +func TestResourceFractionList_GetByName(t *testing.T) { + factory := testFactory() + a := testResourceFractionList(factory, 0.1, 0.2) + + cpu, err := a.GetByName("cpu") + assert.Nil(t, err) + assert.Equal(t, 0.1, cpu) + + _, err = a.GetByName("missing") + assert.NotNil(t, err) +} + +func TestResourceFractionList_GetByName_HandlesEmptyCorrectly(t *testing.T) { + empty := ResourceFractionList{} + _, err := empty.GetByName("cpu") + assert.NotNil(t, err) +} diff --git a/internal/scheduler/internaltypes/resource_list.go b/internal/scheduler/internaltypes/resource_list.go index 6d8ecdca1bf..b8d46ff692d 100644 --- a/internal/scheduler/internaltypes/resource_list.go +++ b/internal/scheduler/internaltypes/resource_list.go @@ -30,15 +30,13 @@ type Resource struct { } func (rl ResourceList) Equal(other ResourceList) bool { + rl.assertSameResourceListFactory(other) if rl.IsEmpty() && other.IsEmpty() { return true } if rl.IsEmpty() || other.IsEmpty() { return false } - if rl.factory != other.factory { - panic("mismatched ResourceListFactory") - } return slices.Equal(rl.resources, other.resources) } @@ -159,14 +157,12 @@ func (rl ResourceList) IsEmpty() bool { // - if no resources in this ResourceList exceed available, the last return value is false. // - empty resource lists are considered equivalent to all zero. func (rl ResourceList) ExceedsAvailable(available ResourceList) (string, k8sResource.Quantity, k8sResource.Quantity, bool) { + rl.assertSameResourceListFactory(available) + if rl.IsEmpty() && available.IsEmpty() { return "", k8sResource.Quantity{}, k8sResource.Quantity{}, false } - if available.factory != nil && rl.factory != nil && rl.factory != available.factory { - panic("mismatched ResourceListFactory") - } - var factory *ResourceListFactory if available.IsEmpty() { factory = rl.factory @@ -200,15 +196,13 @@ func (rl ResourceList) OfType(t ResourceType) ResourceList { } func (rl ResourceList) Add(other ResourceList) ResourceList { + rl.assertSameResourceListFactory(other) if rl.IsEmpty() { return other } if other.IsEmpty() { return rl } - if rl.factory != other.factory { - panic("mismatched ResourceListFactory") - } result := make([]int64, len(rl.resources)) for i, r := range rl.resources { result[i] = r + other.resources[i] @@ -217,15 +211,13 @@ func (rl ResourceList) Add(other ResourceList) ResourceList { } func (rl ResourceList) Subtract(other ResourceList) ResourceList { + rl.assertSameResourceListFactory(other) if other.IsEmpty() { return rl } if rl.IsEmpty() { return other.Negate() } - if rl.factory != other.factory { - panic("mismatched ResourceListFactory") - } result := make([]int64, len(rl.resources)) for i, r := range rl.resources { result[i] = r - other.resources[i] @@ -233,6 +225,19 @@ func (rl ResourceList) Subtract(other ResourceList) ResourceList { return ResourceList{factory: rl.factory, resources: result} } +func (rl ResourceList) Multiply(multipliers ResourceFractionList) ResourceList { + multipliers.assertSameResourceListFactory(rl) + if rl.IsEmpty() || multipliers.IsEmpty() { + return ResourceList{} + } + + result := make([]int64, len(rl.resources)) + for i, r := range rl.resources { + result[i] = multiplyResource(r, multipliers.fractions[i]) + } + return ResourceList{factory: rl.factory, resources: result} +} + func (rl ResourceList) Negate() ResourceList { if rl.IsEmpty() { return rl @@ -244,6 +249,17 @@ func (rl ResourceList) Negate() ResourceList { return ResourceList{factory: rl.factory, resources: result} } +func (rl ResourceList) Scale(factor float64) ResourceList { + if rl.IsEmpty() { + return rl + } + result := make([]int64, len(rl.resources)) + for i, r := range rl.resources { + result[i] = multiplyResource(r, factor) + } + return ResourceList{resources: result, factory: rl.factory} +} + func (rl ResourceList) asQuantity(index int) *k8sResource.Quantity { if rl.factory == nil { return &k8sResource.Quantity{} @@ -257,3 +273,13 @@ func resourcesZeroIfEmpty(resources []int64, factory *ResourceListFactory) []int } return resources } + +func (rl ResourceList) assertSameResourceListFactory(other ResourceList) { + if rl.factory != nil && other.factory != nil && rl.factory != other.factory { + panic("mismatched ResourceListFactory") + } +} + +func multiplyResource(res int64, multiplier float64) int64 { + return int64(float64(res) * multiplier) +} diff --git a/internal/scheduler/internaltypes/resource_list_factory.go b/internal/scheduler/internaltypes/resource_list_factory.go index ba130696fb9..c3bc5f81685 100644 --- a/internal/scheduler/internaltypes/resource_list_factory.go +++ b/internal/scheduler/internaltypes/resource_list_factory.go @@ -113,6 +113,19 @@ func (factory *ResourceListFactory) FromJobResourceListFailOnUnknown(resources m return ResourceList{resources: result, factory: factory}, nil } +func (factory *ResourceListFactory) MakeResourceFractionList(m map[string]float64, defaultValue float64) ResourceFractionList { + result := make([]float64, len(factory.indexToName)) + for index, name := range factory.indexToName { + val, ok := m[name] + if ok { + result[index] = val + } else { + result[index] = defaultValue + } + } + return ResourceFractionList{fractions: result, factory: factory} +} + func (factory *ResourceListFactory) SummaryString() string { result := "" for i, name := range factory.indexToName { diff --git a/internal/scheduler/internaltypes/resource_list_factory_test.go b/internal/scheduler/internaltypes/resource_list_factory_test.go index fd9b143b781..bad054db78c 100644 --- a/internal/scheduler/internaltypes/resource_list_factory_test.go +++ b/internal/scheduler/internaltypes/resource_list_factory_test.go @@ -92,6 +92,20 @@ func TestFromJobResourceListIgnoreUnknownDoesNotErrorIfMissing(t *testing.T) { assert.Equal(t, int64(100*1024*1024), testGet(&result, "memory")) } +func TestMakeResourceFractionList(t *testing.T) { + factory := testFactory() + result := factory.MakeResourceFractionList(map[string]float64{ + "memory": 0.1, + "nvidia.com/gpu": 0.2, + "external-storage-connections": 0.3, + }, 0.99) + assert.Equal(t, 0.1, testGetFraction(&result, "memory")) + assert.Equal(t, 0.99, testGetFraction(&result, "cpu")) + assert.Equal(t, 0.2, testGetFraction(&result, "nvidia.com/gpu")) + assert.Equal(t, 0.3, testGetFraction(&result, "external-storage-connections")) + assert.Equal(t, 0.99, testGetFraction(&result, "external-storage-bytes")) +} + func TestGetScale(t *testing.T) { factory := testFactory() @@ -134,3 +148,11 @@ func testGet(rl *ResourceList, name string) int64 { } return val } + +func testGetFraction(rfl *ResourceFractionList, name string) float64 { + val, err := rfl.GetByName(name) + if err != nil { + return math.MinInt64 + } + return val +} diff --git a/internal/scheduler/internaltypes/resource_list_test.go b/internal/scheduler/internaltypes/resource_list_test.go index 51a8a6b7a15..b1942b1d895 100644 --- a/internal/scheduler/internaltypes/resource_list_test.go +++ b/internal/scheduler/internaltypes/resource_list_test.go @@ -252,6 +252,31 @@ func TestSubtract_HandlesEmptyCorrectly(t *testing.T) { assert.Equal(t, ResourceList{}, ResourceList{}.Subtract(ResourceList{})) } +func TestMultiply(t *testing.T) { + factory := testFactory() + + assert.Equal(t, + testResourceList(factory, "100", "150Ki"), + testResourceList(factory, "400", "200Ki").Multiply( + testResourceFractionList(factory, 0.25, 0.75))) + assert.Equal(t, + testResourceList(factory, "0", "0"), + testResourceList(factory, "0", "200Ki").Multiply( + testResourceFractionList(factory, 0.25, 0))) + assert.Equal(t, + testResourceList(factory, "-100", "150Ki"), + testResourceList(factory, "400", "-200Ki").Multiply( + testResourceFractionList(factory, -0.25, -0.75))) +} + +func TestMultiply_HandlesEmptyCorrectly(t *testing.T) { + factory := testFactory() + + assert.Equal(t, ResourceList{}, ResourceList{}.Multiply(ResourceFractionList{})) + assert.Equal(t, ResourceList{}, ResourceList{}.Multiply(testResourceFractionList(factory, 1, 1))) + assert.Equal(t, ResourceList{}, testResourceList(factory, "1", "1Ki").Multiply(ResourceFractionList{})) +} + func TestNegate(t *testing.T) { factory := testFactory() @@ -263,9 +288,29 @@ func TestNegate_HandlesEmptyCorrectly(t *testing.T) { assert.Equal(t, ResourceList{}, ResourceList{}.Negate()) } +func TestScale(t *testing.T) { + factory := testFactory() + assert.Equal(t, testResourceList(factory, "4", "2Ki"), testResourceList(factory, "2", "1Ki").Scale(2.0)) + assert.Equal(t, testResourceList(factory, "2", "1Ki"), testResourceList(factory, "2", "1Ki").Scale(1.0)) + assert.Equal(t, testResourceList(factory, "0", "0Ki"), testResourceList(factory, "2", "1Ki").Scale(0.0)) + assert.Equal(t, testResourceList(factory, "2", "-1Ki"), testResourceList(factory, "-2", "1Ki").Scale(-1.0)) +} + +func TestScale_HandlesEmptyCorrectly(t *testing.T) { + assert.Equal(t, ResourceList{}, ResourceList{}.Scale(0.0)) + assert.Equal(t, ResourceList{}, ResourceList{}.Scale(1.0)) +} + func testResourceList(factory *ResourceListFactory, cpu string, memory string) ResourceList { return factory.FromJobResourceListIgnoreUnknown(map[string]k8sResource.Quantity{ "cpu": k8sResource.MustParse(cpu), "memory": k8sResource.MustParse(memory), }) } + +func testResourceFractionList(factory *ResourceListFactory, cpu float64, memory float64) ResourceFractionList { + return factory.MakeResourceFractionList(map[string]float64{ + "cpu": cpu, + "memory": memory, + }, 1) +} diff --git a/internal/scheduler/scheduling/constraints/constraints.go b/internal/scheduler/scheduling/constraints/constraints.go index c135757c857..1f8a5b4faf3 100644 --- a/internal/scheduler/scheduling/constraints/constraints.go +++ b/internal/scheduler/scheduling/constraints/constraints.go @@ -60,8 +60,6 @@ func IsTerminalQueueUnschedulableReason(reason string) bool { // SchedulingConstraints contains scheduling constraints, e.g., per-queue resource limits. type SchedulingConstraints struct { - // Max number of jobs to consider for a queue before giving up. - maxQueueLookBack uint // Scheduling constraints by priority class. priorityClassSchedulingConstraintsByPriorityClassName map[string]priorityClassSchedulingConstraints // Scheduling constraints for specific queues. @@ -127,8 +125,7 @@ func NewSchedulingConstraints(pool string, totalResources schedulerobjects.Resou maximumResourceFractionToSchedule = m } return SchedulingConstraints{ - maxQueueLookBack: config.MaxQueueLookback, - maximumResourcesToSchedule: absoluteFromRelativeLimits(totalResources.Resources, maximumResourceFractionToSchedule), + maximumResourcesToSchedule: absoluteFromRelativeLimits(totalResources.Resources, maximumResourceFractionToSchedule), priorityClassSchedulingConstraintsByPriorityClassName: priorityClassSchedulingConstraintsByPriorityClassName, queueSchedulingConstraintsByQueueName: queueSchedulingConstraintsByQueueName, } @@ -243,10 +240,6 @@ func (constraints *SchedulingConstraints) getPriorityClassResourceLimits(priorit return map[string]resource.Quantity{} } -func (constraints *SchedulingConstraints) GetMaxQueueLookBack() uint { - return constraints.maxQueueLookBack -} - // isStrictlyLessOrEqual returns false if // - there is a quantity in b greater than that in a or // - there is a non-zero quantity in b not in a diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler.go b/internal/scheduler/scheduling/preempting_queue_scheduler.go index 84c88cb0518..428db27ff9f 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler.go @@ -29,6 +29,7 @@ type PreemptingQueueScheduler struct { constraints schedulerconstraints.SchedulingConstraints floatingResourceTypes *floatingresources.FloatingResourceTypes protectedFractionOfFairShare float64 + maxQueueLookBack uint jobRepo JobRepository nodeDb *nodedb.NodeDb // Maps job ids to the id of the node the job is associated with. @@ -46,6 +47,7 @@ func NewPreemptingQueueScheduler( constraints schedulerconstraints.SchedulingConstraints, floatingResourceTypes *floatingresources.FloatingResourceTypes, protectedFractionOfFairShare float64, + maxQueueLookBack uint, jobRepo JobRepository, nodeDb *nodedb.NodeDb, initialNodeIdByJobId map[string]string, @@ -70,6 +72,7 @@ func NewPreemptingQueueScheduler( constraints: constraints, floatingResourceTypes: floatingResourceTypes, protectedFractionOfFairShare: protectedFractionOfFairShare, + maxQueueLookBack: maxQueueLookBack, jobRepo: jobRepo, nodeDb: nodeDb, nodeIdByJobId: maps.Clone(initialNodeIdByJobId), @@ -511,7 +514,13 @@ func addEvictedJobsToNodeDb(_ *armadacontext.Context, sctx *schedulercontext.Sch return nil } -func (sch *PreemptingQueueScheduler) schedule(ctx *armadacontext.Context, inMemoryJobRepo *InMemoryJobRepository, jobRepo JobRepository, skipUnsuccessfulSchedulingKeyCheck bool, considerPriorityCLassPriority bool) (*SchedulerResult, error) { +func (sch *PreemptingQueueScheduler) schedule( + ctx *armadacontext.Context, + inMemoryJobRepo *InMemoryJobRepository, + jobRepo JobRepository, + skipUnsuccessfulSchedulingKeyCheck bool, + considerPriorityCLassPriority bool, +) (*SchedulerResult, error) { jobIteratorByQueue := make(map[string]JobContextIterator) for _, qctx := range sch.schedulingContext.QueueSchedulingContexts { evictedIt := inMemoryJobRepo.GetJobIterator(qctx.Queue) @@ -534,6 +543,7 @@ func (sch *PreemptingQueueScheduler) schedule(ctx *armadacontext.Context, inMemo jobIteratorByQueue, skipUnsuccessfulSchedulingKeyCheck, considerPriorityCLassPriority, + sch.maxQueueLookBack, ) if err != nil { return nil, err diff --git a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go index 550b397a47b..1d2da7c4dcf 100644 --- a/internal/scheduler/scheduling/preempting_queue_scheduler_test.go +++ b/internal/scheduler/scheduling/preempting_queue_scheduler_test.go @@ -2065,6 +2065,7 @@ func TestPreemptingQueueScheduler(t *testing.T) { constraints, testfixtures.TestEmptyFloatingResources, tc.SchedulingConfig.ProtectedFractionOfFairShare, + tc.SchedulingConfig.MaxQueueLookback, jobDbTxn, nodeDb, nodeIdByJobId, @@ -2413,6 +2414,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { constraints, testfixtures.TestEmptyFloatingResources, tc.SchedulingConfig.ProtectedFractionOfFairShare, + tc.SchedulingConfig.MaxQueueLookback, jobDbTxn, nodeDb, nil, @@ -2474,6 +2476,7 @@ func BenchmarkPreemptingQueueScheduler(b *testing.B) { constraints, testfixtures.TestEmptyFloatingResources, tc.SchedulingConfig.ProtectedFractionOfFairShare, + tc.SchedulingConfig.MaxQueueLookback, jobDbTxn, nodeDb, nil, diff --git a/internal/scheduler/scheduling/queue_scheduler.go b/internal/scheduler/scheduling/queue_scheduler.go index 2ee793e028c..6938922051a 100644 --- a/internal/scheduler/scheduling/queue_scheduler.go +++ b/internal/scheduler/scheduling/queue_scheduler.go @@ -35,6 +35,7 @@ func NewQueueScheduler( jobIteratorByQueue map[string]JobContextIterator, skipUnsuccessfulSchedulingKeyCheck bool, considerPriorityClassPriority bool, + maxQueueLookBack uint, ) (*QueueScheduler, error) { for queue := range jobIteratorByQueue { if _, ok := sctx.QueueSchedulingContexts[queue]; !ok { @@ -47,7 +48,7 @@ func NewQueueScheduler( } gangIteratorsByQueue := make(map[string]*QueuedGangIterator) for queue, it := range jobIteratorByQueue { - gangIteratorsByQueue[queue] = NewQueuedGangIterator(sctx, it, constraints.GetMaxQueueLookBack(), true) + gangIteratorsByQueue[queue] = NewQueuedGangIterator(sctx, it, maxQueueLookBack, true) } candidateGangIterator, err := NewCandidateGangIterator(sctx.Pool, sctx, sctx.FairnessCostProvider, gangIteratorsByQueue, considerPriorityClassPriority) if err != nil { diff --git a/internal/scheduler/scheduling/queue_scheduler_test.go b/internal/scheduler/scheduling/queue_scheduler_test.go index 6657caff8df..c8e002422f0 100644 --- a/internal/scheduler/scheduling/queue_scheduler_test.go +++ b/internal/scheduler/scheduling/queue_scheduler_test.go @@ -538,7 +538,7 @@ func TestQueueScheduler(t *testing.T) { it := jobRepo.GetJobIterator(q.Name) jobIteratorByQueue[q.Name] = it } - sch, err := NewQueueScheduler(sctx, constraints, testfixtures.TestEmptyFloatingResources, nodeDb, jobIteratorByQueue, false, false) + sch, err := NewQueueScheduler(sctx, constraints, testfixtures.TestEmptyFloatingResources, nodeDb, jobIteratorByQueue, false, false, tc.SchedulingConfig.MaxQueueLookback) require.NoError(t, err) result, err := sch.Schedule(armadacontext.Background()) diff --git a/internal/scheduler/scheduling/scheduling_algo.go b/internal/scheduler/scheduling/scheduling_algo.go index bb9aa5b4814..f4b29b18724 100644 --- a/internal/scheduler/scheduling/scheduling_algo.go +++ b/internal/scheduler/scheduling/scheduling_algo.go @@ -544,6 +544,7 @@ func (l *FairSchedulingAlgo) SchedulePool( constraints, l.floatingResourceTypes, l.schedulingConfig.ProtectedFractionOfFairShare, + l.schedulingConfig.MaxQueueLookback, fsctx.Txn, fsctx.nodeDb, fsctx.nodeIdByJobId, diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index d581450aa00..b1f5a7e1838 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -596,6 +596,7 @@ func (s *Simulator) handleScheduleEvent(ctx *armadacontext.Context) error { constraints, s.floatingResourceTypes, s.schedulingConfig.ProtectedFractionOfFairShare, + s.schedulingConfig.MaxQueueLookback, txn, nodeDb, maps.Clone(s.accounting.nodeIdByJobId), From 8b9608b37a1c7056cc2f09ee03bfe49f898f6c8e Mon Sep 17 00:00:00 2001 From: Mustafa Ilyas Date: Mon, 4 Nov 2024 14:44:18 +0000 Subject: [PATCH 08/10] Removing legacy scheduler run configuration (#270) (#4033) Co-authored-by: Mustafa Ilyas --- .run/Armada.run.xml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.run/Armada.run.xml b/.run/Armada.run.xml index 3f2955386f5..766791134b7 100644 --- a/.run/Armada.run.xml +++ b/.run/Armada.run.xml @@ -1,24 +1,12 @@ - - - - - - - - - - - - - + - + \ No newline at end of file From 7fe5217ace6bda500b419b8829a80a88211ed5b1 Mon Sep 17 00:00:00 2001 From: Mustafa Ilyas Date: Tue, 5 Nov 2024 12:32:41 +0000 Subject: [PATCH 09/10] Preempt/Cancel on executor (#4022) * Preempt/Cancel on executor proto changes * typo * Cancelling and preempting by executor * Adding tests * Removing unused proto * Changing armadactl cancel/preempt on executor flags, adding correctly formatted priority classes message to Armadactl app * Update preempt.go * Update cancel.go * Update cancel.go * Update preempt.go * Update cancel.go * Update preempt.go * Update cancel.go --------- Co-authored-by: Mustafa --- cmd/armadactl/cmd/cancel.go | 43 + cmd/armadactl/cmd/params.go | 3 + cmd/armadactl/cmd/preempt.go | 46 +- developer/config/insecure-armada.yaml | 1 + internal/armadactl/app.go | 6 +- internal/armadactl/cancel.go | 20 + internal/armadactl/preempt.go | 22 + internal/common/ingest/testfixtures/event.go | 20 + internal/scheduler/database/query.sql.go | 188 ++-- internal/scheduleringester/dbops.go | 94 +- internal/scheduleringester/instructions.go | 28 + .../scheduleringester/instructions_test.go | 20 + internal/scheduleringester/schedulerdb.go | 115 +++ internal/server/executor/executor.go | 66 ++ pkg/api/executor.pb.go | 691 ++++++++++++++- pkg/api/executor.proto | 26 + pkg/client/executor/cancel.go | 39 + pkg/client/executor/preempt.go | 39 + pkg/controlplaneevents/events.pb.go | 805 +++++++++++++++++- pkg/controlplaneevents/events.proto | 14 + 20 files changed, 2140 insertions(+), 146 deletions(-) create mode 100644 pkg/client/executor/cancel.go create mode 100644 pkg/client/executor/preempt.go diff --git a/cmd/armadactl/cmd/cancel.go b/cmd/armadactl/cmd/cancel.go index 80bff893c96..15e264db588 100644 --- a/cmd/armadactl/cmd/cancel.go +++ b/cmd/armadactl/cmd/cancel.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "github.com/armadaproject/armada/internal/armadactl" @@ -16,6 +18,7 @@ func cancelCmd() *cobra.Command { cmd.AddCommand( cancelJobCmd(), cancelJobSetCmd(), + cancelExecutorCmd(), ) return cmd } @@ -58,3 +61,43 @@ func cancelJobSetCmd() *cobra.Command { } return cmd } + +func cancelExecutorCmd() *cobra.Command { + a := armadactl.New() + cmd := &cobra.Command{ + Use: "executor ", + Short: "Cancels jobs on executor.", + Long: `Cancels jobs on executor with provided executor name, priority classes and queues.`, + Args: cobra.ExactArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := cmd.MarkFlagRequired("priority-classes"); err != nil { + return fmt.Errorf("error marking priority-class flag as required: %s", err) + } + return initParams(cmd, a.Params) + }, + RunE: func(cmd *cobra.Command, args []string) error { + onExecutor := args[0] + + priorityClasses, err := cmd.Flags().GetStringSlice("priority-classes") + if err != nil { + return fmt.Errorf("error reading priority-class selection: %s", err) + } + + queues, err := cmd.Flags().GetStringSlice("queues") + if err != nil { + return fmt.Errorf("error reading queue selection: %s", err) + } + + return a.CancelOnExecutor(onExecutor, queues, priorityClasses) + }, + } + + cmd.Flags().StringSliceP( + "queues", + "q", + []string{}, + "Cancel jobs on executor matching the specified queue names. If no queues are provided, jobs across all queues will be cancelled. Provided queues should be comma separated, as in the following example: queueA,queueB,queueC.", + ) + cmd.Flags().StringSliceP("priority-classes", "p", []string{}, "Cancel jobs on executor matching the specified priority classes. Provided priority classes should be comma separated, as in the following example: armada-default,armada-preemptible.") + return cmd +} diff --git a/cmd/armadactl/cmd/params.go b/cmd/armadactl/cmd/params.go index 57c2263f3a2..c1c6e77dd2d 100644 --- a/cmd/armadactl/cmd/params.go +++ b/cmd/armadactl/cmd/params.go @@ -33,5 +33,8 @@ func initParams(cmd *cobra.Command, params *armadactl.Params) error { params.ExecutorAPI.Cordon = ce.CordonExecutor(client.ExtractCommandlineArmadaApiConnectionDetails) params.ExecutorAPI.Uncordon = ce.UncordonExecutor(client.ExtractCommandlineArmadaApiConnectionDetails) + params.ExecutorAPI.CancelOnExecutor = ce.CancelOnExecutor(client.ExtractCommandlineArmadaApiConnectionDetails) + params.ExecutorAPI.PreemptOnExecutor = ce.PreemptOnExecutor(client.ExtractCommandlineArmadaApiConnectionDetails) + return nil } diff --git a/cmd/armadactl/cmd/preempt.go b/cmd/armadactl/cmd/preempt.go index 4458e905b0e..8380f691d75 100644 --- a/cmd/armadactl/cmd/preempt.go +++ b/cmd/armadactl/cmd/preempt.go @@ -1,6 +1,8 @@ package cmd import ( + "fmt" + "github.com/spf13/cobra" "github.com/armadaproject/armada/internal/armadactl" @@ -12,7 +14,10 @@ func preemptCmd() *cobra.Command { Short: "Preempt jobs in armada.", Args: cobra.ExactArgs(0), } - cmd.AddCommand(preemptJobCmd()) + cmd.AddCommand( + preemptJobCmd(), + preemptExecutorCmd(), + ) return cmd } @@ -35,3 +40,42 @@ func preemptJobCmd() *cobra.Command { } return cmd } + +func preemptExecutorCmd() *cobra.Command { + a := armadactl.New() + cmd := &cobra.Command{ + Use: "executor ", + Short: "Preempts jobs on executor.", + Long: `Preempts jobs on executor with provided executor name, priority classes and queues.`, + Args: cobra.ExactArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := cmd.MarkFlagRequired("priority-classes"); err != nil { + return fmt.Errorf("error marking priority-class flag as required: %s", err) + } + return initParams(cmd, a.Params) + }, + RunE: func(cmd *cobra.Command, args []string) error { + onExecutor := args[0] + + priorityClasses, err := cmd.Flags().GetStringSlice("priority-classes") + if err != nil { + return fmt.Errorf("error reading priority-class selection: %s", err) + } + + queues, err := cmd.Flags().GetStringSlice("queues") + if err != nil { + return fmt.Errorf("error reading queue selection: %s", err) + } + + return a.PreemptOnExecutor(onExecutor, queues, priorityClasses) + }, + } + cmd.Flags().StringSliceP( + "queues", + "q", + []string{}, + "Preempt jobs on executor matching the specified queue names. If no queues are provided, jobs across all queues will be preempted. Provided queues should be comma separated, as in the following example: queueA,queueB,queueC.", + ) + cmd.Flags().StringSliceP("priority-classes", "p", []string{}, "Preempt jobs on executor matching the specified priority classes. Provided priority classes should be comma separated, as in the following example: armada-default,armada-preemptible.") + return cmd +} diff --git a/developer/config/insecure-armada.yaml b/developer/config/insecure-armada.yaml index a006ec84c24..586176ad988 100644 --- a/developer/config/insecure-armada.yaml +++ b/developer/config/insecure-armada.yaml @@ -9,6 +9,7 @@ auth: cordon_queue: ["everyone"] cancel_any_jobs: ["everyone"] reprioritize_any_jobs: ["everyone"] + preempt_any_jobs: ["everyone"] watch_all_events: ["everyone"] execute_jobs: ["everyone"] update_executor_settings: ["everyone"] diff --git a/internal/armadactl/app.go b/internal/armadactl/app.go index 8bc1f5eccc3..ef87562ac9e 100644 --- a/internal/armadactl/app.go +++ b/internal/armadactl/app.go @@ -59,8 +59,10 @@ type QueueAPI struct { } type ExecutorAPI struct { - Cordon executor.CordonAPI - Uncordon executor.UncordonAPI + Cordon executor.CordonAPI + Uncordon executor.UncordonAPI + CancelOnExecutor executor.CancelAPI + PreemptOnExecutor executor.PreemptAPI } // New instantiates an App with default parameters, including standard output diff --git a/internal/armadactl/cancel.go b/internal/armadactl/cancel.go index 39cf37b19ae..4196b6e8339 100644 --- a/internal/armadactl/cancel.go +++ b/internal/armadactl/cancel.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/armadaproject/armada/internal/common" + armadaslices "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/client" ) @@ -53,3 +54,22 @@ func (a *App) CancelJobSet(queue string, jobSetId string) (outerErr error) { return nil }) } + +func (a *App) CancelOnExecutor(executor string, queues []string, priorityClasses []string) error { + queueMsg := strings.Join(queues, ",") + priorityClassesMsg := strings.Join(priorityClasses, ",") + // If the provided slice of queues is empty, jobs on all queues will be cancelled + if len(queues) == 0 { + apiQueues, err := a.getAllQueuesAsAPIQueue(&QueueQueryArgs{}) + if err != nil { + return fmt.Errorf("error cancelling jobs on executor %s: %s", executor, err) + } + queues = armadaslices.Map(apiQueues, func(q *api.Queue) string { return q.Name }) + queueMsg = "all" + } + fmt.Fprintf(a.Out, "Requesting cancellation of jobs matching executor: %s, queues: %s, priority-classes: %s\n", executor, queueMsg, priorityClassesMsg) + if err := a.Params.ExecutorAPI.CancelOnExecutor(executor, queues, priorityClasses); err != nil { + return fmt.Errorf("error cancelling jobs on executor %s: %s", executor, err) + } + return nil +} diff --git a/internal/armadactl/preempt.go b/internal/armadactl/preempt.go index 8c9b183a072..185f7b8e70f 100644 --- a/internal/armadactl/preempt.go +++ b/internal/armadactl/preempt.go @@ -2,10 +2,12 @@ package armadactl import ( "fmt" + "strings" "github.com/pkg/errors" "github.com/armadaproject/armada/internal/common" + armadaslices "github.com/armadaproject/armada/internal/common/slices" "github.com/armadaproject/armada/pkg/api" "github.com/armadaproject/armada/pkg/client" ) @@ -32,3 +34,23 @@ func (a *App) Preempt(queue string, jobSetId string, jobId string) (outerErr err return nil }) } + +func (a *App) PreemptOnExecutor(executor string, queues []string, priorityClasses []string) error { + queueMsg := strings.Join(queues, ",") + priorityClassesMsg := strings.Join(priorityClasses, ",") + // If the provided slice of queues is empty, jobs on all queues will be cancelled + if len(queues) == 0 { + apiQueues, err := a.getAllQueuesAsAPIQueue(&QueueQueryArgs{}) + if err != nil { + return fmt.Errorf("error preempting jobs on executor %s: %s", executor, err) + } + queues = armadaslices.Map(apiQueues, func(q *api.Queue) string { return q.Name }) + queueMsg = "all" + } + + fmt.Fprintf(a.Out, "Requesting preemption of jobs matching executor: %s, queues: %s, priority-classes: %s\n", executor, queueMsg, priorityClassesMsg) + if err := a.Params.ExecutorAPI.PreemptOnExecutor(executor, queues, priorityClasses); err != nil { + return fmt.Errorf("error preempting jobs on executor %s: %s", executor, err) + } + return nil +} diff --git a/internal/common/ingest/testfixtures/event.go b/internal/common/ingest/testfixtures/event.go index ecdb173b5dd..05a38e665aa 100644 --- a/internal/common/ingest/testfixtures/event.go +++ b/internal/common/ingest/testfixtures/event.go @@ -516,6 +516,26 @@ var DeleteExecutorSettings = &controlplaneevents.Event{ }, } +var PreemptOnExecutor = &controlplaneevents.Event{ + Event: &controlplaneevents.Event_PreemptOnExecutor{ + PreemptOnExecutor: &controlplaneevents.PreemptOnExecutor{ + Name: ExecutorId, + Queues: []string{Queue}, + PriorityClasses: []string{PriorityClassName}, + }, + }, +} + +var CancelOnExecutor = &controlplaneevents.Event{ + Event: &controlplaneevents.Event_CancelOnExecutor{ + CancelOnExecutor: &controlplaneevents.CancelOnExecutor{ + Name: ExecutorId, + Queues: []string{Queue}, + PriorityClasses: []string{PriorityClassName}, + }, + }, +} + func JobSetCancelRequestedWithStateFilter(states ...armadaevents.JobState) *armadaevents.EventSequence_Event { return &armadaevents.EventSequence_Event{ Created: testfixtures.BasetimeProto, diff --git a/internal/scheduler/database/query.sql.go b/internal/scheduler/database/query.sql.go index f4e8f9a95a4..6e69dd08d07 100644 --- a/internal/scheduler/database/query.sql.go +++ b/internal/scheduler/database/query.sql.go @@ -430,38 +430,38 @@ type SelectInitialJobsRow struct { func (q *Queries) SelectInitialJobs(ctx context.Context, arg SelectInitialJobsParams) ([]SelectInitialJobsRow, error) { rows, err := q.db.Query(ctx, selectInitialJobs, arg.Serial, arg.Limit) if err != nil { - return nil, err - } + return nil, err + } defer rows.Close() var items []SelectInitialJobsRow for rows.Next() { - var i SelectInitialJobsRow - if err := rows.Scan( - &i.JobID, - &i.JobSet, - &i.Queue, - &i.Priority, - &i.Submitted, - &i.Queued, - &i.QueuedVersion, - &i.Validated, - &i.CancelRequested, - &i.CancelByJobsetRequested, - &i.Cancelled, - &i.Succeeded, - &i.Failed, - &i.SchedulingInfo, - &i.SchedulingInfoVersion, - &i.Pools, - &i.Serial, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { + var i SelectInitialJobsRow + if err := rows.Scan( + &i.JobID, + &i.JobSet, + &i.Queue, + &i.Priority, + &i.Submitted, + &i.Queued, + &i.QueuedVersion, + &i.Validated, + &i.CancelRequested, + &i.CancelByJobsetRequested, + &i.Cancelled, + &i.Succeeded, + &i.Failed, + &i.SchedulingInfo, + &i.SchedulingInfoVersion, + &i.Pools, + &i.Serial, + ); err != nil { return nil, err } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } return items, nil } @@ -529,47 +529,47 @@ type SelectInitialRunsParams struct { func (q *Queries) SelectInitialRuns(ctx context.Context, arg SelectInitialRunsParams) ([]Run, error) { rows, err := q.db.Query(ctx, selectInitialRuns, arg.Serial, arg.Limit, arg.JobIds) if err != nil { - return nil, err - } + return nil, err + } defer rows.Close() var items []Run for rows.Next() { - var i Run - if err := rows.Scan( - &i.RunID, - &i.JobID, - &i.Created, - &i.JobSet, - &i.Executor, - &i.Node, - &i.Cancelled, - &i.Running, - &i.Succeeded, - &i.Failed, - &i.Returned, - &i.RunAttempted, - &i.Serial, - &i.LastModified, - &i.LeasedTimestamp, - &i.PendingTimestamp, - &i.RunningTimestamp, - &i.TerminatedTimestamp, - &i.ScheduledAtPriority, - &i.Preempted, - &i.Pending, - &i.PreemptedTimestamp, - &i.PodRequirementsOverlay, - &i.PreemptRequested, - &i.Queue, - &i.Pool, - ); err != nil { - return nil, err - } - items = append(items, i) - } - if err := rows.Err(); err != nil { + var i Run + if err := rows.Scan( + &i.RunID, + &i.JobID, + &i.Created, + &i.JobSet, + &i.Executor, + &i.Node, + &i.Cancelled, + &i.Running, + &i.Succeeded, + &i.Failed, + &i.Returned, + &i.RunAttempted, + &i.Serial, + &i.LastModified, + &i.LeasedTimestamp, + &i.PendingTimestamp, + &i.RunningTimestamp, + &i.TerminatedTimestamp, + &i.ScheduledAtPriority, + &i.Preempted, + &i.Pending, + &i.PreemptedTimestamp, + &i.PodRequirementsOverlay, + &i.PreemptRequested, + &i.Queue, + &i.Pool, + ); err != nil { return nil, err } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } return items, nil } @@ -892,11 +892,11 @@ VALUES($1::text, $2::boolean, $3::text, $4::text, $5::timestamptz) ON CONFLICT (executor_id) DO UPDATE SET (cordoned, cordon_reason, set_by_user, set_at_time) = (excluded.cordoned, excluded.cordon_reason, excluded.set_by_user, excluded.set_at_time)` type UpsertExecutorSettingsParams struct { - ExecutorID string `db:"executor_id"` - Cordoned bool `db:"cordoned"` - CordonReason string `db:"cordon_reason"` - SetByUser string `db:"set_by_user"` - SetAtTime time.Time `db:"set_at_time"` + ExecutorID string `db:"executor_id"` + Cordoned bool `db:"cordoned"` + CordonReason string `db:"cordon_reason"` + SetByUser string `db:"set_by_user"` + SetAtTime time.Time `db:"set_at_time"` } func (q *Queries) UpsertExecutorSettings(ctx context.Context, arg UpsertExecutorSettingsParams) error { @@ -962,3 +962,55 @@ func (q *Queries) SelectLatestJobRunSerial(ctx context.Context) (int64, error) { err := row.Scan(&serial) return serial, err } + +const selectJobsByExecutorAndQueues = `-- name: SelectJobsByExecutorAndQueues :many +SELECT j.* +FROM runs jr + JOIN jobs j + ON jr.job_id = j.job_id +WHERE jr.executor = $1 + AND j.queue = ANY($2::text[]) + AND jr.succeeded = false AND jr.failed = false AND jr.cancelled = false AND jr.preempted = false +` + +func (q *Queries) SelectAllJobsByExecutorAndQueues(ctx context.Context, executor string, queues []string) ([]Job, error) { + rows, err := q.db.Query(ctx, selectJobsByExecutorAndQueues, executor, queues) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Job + for rows.Next() { + var i Job + if err := rows.Scan( + &i.JobID, + &i.JobSet, + &i.Queue, + &i.UserID, + &i.Submitted, + &i.Groups, + &i.Priority, + &i.Queued, + &i.QueuedVersion, + &i.CancelRequested, + &i.Cancelled, + &i.CancelByJobsetRequested, + &i.Succeeded, + &i.Failed, + &i.SubmitMessage, + &i.SchedulingInfo, + &i.SchedulingInfoVersion, + &i.Serial, + &i.LastModified, + &i.Validated, + &i.Pools, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/scheduleringester/dbops.go b/internal/scheduleringester/dbops.go index 4ac250d9931..dbf3c7ec93f 100644 --- a/internal/scheduleringester/dbops.go +++ b/internal/scheduleringester/dbops.go @@ -75,6 +75,18 @@ type ExecutorSettingsDelete struct { ExecutorID string } +type PreemptOnExecutor struct { + Name string + Queues []string + PriorityClasses []string +} + +type CancelOnExecutor struct { + Name string + Queues []string + PriorityClasses []string +} + // DbOperation captures a generic batch database operation. // // There are 5 types of operations: @@ -171,6 +183,8 @@ type ( UpsertExecutorSettings map[string]*ExecutorSettingsUpsert DeleteExecutorSettings map[string]*ExecutorSettingsDelete + PreemptExecutor map[string]*PreemptOnExecutor + CancelExecutor map[string]*CancelOnExecutor ) type jobSetOperation interface { @@ -315,6 +329,14 @@ func (a DeleteExecutorSettings) Merge(_ DbOperation) bool { return false } +func (pe PreemptExecutor) Merge(_ DbOperation) bool { + return false +} + +func (ce CancelExecutor) Merge(_ DbOperation) bool { + return false +} + // MergeInMap merges an op b into a, provided that b is of the same type as a. // For example, if a is of type MarkJobSetsCancelRequested, b is only merged if also of type MarkJobSetsCancelRequested. // Returns true if the ops were merged and false otherwise. @@ -467,15 +489,22 @@ func (a MarkJobsValidated) CanBeAppliedBefore(b DbOperation) bool { // Can be applied before another operation only if it relates to a different executor func (a UpsertExecutorSettings) CanBeAppliedBefore(b DbOperation) bool { switch op := b.(type) { - case UpsertExecutorSettings: - for k := range a { - if _, ok := op[k]; ok { + case executorOperation: + for executor := range a { + if affectsExecutor := op.affectsExecutor(executor); affectsExecutor { return false } } - case DeleteExecutorSettings: - for k := range a { - if _, ok := op[k]; ok { + } + return true +} + +// Can be applied before another operation only if it relates to a different executor +func (a DeleteExecutorSettings) CanBeAppliedBefore(b DbOperation) bool { + switch op := b.(type) { + case executorOperation: + for executor := range a { + if affectsExecutor := op.affectsExecutor(executor); affectsExecutor { return false } } @@ -483,18 +512,23 @@ func (a UpsertExecutorSettings) CanBeAppliedBefore(b DbOperation) bool { return true } -// Can be applied before another operation only if it relates to a different executor -func (a DeleteExecutorSettings) CanBeAppliedBefore(b DbOperation) bool { +func (pe PreemptExecutor) CanBeAppliedBefore(b DbOperation) bool { switch op := b.(type) { - case UpsertExecutorSettings: - for k := range a { - if _, ok := op[k]; ok { + case executorOperation: + for executor := range pe { + if affectsExecutor := op.affectsExecutor(executor); affectsExecutor { return false } } - case DeleteExecutorSettings: - for k := range a { - if _, ok := op[k]; ok { + } + return true +} + +func (ce CancelExecutor) CanBeAppliedBefore(b DbOperation) bool { + switch op := b.(type) { + case executorOperation: + for executor := range ce { + if affectsExecutor := op.affectsExecutor(executor); affectsExecutor { return false } } @@ -641,3 +675,35 @@ func (a UpsertExecutorSettings) GetOperation() Operation { func (a DeleteExecutorSettings) GetOperation() Operation { return ControlPlaneOperation } + +func (pe PreemptExecutor) GetOperation() Operation { + return ControlPlaneOperation +} + +func (ce CancelExecutor) GetOperation() Operation { + return ControlPlaneOperation +} + +type executorOperation interface { + affectsExecutor(string) bool +} + +func (a UpsertExecutorSettings) affectsExecutor(executor string) bool { + _, ok := a[executor] + return ok +} + +func (a DeleteExecutorSettings) affectsExecutor(executor string) bool { + _, ok := a[executor] + return ok +} + +func (pe PreemptExecutor) affectsExecutor(executor string) bool { + _, ok := pe[executor] + return ok +} + +func (ce CancelExecutor) affectsExecutor(executor string) bool { + _, ok := ce[executor] + return ok +} diff --git a/internal/scheduleringester/instructions.go b/internal/scheduleringester/instructions.go index 9834f25be71..13a526bb57d 100644 --- a/internal/scheduleringester/instructions.go +++ b/internal/scheduleringester/instructions.go @@ -410,6 +410,10 @@ func (c *ControlPlaneEventsInstructionConverter) dbOperationFromControlPlaneEven operations, err = c.handleExecutorSettingsUpsert(event.GetExecutorSettingsUpsert(), eventTime) case *controlplaneevents.Event_ExecutorSettingsDelete: operations, err = c.handleExecutorSettingsDelete(event.GetExecutorSettingsDelete()) + case *controlplaneevents.Event_PreemptOnExecutor: + operations, err = c.handlePreemptOnExecutor(event.GetPreemptOnExecutor()) + case *controlplaneevents.Event_CancelOnExecutor: + operations, err = c.handleCancelOnExecutor(event.GetCancelOnExecutor()) default: log.Errorf("Unknown event of type %T", ev) } @@ -445,6 +449,30 @@ func (c *ControlPlaneEventsInstructionConverter) handleExecutorSettingsDelete(de }, nil } +func (c *ControlPlaneEventsInstructionConverter) handlePreemptOnExecutor(preempt *controlplaneevents.PreemptOnExecutor) ([]DbOperation, error) { + return []DbOperation{ + PreemptExecutor{ + preempt.Name: { + Name: preempt.Name, + Queues: preempt.Queues, + PriorityClasses: preempt.PriorityClasses, + }, + }, + }, nil +} + +func (c *ControlPlaneEventsInstructionConverter) handleCancelOnExecutor(cancel *controlplaneevents.CancelOnExecutor) ([]DbOperation, error) { + return []DbOperation{ + CancelExecutor{ + cancel.Name: { + Name: cancel.Name, + Queues: cancel.Queues, + PriorityClasses: cancel.PriorityClasses, + }, + }, + }, nil +} + // schedulingInfoFromSubmitJob returns a minimal representation of a job containing only the info needed by the scheduler. func (c *JobSetEventsInstructionConverter) schedulingInfoFromSubmitJob(submitJob *armadaevents.SubmitJob, submitTime time.Time) (*schedulerobjects.JobSchedulingInfo, error) { return SchedulingInfoFromSubmitJob(submitJob, submitTime) diff --git a/internal/scheduleringester/instructions_test.go b/internal/scheduleringester/instructions_test.go index 6b8cc4af692..ca4c6760e08 100644 --- a/internal/scheduleringester/instructions_test.go +++ b/internal/scheduleringester/instructions_test.go @@ -272,6 +272,26 @@ func TestConvertControlPlaneEvent(t *testing.T) { }, }}, }, + "preempt on executor": { + event: f.PreemptOnExecutor, + expected: []DbOperation{PreemptExecutor{ + f.ExecutorId: &PreemptOnExecutor{ + Name: f.ExecutorId, + Queues: []string{f.Queue}, + PriorityClasses: []string{f.PriorityClassName}, + }, + }}, + }, + "cancel on executor": { + event: f.CancelOnExecutor, + expected: []DbOperation{CancelExecutor{ + f.ExecutorId: &CancelOnExecutor{ + Name: f.ExecutorId, + Queues: []string{f.Queue}, + PriorityClasses: []string{f.PriorityClassName}, + }, + }}, + }, } for name, tc := range tests { diff --git a/internal/scheduleringester/schedulerdb.go b/internal/scheduleringester/schedulerdb.go index 5b99b1f22a2..3ab11bfb8e6 100644 --- a/internal/scheduleringester/schedulerdb.go +++ b/internal/scheduleringester/schedulerdb.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/gogo/protobuf/proto" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/pkg/errors" @@ -15,6 +16,7 @@ import ( "github.com/armadaproject/armada/internal/common/ingest/metrics" "github.com/armadaproject/armada/internal/common/slices" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" + "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" ) const ( @@ -378,12 +380,125 @@ func (s *SchedulerDb) WriteDbOp(ctx *armadacontext.Context, tx pgx.Tx, op DbOper } } return nil + case CancelExecutor: + for executor, cancelRequest := range o { + jobs, err := queries.SelectAllJobsByExecutorAndQueues(ctx, executor, cancelRequest.Queues) + if err != nil { + return errors.Wrapf(err, "error cancelling jobs on executor %s by queue and priority class", executor) + } + inPriorityClasses := jobInPriorityClasses(cancelRequest.PriorityClasses) + jobsToCancel := make([]schedulerdb.Job, 0) + for _, job := range jobs { + ok, err := inPriorityClasses(job) + if err != nil { + return errors.Wrapf(err, "error cancelling jobs on executor %s by queue and priority class", executor) + } + if ok { + jobsToCancel = append(jobsToCancel, job) + } + } + for _, requestCancelParams := range createMarkJobsCancelRequestedByIdParams(jobsToCancel) { + err = queries.MarkJobsCancelRequestedById(ctx, *requestCancelParams) + if err != nil { + return errors.Wrapf(err, "error cancelling jobs on executor %s by queue and priority class", executor) + } + } + } + case PreemptExecutor: + for executor, preemptRequest := range o { + jobs, err := queries.SelectAllJobsByExecutorAndQueues(ctx, executor, preemptRequest.Queues) + if err != nil { + return errors.Wrapf(err, "error preempting jobs on executor %s by queue and priority class", executor) + } + inPriorityClasses := jobInPriorityClasses(preemptRequest.PriorityClasses) + jobsToPreempt := make([]schedulerdb.Job, 0) + for _, job := range jobs { + ok, err := inPriorityClasses(job) + if err != nil { + return errors.Wrapf(err, "error preempting jobs on executor %s by queue and priority class", executor) + } + if ok { + jobsToPreempt = append(jobsToPreempt, job) + } + } + for _, requestPreemptParams := range createMarkJobRunsPreemptRequestedByJobIdParams(jobsToPreempt) { + err = queries.MarkJobRunsPreemptRequestedByJobId(ctx, *requestPreemptParams) + if err != nil { + return errors.Wrapf(err, "error preempting jobs on executor %s by queue and priority class", executor) + } + } + } default: return errors.Errorf("received unexpected op %+v", op) } return nil } +// createMarkJobCancelRequestedById returns []*schedulerdb.MarkJobsCancelRequestedByIdParams for the specified jobs such +// that no two MarkJobsCancelRequestedByIdParams are for the same queue and jobset +func createMarkJobsCancelRequestedByIdParams(jobs []schedulerdb.Job) []*schedulerdb.MarkJobsCancelRequestedByIdParams { + result := make([]*schedulerdb.MarkJobsCancelRequestedByIdParams, 0) + mapping := map[string]map[string]*schedulerdb.MarkJobsCancelRequestedByIdParams{} + for _, job := range jobs { + if _, ok := mapping[job.Queue]; !ok { + mapping[job.Queue] = map[string]*schedulerdb.MarkJobsCancelRequestedByIdParams{} + } + if _, ok := mapping[job.Queue][job.JobSet]; !ok { + mapping[job.Queue][job.JobSet] = &schedulerdb.MarkJobsCancelRequestedByIdParams{ + Queue: job.Queue, + JobSet: job.JobSet, + JobIds: make([]string, 0), + } + result = append(result, mapping[job.Queue][job.JobSet]) + } + + mapping[job.Queue][job.JobSet].JobIds = append(mapping[job.Queue][job.JobSet].JobIds, job.JobID) + } + + return result +} + +// createMarkJobRunsPreemptRequestedByJobIdParams returns []schedulerdb.MarkJobRunsPreemptRequestedByJobIdParams for the specified jobs such +// that no two MarkJobRunsPreemptRequestedByJobIdParams are for the same queue and jobset +func createMarkJobRunsPreemptRequestedByJobIdParams(jobs []schedulerdb.Job) []*schedulerdb.MarkJobRunsPreemptRequestedByJobIdParams { + result := make([]*schedulerdb.MarkJobRunsPreemptRequestedByJobIdParams, 0) + mapping := map[string]map[string]*schedulerdb.MarkJobRunsPreemptRequestedByJobIdParams{} + for _, job := range jobs { + if _, ok := mapping[job.Queue]; !ok { + mapping[job.Queue] = map[string]*schedulerdb.MarkJobRunsPreemptRequestedByJobIdParams{} + } + if _, ok := mapping[job.Queue][job.JobSet]; !ok { + mapping[job.Queue][job.JobSet] = &schedulerdb.MarkJobRunsPreemptRequestedByJobIdParams{ + Queue: job.Queue, + JobSet: job.JobSet, + JobIds: make([]string, 0), + } + result = append(result, mapping[job.Queue][job.JobSet]) + } + + mapping[job.Queue][job.JobSet].JobIds = append(mapping[job.Queue][job.JobSet].JobIds, job.JobID) + } + + return result +} + +func jobInPriorityClasses(priorityClasses []string) func(schedulerdb.Job) (bool, error) { + priorityClassMap := map[string]bool{} + for _, priorityClass := range priorityClasses { + priorityClassMap[priorityClass] = true + } + return func(job schedulerdb.Job) (bool, error) { + schedulingInfo := &schedulerobjects.JobSchedulingInfo{} + if err := proto.Unmarshal(job.SchedulingInfo, schedulingInfo); err != nil { + err = errors.Wrapf(err, "error unmarshalling scheduling info for job %s", job.JobID) + return false, err + } + + _, ok := priorityClassMap[schedulingInfo.PriorityClassName] + return ok, nil + } +} + // getLockKey is a local method to determine a lock key for the provided set of operations. An error will be returned // if the operations don't have an associated lock key. Currently, only jobSet events require locking as they rely // on row sequence numbers being monotonically increasing. diff --git a/internal/server/executor/executor.go b/internal/server/executor/executor.go index fff09bf7ef9..3cbc4509deb 100644 --- a/internal/server/executor/executor.go +++ b/internal/server/executor/executor.go @@ -105,3 +105,69 @@ func (s *Server) DeleteExecutorSettings(grpcCtx context.Context, req *api.Execut return &types.Empty{}, nil } + +func (s *Server) PreemptOnExecutor(grpcCtx context.Context, req *api.ExecutorPreemptRequest) (*types.Empty, error) { + ctx := armadacontext.FromGrpcCtx(grpcCtx) + err := s.authorizer.AuthorizeAction(ctx, permissions.PreemptAnyJobs) + var ep *armadaerrors.ErrUnauthorized + if errors.As(err, &ep) { + return nil, status.Errorf(codes.PermissionDenied, "error preempting jobs on executor %s: %s", req.Name, ep) + } else if err != nil { + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) + } + + if req.Name == "" { + return nil, fmt.Errorf("must provide non-empty executor name when determining what to preempt") + } + + es := &controlplaneevents.Event{ + Created: protoutil.ToTimestamp(s.clock.Now().UTC()), + Event: &controlplaneevents.Event_PreemptOnExecutor{ + PreemptOnExecutor: &controlplaneevents.PreemptOnExecutor{ + Name: req.Name, + Queues: req.Queues, + PriorityClasses: req.PriorityClasses, + }, + }, + } + + err = s.publisher.PublishMessages(ctx, es) + if err != nil { + return nil, status.Error(codes.Internal, "Failed to send events to Pulsar") + } + + return &types.Empty{}, nil +} + +func (s *Server) CancelOnExecutor(grpcCtx context.Context, req *api.ExecutorCancelRequest) (*types.Empty, error) { + ctx := armadacontext.FromGrpcCtx(grpcCtx) + err := s.authorizer.AuthorizeAction(ctx, permissions.CancelAnyJobs) + var ep *armadaerrors.ErrUnauthorized + if errors.As(err, &ep) { + return nil, status.Errorf(codes.PermissionDenied, "error cancelling jobs on executor %s: %s", req.Name, ep) + } else if err != nil { + return nil, status.Errorf(codes.Unavailable, "error checking permissions: %s", err) + } + + if req.Name == "" { + return nil, fmt.Errorf("must provide non-empty executor name when determining what to cancel") + } + + es := &controlplaneevents.Event{ + Created: protoutil.ToTimestamp(s.clock.Now().UTC()), + Event: &controlplaneevents.Event_CancelOnExecutor{ + CancelOnExecutor: &controlplaneevents.CancelOnExecutor{ + Name: req.Name, + Queues: req.Queues, + PriorityClasses: req.PriorityClasses, + }, + }, + } + + err = s.publisher.PublishMessages(ctx, es) + if err != nil { + return nil, status.Error(codes.Internal, "Failed to send events to Pulsar") + } + + return &types.Empty{}, nil +} diff --git a/pkg/api/executor.pb.go b/pkg/api/executor.pb.go index be71aab0201..c2966395808 100644 --- a/pkg/api/executor.pb.go +++ b/pkg/api/executor.pb.go @@ -133,38 +133,171 @@ func (m *ExecutorSettingsDeleteRequest) GetName() string { return "" } +// Jobs on the specified executor matching both the provided queues and priority classes will be preempted +type ExecutorPreemptRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Queues []string `protobuf:"bytes,2,rep,name=queues,proto3" json:"queues,omitempty"` + PriorityClasses []string `protobuf:"bytes,3,rep,name=priorityClasses,proto3" json:"priorityClasses,omitempty"` +} + +func (m *ExecutorPreemptRequest) Reset() { *m = ExecutorPreemptRequest{} } +func (m *ExecutorPreemptRequest) String() string { return proto.CompactTextString(m) } +func (*ExecutorPreemptRequest) ProtoMessage() {} +func (*ExecutorPreemptRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_506cd9cd149291ea, []int{2} +} +func (m *ExecutorPreemptRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ExecutorPreemptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ExecutorPreemptRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ExecutorPreemptRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecutorPreemptRequest.Merge(m, src) +} +func (m *ExecutorPreemptRequest) XXX_Size() int { + return m.Size() +} +func (m *ExecutorPreemptRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ExecutorPreemptRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecutorPreemptRequest proto.InternalMessageInfo + +func (m *ExecutorPreemptRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ExecutorPreemptRequest) GetQueues() []string { + if m != nil { + return m.Queues + } + return nil +} + +func (m *ExecutorPreemptRequest) GetPriorityClasses() []string { + if m != nil { + return m.PriorityClasses + } + return nil +} + +// Jobs on the specified executor matching both the provided queues and priority classes will be cancelled +type ExecutorCancelRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Queues []string `protobuf:"bytes,2,rep,name=queues,proto3" json:"queues,omitempty"` + PriorityClasses []string `protobuf:"bytes,3,rep,name=priorityClasses,proto3" json:"priorityClasses,omitempty"` +} + +func (m *ExecutorCancelRequest) Reset() { *m = ExecutorCancelRequest{} } +func (m *ExecutorCancelRequest) String() string { return proto.CompactTextString(m) } +func (*ExecutorCancelRequest) ProtoMessage() {} +func (*ExecutorCancelRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_506cd9cd149291ea, []int{3} +} +func (m *ExecutorCancelRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ExecutorCancelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ExecutorCancelRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ExecutorCancelRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ExecutorCancelRequest.Merge(m, src) +} +func (m *ExecutorCancelRequest) XXX_Size() int { + return m.Size() +} +func (m *ExecutorCancelRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ExecutorCancelRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ExecutorCancelRequest proto.InternalMessageInfo + +func (m *ExecutorCancelRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ExecutorCancelRequest) GetQueues() []string { + if m != nil { + return m.Queues + } + return nil +} + +func (m *ExecutorCancelRequest) GetPriorityClasses() []string { + if m != nil { + return m.PriorityClasses + } + return nil +} + func init() { proto.RegisterType((*ExecutorSettingsUpsertRequest)(nil), "api.ExecutorSettingsUpsertRequest") proto.RegisterType((*ExecutorSettingsDeleteRequest)(nil), "api.ExecutorSettingsDeleteRequest") + proto.RegisterType((*ExecutorPreemptRequest)(nil), "api.ExecutorPreemptRequest") + proto.RegisterType((*ExecutorCancelRequest)(nil), "api.ExecutorCancelRequest") } func init() { proto.RegisterFile("pkg/api/executor.proto", fileDescriptor_506cd9cd149291ea) } var fileDescriptor_506cd9cd149291ea = []byte{ - // 369 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x3d, 0x4f, 0x02, 0x41, - 0x10, 0x65, 0xc1, 0x18, 0xbc, 0x18, 0x8b, 0x2d, 0x2e, 0xe4, 0xc4, 0x03, 0xaf, 0x10, 0x42, 0xcc, - 0x6d, 0xc4, 0xce, 0xc2, 0x44, 0x22, 0xb1, 0xc7, 0xd8, 0xd8, 0x98, 0xe5, 0x18, 0xcf, 0x55, 0x6e, - 0x77, 0xbd, 0xdb, 0x33, 0x1a, 0x43, 0x63, 0x65, 0x69, 0xe2, 0xcf, 0xf1, 0x0f, 0x58, 0x92, 0xd8, - 0x58, 0x11, 0x03, 0x56, 0xfc, 0x0a, 0x73, 0x1f, 0x10, 0xd0, 0x44, 0xb4, 0xdb, 0x99, 0xf7, 0xf6, - 0xbd, 0x99, 0xb7, 0xab, 0xe9, 0xf2, 0xca, 0x25, 0x54, 0x32, 0x02, 0xb7, 0xe0, 0x84, 0x4a, 0xf8, - 0xb6, 0xf4, 0x85, 0x12, 0x38, 0x47, 0x25, 0x33, 0x8a, 0xae, 0x10, 0x6e, 0x17, 0x62, 0x9c, 0x72, - 0x2e, 0x14, 0x55, 0x4c, 0xf0, 0x20, 0xa1, 0x18, 0xeb, 0x29, 0x1a, 0x57, 0xed, 0xf0, 0x9c, 0x80, - 0x27, 0xd5, 0x5d, 0x02, 0x5a, 0x2f, 0x48, 0xdb, 0x68, 0xa6, 0x92, 0xc7, 0xa0, 0x14, 0xe3, 0x6e, - 0x70, 0x22, 0x03, 0xf0, 0x55, 0x0b, 0xae, 0x43, 0x08, 0x14, 0xde, 0xd2, 0x96, 0x38, 0xf5, 0xa0, - 0x80, 0xca, 0xa8, 0xba, 0xd2, 0xc0, 0xe3, 0x41, 0x69, 0x2d, 0xaa, 0xb7, 0x85, 0xc7, 0x54, 0xac, - 0xd4, 0x8a, 0x71, 0x5c, 0xd7, 0xf2, 0x8e, 0xf0, 0x3b, 0x82, 0x43, 0xa7, 0x90, 0x2d, 0xa3, 0x6a, - 0xbe, 0xa1, 0x8f, 0x07, 0x25, 0x3c, 0xe9, 0xcd, 0xf0, 0xa7, 0x3c, 0xbc, 0xaf, 0xad, 0x26, 0xe7, - 0x16, 0xd0, 0x40, 0xf0, 0x42, 0x2e, 0xf6, 0x30, 0xc6, 0x83, 0x92, 0x3e, 0xdb, 0x9f, 0xb9, 0x3b, - 0xc7, 0xb7, 0x8e, 0x7e, 0x0e, 0x7f, 0x08, 0x5d, 0x50, 0xf0, 0xcf, 0xe1, 0xeb, 0x8f, 0x59, 0x2d, - 0x3f, 0x51, 0xc2, 0x3d, 0x4d, 0x4f, 0x22, 0xf8, 0xae, 0x8d, 0x2d, 0x9b, 0x4a, 0x66, 0xff, 0x9a, - 0x97, 0xa1, 0xdb, 0x49, 0xde, 0xf6, 0x24, 0x6f, 0xbb, 0x19, 0x19, 0x59, 0x95, 0x87, 0xb7, 0xcf, - 0xe7, 0xec, 0xa6, 0x51, 0x24, 0x37, 0x3b, 0xd3, 0x57, 0x3c, 0x0b, 0x52, 0x0d, 0x72, 0x1f, 0xcd, - 0xd1, 0xdb, 0x43, 0xb5, 0xc8, 0x3e, 0x59, 0xe2, 0x8f, 0xf6, 0x73, 0x1b, 0x2f, 0xb2, 0xaf, 0x2d, - 0xb2, 0x6f, 0x1c, 0xbc, 0x0e, 0x4d, 0xd4, 0x1f, 0x9a, 0xe8, 0x63, 0x68, 0xa2, 0xa7, 0x91, 0x99, - 0xe9, 0x8f, 0xcc, 0xcc, 0xfb, 0xc8, 0xcc, 0x9c, 0x56, 0x5c, 0xa6, 0x2e, 0xc2, 0xb6, 0xed, 0x08, - 0x8f, 0x50, 0xdf, 0xa3, 0x1d, 0x2a, 0x7d, 0x71, 0x09, 0x8e, 0x4a, 0x2b, 0x92, 0xfe, 0xd1, 0xf6, - 0x72, 0xec, 0xbd, 0xfb, 0x15, 0x00, 0x00, 0xff, 0xff, 0x63, 0xab, 0xb0, 0x13, 0xb5, 0x02, 0x00, + // 513 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x94, 0x31, 0x6f, 0xd3, 0x40, + 0x14, 0xc7, 0x73, 0x09, 0xaa, 0xd2, 0x13, 0x82, 0x62, 0x81, 0x65, 0x9c, 0xd6, 0x0e, 0x96, 0x68, + 0xa3, 0xaa, 0xb2, 0x45, 0xd9, 0x18, 0x90, 0x48, 0xa9, 0x3a, 0x82, 0x82, 0x58, 0x58, 0xd0, 0xc5, + 0x79, 0x18, 0x43, 0xec, 0xbb, 0xfa, 0xce, 0x88, 0x0a, 0x75, 0xe1, 0x13, 0x20, 0xf1, 0x5d, 0x60, + 0xe0, 0x0b, 0x30, 0x56, 0x62, 0x61, 0xb2, 0x50, 0xc2, 0x82, 0x3f, 0x05, 0xf2, 0x9d, 0x1d, 0xec, + 0xa0, 0x52, 0x3a, 0x76, 0xcb, 0xbd, 0xf7, 0xff, 0xbf, 0xdf, 0xbb, 0xf8, 0x6f, 0x63, 0x9d, 0xbd, + 0x0e, 0x3c, 0xc2, 0x42, 0x0f, 0xde, 0x82, 0x9f, 0x0a, 0x9a, 0xb8, 0x2c, 0xa1, 0x82, 0x6a, 0x1d, + 0xc2, 0x42, 0x73, 0x3d, 0xa0, 0x34, 0x98, 0x82, 0xec, 0x93, 0x38, 0xa6, 0x82, 0x88, 0x90, 0xc6, + 0x5c, 0x49, 0xcc, 0x5e, 0xd9, 0x95, 0xa7, 0x71, 0xfa, 0xc2, 0x83, 0x88, 0x89, 0x23, 0xd5, 0x74, + 0xbe, 0x20, 0xbc, 0xb1, 0x5f, 0x8e, 0x7c, 0x02, 0x42, 0x84, 0x71, 0xc0, 0x9f, 0x32, 0x0e, 0x89, + 0x18, 0xc1, 0x61, 0x0a, 0x5c, 0x68, 0x9b, 0xf8, 0x52, 0x4c, 0x22, 0x30, 0x50, 0x1f, 0x0d, 0x56, + 0x87, 0x5a, 0x9e, 0xd9, 0x57, 0x8a, 0xf3, 0x0e, 0x8d, 0x42, 0x21, 0x27, 0x8d, 0x64, 0x5f, 0xdb, + 0xc5, 0x5d, 0x9f, 0x26, 0x13, 0x1a, 0xc3, 0xc4, 0x68, 0xf7, 0xd1, 0xa0, 0x3b, 0xd4, 0xf3, 0xcc, + 0xd6, 0xaa, 0x5a, 0x4d, 0xbf, 0xd0, 0x69, 0xf7, 0xf1, 0x65, 0xf5, 0x7b, 0x04, 0x84, 0xd3, 0xd8, + 0xe8, 0x48, 0x86, 0x99, 0x67, 0xb6, 0x5e, 0xaf, 0xd7, 0xbc, 0x0d, 0xbd, 0x73, 0xf0, 0xf7, 0xf2, + 0x0f, 0x61, 0x0a, 0x02, 0xce, 0xb9, 0xbc, 0xf3, 0x19, 0x61, 0xbd, 0x9a, 0xf4, 0x38, 0x81, 0xa2, + 0x75, 0xde, 0xfb, 0xef, 0xe0, 0x95, 0xc3, 0x14, 0x52, 0xe0, 0x46, 0xbb, 0xdf, 0x19, 0xac, 0x0e, + 0xaf, 0xe7, 0x99, 0xbd, 0xa6, 0x2a, 0x35, 0x6d, 0xa9, 0xd1, 0x0e, 0xf0, 0x55, 0x96, 0x84, 0x34, + 0x09, 0xc5, 0xd1, 0xde, 0x94, 0x70, 0x0e, 0xdc, 0xe8, 0x48, 0xdb, 0x46, 0x9e, 0xd9, 0x37, 0x97, + 0x5a, 0x35, 0xff, 0xb2, 0xcb, 0xf9, 0x84, 0xf0, 0x8d, 0x6a, 0xf3, 0x3d, 0x12, 0xfb, 0x30, 0xbd, + 0x18, 0x8b, 0xef, 0xfe, 0xea, 0xe0, 0x6e, 0xb5, 0xb8, 0x76, 0x8c, 0x75, 0x95, 0xba, 0xe5, 0xc7, + 0xa9, 0x39, 0x2e, 0x61, 0xa1, 0xfb, 0xcf, 0x88, 0x9a, 0xba, 0xab, 0x22, 0xee, 0x56, 0x11, 0x77, + 0xf7, 0x0b, 0xa6, 0xb3, 0xf5, 0xfe, 0xdb, 0xcf, 0x8f, 0xed, 0x5b, 0xe6, 0xba, 0xf7, 0xe6, 0xce, + 0xe2, 0xc5, 0x79, 0xce, 0xcb, 0x19, 0xde, 0xbb, 0xe2, 0xfa, 0xc7, 0xf7, 0xd0, 0x76, 0x81, 0x57, + 0xb9, 0xf9, 0x4f, 0x7c, 0x23, 0x64, 0x67, 0xe1, 0xb7, 0xcf, 0xc4, 0x33, 0x7c, 0xad, 0x0c, 0xdd, + 0xa3, 0x78, 0xf1, 0x97, 0xf4, 0x1a, 0xe4, 0x66, 0x28, 0x4f, 0x45, 0x6e, 0x4a, 0x64, 0xdf, 0xe9, + 0xd5, 0x91, 0x1e, 0x53, 0xe6, 0x1a, 0x31, 0xc2, 0x6b, 0x2a, 0x2c, 0x35, 0xa0, 0xd9, 0x00, 0x36, + 0xb2, 0x74, 0x2a, 0xef, 0xb6, 0xe4, 0xd9, 0x8e, 0xd9, 0xe0, 0xf9, 0xd2, 0xfb, 0x07, 0x37, 0x7c, + 0xf0, 0x75, 0x66, 0xa1, 0x93, 0x99, 0x85, 0x7e, 0xcc, 0x2c, 0xf4, 0x61, 0x6e, 0xb5, 0x4e, 0xe6, + 0x56, 0xeb, 0xfb, 0xdc, 0x6a, 0x3d, 0xdb, 0x0a, 0x42, 0xf1, 0x32, 0x1d, 0xbb, 0x3e, 0x8d, 0x3c, + 0x92, 0x44, 0x64, 0x42, 0x58, 0x42, 0x5f, 0x81, 0x2f, 0xca, 0x93, 0x57, 0x7e, 0xf7, 0xc6, 0x2b, + 0x92, 0x7c, 0xf7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xed, 0x8b, 0x50, 0xc7, 0x09, 0x05, 0x00, 0x00, } @@ -182,6 +315,8 @@ const _ = grpc.SupportPackageIsVersion4 type ExecutorClient interface { UpsertExecutorSettings(ctx context.Context, in *ExecutorSettingsUpsertRequest, opts ...grpc.CallOption) (*types.Empty, error) DeleteExecutorSettings(ctx context.Context, in *ExecutorSettingsDeleteRequest, opts ...grpc.CallOption) (*types.Empty, error) + PreemptOnExecutor(ctx context.Context, in *ExecutorPreemptRequest, opts ...grpc.CallOption) (*types.Empty, error) + CancelOnExecutor(ctx context.Context, in *ExecutorCancelRequest, opts ...grpc.CallOption) (*types.Empty, error) } type executorClient struct { @@ -210,10 +345,30 @@ func (c *executorClient) DeleteExecutorSettings(ctx context.Context, in *Executo return out, nil } +func (c *executorClient) PreemptOnExecutor(ctx context.Context, in *ExecutorPreemptRequest, opts ...grpc.CallOption) (*types.Empty, error) { + out := new(types.Empty) + err := c.cc.Invoke(ctx, "/api.Executor/PreemptOnExecutor", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executorClient) CancelOnExecutor(ctx context.Context, in *ExecutorCancelRequest, opts ...grpc.CallOption) (*types.Empty, error) { + out := new(types.Empty) + err := c.cc.Invoke(ctx, "/api.Executor/CancelOnExecutor", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ExecutorServer is the server API for Executor service. type ExecutorServer interface { UpsertExecutorSettings(context.Context, *ExecutorSettingsUpsertRequest) (*types.Empty, error) DeleteExecutorSettings(context.Context, *ExecutorSettingsDeleteRequest) (*types.Empty, error) + PreemptOnExecutor(context.Context, *ExecutorPreemptRequest) (*types.Empty, error) + CancelOnExecutor(context.Context, *ExecutorCancelRequest) (*types.Empty, error) } // UnimplementedExecutorServer can be embedded to have forward compatible implementations. @@ -226,6 +381,12 @@ func (*UnimplementedExecutorServer) UpsertExecutorSettings(ctx context.Context, func (*UnimplementedExecutorServer) DeleteExecutorSettings(ctx context.Context, req *ExecutorSettingsDeleteRequest) (*types.Empty, error) { return nil, status.Errorf(codes.Unimplemented, "method DeleteExecutorSettings not implemented") } +func (*UnimplementedExecutorServer) PreemptOnExecutor(ctx context.Context, req *ExecutorPreemptRequest) (*types.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method PreemptOnExecutor not implemented") +} +func (*UnimplementedExecutorServer) CancelOnExecutor(ctx context.Context, req *ExecutorCancelRequest) (*types.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelOnExecutor not implemented") +} func RegisterExecutorServer(s *grpc.Server, srv ExecutorServer) { s.RegisterService(&_Executor_serviceDesc, srv) @@ -267,6 +428,42 @@ func _Executor_DeleteExecutorSettings_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _Executor_PreemptOnExecutor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ExecutorPreemptRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutorServer).PreemptOnExecutor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.Executor/PreemptOnExecutor", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutorServer).PreemptOnExecutor(ctx, req.(*ExecutorPreemptRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Executor_CancelOnExecutor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ExecutorCancelRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutorServer).CancelOnExecutor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/api.Executor/CancelOnExecutor", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutorServer).CancelOnExecutor(ctx, req.(*ExecutorCancelRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _Executor_serviceDesc = grpc.ServiceDesc{ ServiceName: "api.Executor", HandlerType: (*ExecutorServer)(nil), @@ -279,6 +476,14 @@ var _Executor_serviceDesc = grpc.ServiceDesc{ MethodName: "DeleteExecutorSettings", Handler: _Executor_DeleteExecutorSettings_Handler, }, + { + MethodName: "PreemptOnExecutor", + Handler: _Executor_PreemptOnExecutor_Handler, + }, + { + MethodName: "CancelOnExecutor", + Handler: _Executor_CancelOnExecutor_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "pkg/api/executor.proto", @@ -361,6 +566,102 @@ func (m *ExecutorSettingsDeleteRequest) MarshalToSizedBuffer(dAtA []byte) (int, return len(dAtA) - i, nil } +func (m *ExecutorPreemptRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ExecutorPreemptRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ExecutorPreemptRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.PriorityClasses) > 0 { + for iNdEx := len(m.PriorityClasses) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.PriorityClasses[iNdEx]) + copy(dAtA[i:], m.PriorityClasses[iNdEx]) + i = encodeVarintExecutor(dAtA, i, uint64(len(m.PriorityClasses[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Queues) > 0 { + for iNdEx := len(m.Queues) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Queues[iNdEx]) + copy(dAtA[i:], m.Queues[iNdEx]) + i = encodeVarintExecutor(dAtA, i, uint64(len(m.Queues[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintExecutor(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ExecutorCancelRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ExecutorCancelRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ExecutorCancelRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.PriorityClasses) > 0 { + for iNdEx := len(m.PriorityClasses) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.PriorityClasses[iNdEx]) + copy(dAtA[i:], m.PriorityClasses[iNdEx]) + i = encodeVarintExecutor(dAtA, i, uint64(len(m.PriorityClasses[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Queues) > 0 { + for iNdEx := len(m.Queues) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Queues[iNdEx]) + copy(dAtA[i:], m.Queues[iNdEx]) + i = encodeVarintExecutor(dAtA, i, uint64(len(m.Queues[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintExecutor(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintExecutor(dAtA []byte, offset int, v uint64) int { offset -= sovExecutor(v) base := offset @@ -405,6 +706,56 @@ func (m *ExecutorSettingsDeleteRequest) Size() (n int) { return n } +func (m *ExecutorPreemptRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovExecutor(uint64(l)) + } + if len(m.Queues) > 0 { + for _, s := range m.Queues { + l = len(s) + n += 1 + l + sovExecutor(uint64(l)) + } + } + if len(m.PriorityClasses) > 0 { + for _, s := range m.PriorityClasses { + l = len(s) + n += 1 + l + sovExecutor(uint64(l)) + } + } + return n +} + +func (m *ExecutorCancelRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovExecutor(uint64(l)) + } + if len(m.Queues) > 0 { + for _, s := range m.Queues { + l = len(s) + n += 1 + l + sovExecutor(uint64(l)) + } + } + if len(m.PriorityClasses) > 0 { + for _, s := range m.PriorityClasses { + l = len(s) + n += 1 + l + sovExecutor(uint64(l)) + } + } + return n +} + func sovExecutor(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -627,6 +978,298 @@ func (m *ExecutorSettingsDeleteRequest) Unmarshal(dAtA []byte) error { } return nil } +func (m *ExecutorPreemptRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExecutorPreemptRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExecutorPreemptRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExecutor + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExecutor + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExecutor + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExecutor + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Queues = append(m.Queues, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PriorityClasses", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExecutor + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExecutor + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PriorityClasses = append(m.PriorityClasses, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipExecutor(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthExecutor + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ExecutorCancelRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExecutorCancelRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExecutorCancelRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExecutor + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExecutor + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExecutor + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExecutor + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Queues = append(m.Queues, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PriorityClasses", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowExecutor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthExecutor + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthExecutor + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PriorityClasses = append(m.PriorityClasses, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipExecutor(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthExecutor + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipExecutor(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/api/executor.proto b/pkg/api/executor.proto index e3e5b960d8b..1e6f2c91160 100644 --- a/pkg/api/executor.proto +++ b/pkg/api/executor.proto @@ -19,6 +19,18 @@ service Executor { body: "*" }; } + rpc PreemptOnExecutor (ExecutorPreemptRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/executor/preempt/{name}" + body: "*" + }; + } + rpc CancelOnExecutor (ExecutorCancelRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/executor/cancel/{name}" + body: "*" + }; + } } message ExecutorSettingsUpsertRequest { @@ -30,3 +42,17 @@ message ExecutorSettingsUpsertRequest { message ExecutorSettingsDeleteRequest { string name = 1; } + +// Jobs on the specified executor matching both the provided queues and priority classes will be preempted +message ExecutorPreemptRequest { + string name = 1; + repeated string queues = 2; + repeated string priorityClasses = 3; +} + +// Jobs on the specified executor matching both the provided queues and priority classes will be cancelled +message ExecutorCancelRequest { + string name = 1; + repeated string queues = 2; + repeated string priorityClasses = 3; +} diff --git a/pkg/client/executor/cancel.go b/pkg/client/executor/cancel.go new file mode 100644 index 00000000000..5678df93360 --- /dev/null +++ b/pkg/client/executor/cancel.go @@ -0,0 +1,39 @@ +package executor + +import ( + "fmt" + + "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/pkg/api" + "github.com/armadaproject/armada/pkg/client" +) + +type CancelAPI func(string, []string, []string) error + +func CancelOnExecutor(getConnectionDetails client.ConnectionDetails) CancelAPI { + return func(executor string, queues []string, priorityClasses []string) error { + connectionDetails, err := getConnectionDetails() + if err != nil { + return fmt.Errorf("failed to obtain api connection details: %s", err) + } + conn, err := client.CreateApiConnection(connectionDetails) + if err != nil { + return fmt.Errorf("failed to connect to api because %s", err) + } + defer conn.Close() + + ctx, cancel := common.ContextWithDefaultTimeout() + defer cancel() + + executorClient := api.NewExecutorClient(conn) + newCancelOnExecutor := &api.ExecutorCancelRequest{ + Name: executor, + Queues: queues, + PriorityClasses: priorityClasses, + } + if _, err = executorClient.CancelOnExecutor(ctx, newCancelOnExecutor); err != nil { + return err + } + return nil + } +} diff --git a/pkg/client/executor/preempt.go b/pkg/client/executor/preempt.go new file mode 100644 index 00000000000..c5dec15a1b7 --- /dev/null +++ b/pkg/client/executor/preempt.go @@ -0,0 +1,39 @@ +package executor + +import ( + "fmt" + + "github.com/armadaproject/armada/internal/common" + "github.com/armadaproject/armada/pkg/api" + "github.com/armadaproject/armada/pkg/client" +) + +type PreemptAPI func(string, []string, []string) error + +func PreemptOnExecutor(getConnectionDetails client.ConnectionDetails) PreemptAPI { + return func(executor string, queues []string, priorityClasses []string) error { + connectionDetails, err := getConnectionDetails() + if err != nil { + return fmt.Errorf("failed to obtain api connection details: %s", err) + } + conn, err := client.CreateApiConnection(connectionDetails) + if err != nil { + return fmt.Errorf("failed to connect to api because %s", err) + } + defer conn.Close() + + ctx, cancel := common.ContextWithDefaultTimeout() + defer cancel() + + executorClient := api.NewExecutorClient(conn) + newPreemptOnExecutor := &api.ExecutorPreemptRequest{ + Name: executor, + Queues: queues, + PriorityClasses: priorityClasses, + } + if _, err = executorClient.PreemptOnExecutor(ctx, newPreemptOnExecutor); err != nil { + return err + } + return nil + } +} diff --git a/pkg/controlplaneevents/events.pb.go b/pkg/controlplaneevents/events.pb.go index 45e2e1b0bd3..d5812bba0b5 100644 --- a/pkg/controlplaneevents/events.pb.go +++ b/pkg/controlplaneevents/events.pb.go @@ -29,6 +29,8 @@ type Event struct { // Types that are valid to be assigned to Event: // *Event_ExecutorSettingsUpsert // *Event_ExecutorSettingsDelete + // *Event_PreemptOnExecutor + // *Event_CancelOnExecutor Event isEvent_Event `protobuf_oneof:"event"` } @@ -77,9 +79,17 @@ type Event_ExecutorSettingsUpsert struct { type Event_ExecutorSettingsDelete struct { ExecutorSettingsDelete *ExecutorSettingsDelete `protobuf:"bytes,3,opt,name=executorSettingsDelete,proto3,oneof" json:"executorSettingsDelete,omitempty"` } +type Event_PreemptOnExecutor struct { + PreemptOnExecutor *PreemptOnExecutor `protobuf:"bytes,4,opt,name=preemptOnExecutor,proto3,oneof" json:"preemptOnExecutor,omitempty"` +} +type Event_CancelOnExecutor struct { + CancelOnExecutor *CancelOnExecutor `protobuf:"bytes,5,opt,name=cancelOnExecutor,proto3,oneof" json:"cancelOnExecutor,omitempty"` +} func (*Event_ExecutorSettingsUpsert) isEvent_Event() {} func (*Event_ExecutorSettingsDelete) isEvent_Event() {} +func (*Event_PreemptOnExecutor) isEvent_Event() {} +func (*Event_CancelOnExecutor) isEvent_Event() {} func (m *Event) GetEvent() isEvent_Event { if m != nil { @@ -109,11 +119,27 @@ func (m *Event) GetExecutorSettingsDelete() *ExecutorSettingsDelete { return nil } +func (m *Event) GetPreemptOnExecutor() *PreemptOnExecutor { + if x, ok := m.GetEvent().(*Event_PreemptOnExecutor); ok { + return x.PreemptOnExecutor + } + return nil +} + +func (m *Event) GetCancelOnExecutor() *CancelOnExecutor { + if x, ok := m.GetEvent().(*Event_CancelOnExecutor); ok { + return x.CancelOnExecutor + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*Event) XXX_OneofWrappers() []interface{} { return []interface{}{ (*Event_ExecutorSettingsUpsert)(nil), (*Event_ExecutorSettingsDelete)(nil), + (*Event_PreemptOnExecutor)(nil), + (*Event_CancelOnExecutor)(nil), } } @@ -229,10 +255,132 @@ func (m *ExecutorSettingsDelete) GetName() string { return "" } +type PreemptOnExecutor struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Queues []string `protobuf:"bytes,2,rep,name=queues,proto3" json:"queues,omitempty"` + PriorityClasses []string `protobuf:"bytes,3,rep,name=priorityClasses,proto3" json:"priorityClasses,omitempty"` +} + +func (m *PreemptOnExecutor) Reset() { *m = PreemptOnExecutor{} } +func (m *PreemptOnExecutor) String() string { return proto.CompactTextString(m) } +func (*PreemptOnExecutor) ProtoMessage() {} +func (*PreemptOnExecutor) Descriptor() ([]byte, []int) { + return fileDescriptor_2ccee8bdbf348752, []int{3} +} +func (m *PreemptOnExecutor) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *PreemptOnExecutor) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_PreemptOnExecutor.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *PreemptOnExecutor) XXX_Merge(src proto.Message) { + xxx_messageInfo_PreemptOnExecutor.Merge(m, src) +} +func (m *PreemptOnExecutor) XXX_Size() int { + return m.Size() +} +func (m *PreemptOnExecutor) XXX_DiscardUnknown() { + xxx_messageInfo_PreemptOnExecutor.DiscardUnknown(m) +} + +var xxx_messageInfo_PreemptOnExecutor proto.InternalMessageInfo + +func (m *PreemptOnExecutor) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *PreemptOnExecutor) GetQueues() []string { + if m != nil { + return m.Queues + } + return nil +} + +func (m *PreemptOnExecutor) GetPriorityClasses() []string { + if m != nil { + return m.PriorityClasses + } + return nil +} + +type CancelOnExecutor struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Queues []string `protobuf:"bytes,2,rep,name=queues,proto3" json:"queues,omitempty"` + PriorityClasses []string `protobuf:"bytes,3,rep,name=priorityClasses,proto3" json:"priorityClasses,omitempty"` +} + +func (m *CancelOnExecutor) Reset() { *m = CancelOnExecutor{} } +func (m *CancelOnExecutor) String() string { return proto.CompactTextString(m) } +func (*CancelOnExecutor) ProtoMessage() {} +func (*CancelOnExecutor) Descriptor() ([]byte, []int) { + return fileDescriptor_2ccee8bdbf348752, []int{4} +} +func (m *CancelOnExecutor) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *CancelOnExecutor) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_CancelOnExecutor.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *CancelOnExecutor) XXX_Merge(src proto.Message) { + xxx_messageInfo_CancelOnExecutor.Merge(m, src) +} +func (m *CancelOnExecutor) XXX_Size() int { + return m.Size() +} +func (m *CancelOnExecutor) XXX_DiscardUnknown() { + xxx_messageInfo_CancelOnExecutor.DiscardUnknown(m) +} + +var xxx_messageInfo_CancelOnExecutor proto.InternalMessageInfo + +func (m *CancelOnExecutor) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *CancelOnExecutor) GetQueues() []string { + if m != nil { + return m.Queues + } + return nil +} + +func (m *CancelOnExecutor) GetPriorityClasses() []string { + if m != nil { + return m.PriorityClasses + } + return nil +} + func init() { proto.RegisterType((*Event)(nil), "controlplaneevents.Event") proto.RegisterType((*ExecutorSettingsUpsert)(nil), "controlplaneevents.ExecutorSettingsUpsert") proto.RegisterType((*ExecutorSettingsDelete)(nil), "controlplaneevents.ExecutorSettingsDelete") + proto.RegisterType((*PreemptOnExecutor)(nil), "controlplaneevents.PreemptOnExecutor") + proto.RegisterType((*CancelOnExecutor)(nil), "controlplaneevents.CancelOnExecutor") } func init() { @@ -240,34 +388,43 @@ func init() { } var fileDescriptor_2ccee8bdbf348752 = []byte{ - // 431 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x93, 0xb1, 0x8e, 0xd3, 0x40, - 0x10, 0x86, 0xed, 0x70, 0xc7, 0x5d, 0x16, 0x84, 0xc4, 0x22, 0x8c, 0x95, 0xc2, 0x3e, 0xe5, 0x10, - 0x42, 0x08, 0xad, 0xa5, 0x43, 0x50, 0x22, 0x30, 0x9c, 0x04, 0xa2, 0x39, 0x05, 0xae, 0xa1, 0xdb, - 0xd8, 0x83, 0x31, 0xd8, 0x5e, 0x6b, 0x77, 0x72, 0x22, 0xaf, 0x40, 0xc5, 0x73, 0xf0, 0x24, 0x94, - 0x29, 0xa9, 0x2c, 0x94, 0x74, 0xe6, 0x01, 0x68, 0x51, 0x76, 0xe3, 0xc4, 0x28, 0xe6, 0x94, 0xca, - 0xde, 0x99, 0xef, 0x9f, 0xf9, 0x67, 0xec, 0x25, 0xc7, 0xe5, 0xe7, 0x24, 0x88, 0x44, 0x81, 0x52, - 0x64, 0x65, 0xc6, 0x0b, 0x80, 0x0b, 0x28, 0x50, 0x05, 0xe6, 0xc1, 0x4a, 0x29, 0x50, 0x50, 0xba, - 0x0d, 0x0c, 0xfc, 0x44, 0x88, 0x24, 0x83, 0x40, 0x13, 0xe3, 0xc9, 0x87, 0x00, 0xd3, 0x1c, 0x14, - 0xf2, 0xbc, 0x34, 0xa2, 0xe1, 0x9f, 0x1e, 0xd9, 0x3f, 0x5d, 0xb2, 0xf4, 0x0d, 0x39, 0x88, 0x24, - 0x70, 0x84, 0xd8, 0xb5, 0x8f, 0xec, 0xfb, 0xd7, 0x4e, 0x06, 0xcc, 0x88, 0x59, 0x23, 0x66, 0xef, - 0x1a, 0x71, 0x78, 0xbb, 0xae, 0xfc, 0x9b, 0x2b, 0xfc, 0xa1, 0xc8, 0x53, 0x84, 0xbc, 0xc4, 0xe9, - 0xa8, 0xa9, 0x40, 0xbf, 0xda, 0xc4, 0x81, 0x2f, 0x10, 0x4d, 0x50, 0xc8, 0xb7, 0x80, 0x98, 0x16, - 0x89, 0x3a, 0x2f, 0x15, 0x48, 0x74, 0x7b, 0xba, 0xf8, 0x03, 0xb6, 0xed, 0x96, 0x9d, 0x76, 0x2a, - 0xc2, 0xbb, 0x75, 0xe5, 0x1f, 0x75, 0x57, 0xdb, 0xf4, 0x7e, 0x65, 0x8d, 0xfe, 0xd3, 0xb1, 0xd3, - 0xcc, 0x4b, 0xc8, 0x00, 0xc1, 0xbd, 0xb2, 0xbb, 0x19, 0xa3, 0xe8, 0x36, 0x63, 0x72, 0x97, 0x9b, - 0x59, 0xe9, 0x0f, 0xc8, 0xbe, 0x6e, 0x30, 0xfc, 0x6d, 0x13, 0xa7, 0x7b, 0x60, 0x7a, 0x8f, 0xec, - 0x15, 0x3c, 0x07, 0xfd, 0x1d, 0xfa, 0x21, 0xad, 0x2b, 0xff, 0xc6, 0xf2, 0xdc, 0x5a, 0xb4, 0xce, - 0xd3, 0x13, 0x72, 0x18, 0x09, 0x19, 0x8b, 0x02, 0x62, 0xbd, 0xd6, 0xc3, 0xd0, 0xa9, 0x2b, 0x9f, - 0x36, 0xb1, 0x16, 0xbf, 0xe6, 0xe8, 0x53, 0x72, 0xdd, 0xbc, 0x8f, 0x80, 0x2b, 0x51, 0xe8, 0x0d, - 0xf4, 0xc3, 0x41, 0x5d, 0xf9, 0x4e, 0x3b, 0xde, 0xd2, 0xfe, 0xc3, 0xd3, 0xc7, 0xa4, 0xaf, 0x00, - 0xc3, 0xe9, 0xb9, 0x02, 0xe9, 0xee, 0x69, 0xf1, 0x9d, 0xba, 0xf2, 0x6f, 0xad, 0x83, 0x2d, 0xe5, - 0x86, 0x1c, 0x3e, 0xdb, 0x1e, 0xd6, 0x2c, 0x64, 0xd7, 0x61, 0xc3, 0x8b, 0x1f, 0x73, 0xcf, 0x9e, - 0xcd, 0x3d, 0xfb, 0xd7, 0xdc, 0xb3, 0xbf, 0x2d, 0x3c, 0x6b, 0xb6, 0xf0, 0xac, 0x9f, 0x0b, 0xcf, - 0x7a, 0xff, 0x24, 0x49, 0xf1, 0xe3, 0x64, 0xcc, 0x22, 0x91, 0x07, 0x5c, 0xe6, 0x3c, 0xe6, 0xa5, - 0x14, 0x9f, 0x20, 0xc2, 0xd5, 0x29, 0xe8, 0xbe, 0x3d, 0xdf, 0x7b, 0xc7, 0xcf, 0x75, 0xfe, 0xcc, - 0xd0, 0xec, 0xb5, 0x60, 0x2f, 0x0c, 0x75, 0xb6, 0xa4, 0xf4, 0xb5, 0x50, 0xe3, 0xab, 0xfa, 0xf7, - 0x7f, 0xf4, 0x37, 0x00, 0x00, 0xff, 0xff, 0xdc, 0xdc, 0xc1, 0xeb, 0x84, 0x03, 0x00, 0x00, + // 564 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x94, 0x41, 0x8f, 0xd2, 0x40, + 0x14, 0xc7, 0xe9, 0x02, 0xbb, 0xcb, 0x68, 0x14, 0x46, 0xc5, 0x8a, 0xb1, 0x43, 0xd8, 0xd5, 0x6c, + 0xcc, 0xa6, 0x4d, 0xd6, 0xe8, 0xd1, 0xe8, 0xe0, 0x46, 0x8d, 0x07, 0x09, 0xba, 0x17, 0x6f, 0xa5, + 0x3c, 0xb1, 0xda, 0x76, 0xea, 0xcc, 0x40, 0xe4, 0x2b, 0x78, 0xf2, 0x73, 0x78, 0xd3, 0x4f, 0xe1, + 0x71, 0x8f, 0x9e, 0x1a, 0x03, 0xb7, 0xfa, 0x25, 0x0c, 0x53, 0xd8, 0x2d, 0x6d, 0x35, 0x1c, 0x3d, + 0x41, 0xe7, 0xfd, 0xfe, 0xef, 0xff, 0x6f, 0xdf, 0xcb, 0xa0, 0xbd, 0xf0, 0xc3, 0xc8, 0x72, 0x58, + 0x20, 0x39, 0xf3, 0x42, 0xcf, 0x0e, 0x00, 0x26, 0x10, 0x48, 0x61, 0x25, 0x3f, 0x66, 0xc8, 0x99, + 0x64, 0x18, 0xe7, 0x81, 0x16, 0x19, 0x31, 0x36, 0xf2, 0xc0, 0x52, 0xc4, 0x60, 0xfc, 0xd6, 0x92, + 0xae, 0x0f, 0x42, 0xda, 0x7e, 0x98, 0x88, 0x3a, 0xf3, 0x0a, 0xaa, 0x1e, 0x2f, 0x58, 0xfc, 0x02, + 0xed, 0x38, 0x1c, 0x6c, 0x09, 0x43, 0x5d, 0x6b, 0x6b, 0x07, 0x17, 0x8e, 0x5a, 0x66, 0x22, 0x36, + 0x57, 0x62, 0xf3, 0xf5, 0x4a, 0x4c, 0xaf, 0xc5, 0x11, 0x69, 0x2c, 0xf1, 0x43, 0xe6, 0xbb, 0x12, + 0xfc, 0x50, 0x4e, 0xfb, 0xab, 0x0e, 0xf8, 0xb3, 0x86, 0x9a, 0xf0, 0x09, 0x9c, 0xb1, 0x64, 0xfc, + 0x15, 0x48, 0xe9, 0x06, 0x23, 0x71, 0x12, 0x0a, 0xe0, 0x52, 0xdf, 0x52, 0xcd, 0xef, 0x9a, 0xf9, + 0xb4, 0xe6, 0x71, 0xa1, 0x82, 0xee, 0xc7, 0x11, 0x69, 0x17, 0x77, 0x3b, 0xf7, 0x7e, 0x56, 0xea, + 0xff, 0xc5, 0xb1, 0x30, 0xcc, 0x13, 0xf0, 0x40, 0x82, 0x5e, 0xde, 0x3c, 0x4c, 0xa2, 0x28, 0x0e, + 0x93, 0xd4, 0xfe, 0x1d, 0x26, 0x61, 0xf0, 0x04, 0x35, 0x42, 0x0e, 0x0b, 0xea, 0x65, 0xb0, 0xb2, + 0xd0, 0x2b, 0x2a, 0xc6, 0xed, 0xa2, 0x18, 0xbd, 0x2c, 0x4c, 0x49, 0x1c, 0x91, 0x9b, 0xb9, 0x1e, + 0x6b, 0xe6, 0x79, 0x0b, 0xcc, 0x51, 0xdd, 0xb1, 0x03, 0x07, 0xbc, 0x94, 0x6d, 0x55, 0xd9, 0xee, + 0x17, 0xd9, 0x76, 0x33, 0x2c, 0x35, 0xe2, 0x88, 0xb4, 0xb2, 0x1d, 0xd6, 0x4c, 0x73, 0xfd, 0xe9, + 0x0e, 0xaa, 0xaa, 0x76, 0x9d, 0xdf, 0x1a, 0x6a, 0x16, 0x0f, 0x17, 0xdf, 0x41, 0x95, 0xc0, 0xf6, + 0x41, 0xed, 0x5c, 0x8d, 0xe2, 0x38, 0x22, 0x97, 0x16, 0xcf, 0xa9, 0xa5, 0x52, 0x75, 0x7c, 0x84, + 0x76, 0x1d, 0xc6, 0x87, 0x2c, 0x80, 0xa1, 0x5a, 0xa1, 0x5d, 0xda, 0x8c, 0x23, 0x82, 0x57, 0x67, + 0x29, 0xfe, 0x8c, 0xc3, 0x0f, 0xd1, 0xc5, 0xe4, 0x7f, 0x1f, 0x6c, 0xc1, 0x02, 0x35, 0xed, 0x1a, + 0x6d, 0xc5, 0x11, 0x69, 0xa6, 0xcf, 0x53, 0xda, 0x35, 0x1e, 0xdf, 0x47, 0x35, 0x01, 0x92, 0x4e, + 0x4f, 0x04, 0x24, 0x33, 0xaa, 0xd1, 0xeb, 0x71, 0x44, 0xae, 0x9c, 0x1d, 0xa6, 0x94, 0xe7, 0x64, + 0xe7, 0x51, 0xfe, 0x65, 0x97, 0xc3, 0xdf, 0xf0, 0x65, 0x3b, 0xdf, 0x35, 0xd4, 0xc8, 0x0d, 0x7e, + 0xe3, 0x4f, 0x75, 0x88, 0xb6, 0x3f, 0x8e, 0x61, 0x0c, 0x42, 0xdf, 0x6a, 0x97, 0x0f, 0x6a, 0xf4, + 0x6a, 0x1c, 0x91, 0x7a, 0x72, 0x92, 0x62, 0x97, 0x0c, 0x7e, 0x8a, 0x2e, 0x87, 0xdc, 0x65, 0xdc, + 0x95, 0xd3, 0xae, 0x67, 0x0b, 0x01, 0x42, 0x2f, 0x2b, 0xd9, 0xad, 0x38, 0x22, 0x37, 0x32, 0xa5, + 0x94, 0x3e, 0xab, 0xea, 0x7c, 0xd3, 0x50, 0x3d, 0xbb, 0x36, 0xff, 0x79, 0x66, 0x3a, 0xf9, 0x31, + 0x33, 0xb4, 0xd3, 0x99, 0xa1, 0xfd, 0x9a, 0x19, 0xda, 0x97, 0xb9, 0x51, 0x3a, 0x9d, 0x1b, 0xa5, + 0x9f, 0x73, 0xa3, 0xf4, 0xe6, 0xc1, 0xc8, 0x95, 0xef, 0xc6, 0x03, 0xd3, 0x61, 0xbe, 0x65, 0x73, + 0xdf, 0x1e, 0xda, 0x21, 0x67, 0xef, 0xc1, 0x91, 0xcb, 0x27, 0xab, 0xf8, 0x4a, 0xfe, 0xba, 0xb5, + 0xf7, 0x58, 0xd5, 0x7b, 0x09, 0x6d, 0x3e, 0x67, 0x66, 0x37, 0xa1, 0x7a, 0x0b, 0x4a, 0xdd, 0xb5, + 0x62, 0xb0, 0xad, 0xee, 0xd4, 0x7b, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x09, 0xe8, 0xfb, 0x3a, + 0xd9, 0x05, 0x00, 0x00, } func (m *Event) Marshal() (dAtA []byte, err error) { @@ -356,6 +513,48 @@ func (m *Event_ExecutorSettingsDelete) MarshalToSizedBuffer(dAtA []byte) (int, e } return len(dAtA) - i, nil } +func (m *Event_PreemptOnExecutor) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Event_PreemptOnExecutor) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.PreemptOnExecutor != nil { + { + size, err := m.PreemptOnExecutor.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + return len(dAtA) - i, nil +} +func (m *Event_CancelOnExecutor) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Event_CancelOnExecutor) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.CancelOnExecutor != nil { + { + size, err := m.CancelOnExecutor.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintEvents(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + return len(dAtA) - i, nil +} func (m *ExecutorSettingsUpsert) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -440,6 +639,102 @@ func (m *ExecutorSettingsDelete) MarshalToSizedBuffer(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *PreemptOnExecutor) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PreemptOnExecutor) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PreemptOnExecutor) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.PriorityClasses) > 0 { + for iNdEx := len(m.PriorityClasses) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.PriorityClasses[iNdEx]) + copy(dAtA[i:], m.PriorityClasses[iNdEx]) + i = encodeVarintEvents(dAtA, i, uint64(len(m.PriorityClasses[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Queues) > 0 { + for iNdEx := len(m.Queues) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Queues[iNdEx]) + copy(dAtA[i:], m.Queues[iNdEx]) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Queues[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *CancelOnExecutor) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *CancelOnExecutor) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *CancelOnExecutor) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.PriorityClasses) > 0 { + for iNdEx := len(m.PriorityClasses) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.PriorityClasses[iNdEx]) + copy(dAtA[i:], m.PriorityClasses[iNdEx]) + i = encodeVarintEvents(dAtA, i, uint64(len(m.PriorityClasses[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Queues) > 0 { + for iNdEx := len(m.Queues) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Queues[iNdEx]) + copy(dAtA[i:], m.Queues[iNdEx]) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Queues[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintEvents(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintEvents(dAtA []byte, offset int, v uint64) int { offset -= sovEvents(v) base := offset @@ -491,6 +786,30 @@ func (m *Event_ExecutorSettingsDelete) Size() (n int) { } return n } +func (m *Event_PreemptOnExecutor) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.PreemptOnExecutor != nil { + l = m.PreemptOnExecutor.Size() + n += 1 + l + sovEvents(uint64(l)) + } + return n +} +func (m *Event_CancelOnExecutor) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.CancelOnExecutor != nil { + l = m.CancelOnExecutor.Size() + n += 1 + l + sovEvents(uint64(l)) + } + return n +} func (m *ExecutorSettingsUpsert) Size() (n int) { if m == nil { return 0 @@ -528,6 +847,56 @@ func (m *ExecutorSettingsDelete) Size() (n int) { return n } +func (m *PreemptOnExecutor) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + if len(m.Queues) > 0 { + for _, s := range m.Queues { + l = len(s) + n += 1 + l + sovEvents(uint64(l)) + } + } + if len(m.PriorityClasses) > 0 { + for _, s := range m.PriorityClasses { + l = len(s) + n += 1 + l + sovEvents(uint64(l)) + } + } + return n +} + +func (m *CancelOnExecutor) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovEvents(uint64(l)) + } + if len(m.Queues) > 0 { + for _, s := range m.Queues { + l = len(s) + n += 1 + l + sovEvents(uint64(l)) + } + } + if len(m.PriorityClasses) > 0 { + for _, s := range m.PriorityClasses { + l = len(s) + n += 1 + l + sovEvents(uint64(l)) + } + } + return n +} + func sovEvents(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -669,17 +1038,87 @@ func (m *Event) Unmarshal(dAtA []byte) error { } m.Event = &Event_ExecutorSettingsDelete{v} iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipEvents(dAtA[iNdEx:]) - if err != nil { - return err + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PreemptOnExecutor", wireType) } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthEvents + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &PreemptOnExecutor{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Event = &Event_PreemptOnExecutor{v} + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CancelOnExecutor", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &CancelOnExecutor{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Event = &Event_CancelOnExecutor{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF } iNdEx += skippy } @@ -938,6 +1377,298 @@ func (m *ExecutorSettingsDelete) Unmarshal(dAtA []byte) error { } return nil } +func (m *PreemptOnExecutor) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PreemptOnExecutor: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PreemptOnExecutor: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Queues = append(m.Queues, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PriorityClasses", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PriorityClasses = append(m.PriorityClasses, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *CancelOnExecutor) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: CancelOnExecutor: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: CancelOnExecutor: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Queues", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Queues = append(m.Queues, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PriorityClasses", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowEvents + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthEvents + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthEvents + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PriorityClasses = append(m.PriorityClasses, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipEvents(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthEvents + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipEvents(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/controlplaneevents/events.proto b/pkg/controlplaneevents/events.proto index 7241f56763d..5c5c3e6d0b3 100644 --- a/pkg/controlplaneevents/events.proto +++ b/pkg/controlplaneevents/events.proto @@ -11,6 +11,8 @@ message Event { oneof event { ExecutorSettingsUpsert executorSettingsUpsert = 2; ExecutorSettingsDelete executorSettingsDelete = 3; + PreemptOnExecutor preemptOnExecutor = 4; + CancelOnExecutor cancelOnExecutor = 5; } } @@ -24,3 +26,15 @@ message ExecutorSettingsUpsert { message ExecutorSettingsDelete { string name = 1; } + +message PreemptOnExecutor { + string name = 1; + repeated string queues = 2; + repeated string priorityClasses = 3; +} + +message CancelOnExecutor { + string name = 1; + repeated string queues = 2; + repeated string priorityClasses = 3; +} From 0c8239295e45cb5b86b21da328260760703c0071 Mon Sep 17 00:00:00 2001 From: robertdavidsmith <34475852+robertdavidsmith@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:45:39 +0000 Subject: [PATCH 10/10] Scheduler: more resource list fraction functions (#4034) Signed-off-by: Robert Smith --- .../internaltypes/resource_fraction_list.go | 30 +++++++++++--- .../resource_fraction_list_test.go | 35 ++++++++++++++-- .../scheduler/internaltypes/resource_list.go | 35 ++++++++++++---- .../internaltypes/resource_list_test.go | 40 ++++++++++++++++--- 4 files changed, 118 insertions(+), 22 deletions(-) diff --git a/internal/scheduler/internaltypes/resource_fraction_list.go b/internal/scheduler/internaltypes/resource_fraction_list.go index 233b4f6e5e6..b351636f024 100644 --- a/internal/scheduler/internaltypes/resource_fraction_list.go +++ b/internal/scheduler/internaltypes/resource_fraction_list.go @@ -2,6 +2,7 @@ package internaltypes import ( "fmt" + "math" ) type ResourceFractionList struct { @@ -13,6 +14,29 @@ func (rfl ResourceFractionList) IsEmpty() bool { return rfl.factory == nil } +func (rfl ResourceFractionList) Multiply(other ResourceFractionList) ResourceFractionList { + assertSameResourceListFactory(rfl.factory, other.factory) + if rfl.IsEmpty() || other.IsEmpty() { + return ResourceFractionList{} + } + + result := make([]float64, len(rfl.fractions)) + for i, r := range rfl.fractions { + result[i] = r * other.fractions[i] + } + return ResourceFractionList{factory: rfl.factory, fractions: result} +} + +func (rfl ResourceFractionList) Max() float64 { + result := math.Inf(-1) + for _, val := range rfl.fractions { + if val > result { + result = val + } + } + return result +} + func (rfl ResourceFractionList) GetByName(name string) (float64, error) { if rfl.IsEmpty() { return 0, fmt.Errorf("resource type %s not found as resource fraction list is empty", name) @@ -23,9 +47,3 @@ func (rfl ResourceFractionList) GetByName(name string) (float64, error) { } return rfl.fractions[index], nil } - -func (rfl ResourceFractionList) assertSameResourceListFactory(other ResourceList) { - if rfl.factory != nil && other.factory != nil && rfl.factory != other.factory { - panic("mismatched ResourceListFactory") - } -} diff --git a/internal/scheduler/internaltypes/resource_fraction_list_test.go b/internal/scheduler/internaltypes/resource_fraction_list_test.go index 5990282f4bf..a187fa778e1 100644 --- a/internal/scheduler/internaltypes/resource_fraction_list_test.go +++ b/internal/scheduler/internaltypes/resource_fraction_list_test.go @@ -1,6 +1,7 @@ package internaltypes import ( + "math" "testing" "github.com/stretchr/testify/assert" @@ -10,13 +11,41 @@ func TestResourceFractionList_IsEmpty(t *testing.T) { factory := testFactory() assert.True(t, ResourceFractionList{}.IsEmpty()) - assert.False(t, testResourceFractionList(factory, 0, 0).IsEmpty()) - assert.False(t, testResourceFractionList(factory, 1, 1).IsEmpty()) + assert.False(t, testResourceFractionList(factory, 0, 0, 0).IsEmpty()) + assert.False(t, testResourceFractionList(factory, 1, 1, 1).IsEmpty()) +} + +func TestMax(t *testing.T) { + factory := testFactory() + assert.Equal(t, 0.0, testResourceFractionList(factory, -0.1, 0.0, 0.0).Max()) + assert.Equal(t, 0.0, testResourceFractionList(factory, 0.0, 0.0, 0.0).Max()) + assert.Equal(t, 0.9, testResourceFractionList(factory, 0.1, 0.9, 0.7).Max()) +} + +func TestMax_HandlesEmptyCorrectly(t *testing.T) { + assert.Equal(t, math.Inf(-1), ResourceFractionList{}.Max()) +} + +func TestResourceFractionList_Multiply(t *testing.T) { + factory := testFactory() + + assert.Equal(t, + testResourceFractionList(factory, 0.125, 0.25, 1), + testResourceFractionList(factory, 0.25, 0.5, 1).Multiply( + testResourceFractionList(factory, 0.5, 0.5, 1))) +} + +func TestResourceFractionList_Multiply_HandlesEmptyCorrectly(t *testing.T) { + factory := testFactory() + + assert.Equal(t, ResourceFractionList{}, ResourceFractionList{}.Multiply(ResourceFractionList{})) + assert.Equal(t, ResourceFractionList{}, ResourceFractionList{}.Multiply(testResourceFractionList(factory, 1, 1, 1))) + assert.Equal(t, ResourceFractionList{}, testResourceFractionList(factory, 1, 1, 1).Multiply(ResourceFractionList{})) } func TestResourceFractionList_GetByName(t *testing.T) { factory := testFactory() - a := testResourceFractionList(factory, 0.1, 0.2) + a := testResourceFractionList(factory, 0.1, 0.2, 1) cpu, err := a.GetByName("cpu") assert.Nil(t, err) diff --git a/internal/scheduler/internaltypes/resource_list.go b/internal/scheduler/internaltypes/resource_list.go index b8d46ff692d..be4b42825f2 100644 --- a/internal/scheduler/internaltypes/resource_list.go +++ b/internal/scheduler/internaltypes/resource_list.go @@ -30,7 +30,7 @@ type Resource struct { } func (rl ResourceList) Equal(other ResourceList) bool { - rl.assertSameResourceListFactory(other) + assertSameResourceListFactory(rl.factory, other.factory) if rl.IsEmpty() && other.IsEmpty() { return true } @@ -157,7 +157,7 @@ func (rl ResourceList) IsEmpty() bool { // - if no resources in this ResourceList exceed available, the last return value is false. // - empty resource lists are considered equivalent to all zero. func (rl ResourceList) ExceedsAvailable(available ResourceList) (string, k8sResource.Quantity, k8sResource.Quantity, bool) { - rl.assertSameResourceListFactory(available) + assertSameResourceListFactory(rl.factory, available.factory) if rl.IsEmpty() && available.IsEmpty() { return "", k8sResource.Quantity{}, k8sResource.Quantity{}, false @@ -196,7 +196,7 @@ func (rl ResourceList) OfType(t ResourceType) ResourceList { } func (rl ResourceList) Add(other ResourceList) ResourceList { - rl.assertSameResourceListFactory(other) + assertSameResourceListFactory(rl.factory, other.factory) if rl.IsEmpty() { return other } @@ -211,7 +211,7 @@ func (rl ResourceList) Add(other ResourceList) ResourceList { } func (rl ResourceList) Subtract(other ResourceList) ResourceList { - rl.assertSameResourceListFactory(other) + assertSameResourceListFactory(rl.factory, other.factory) if other.IsEmpty() { return rl } @@ -226,7 +226,7 @@ func (rl ResourceList) Subtract(other ResourceList) ResourceList { } func (rl ResourceList) Multiply(multipliers ResourceFractionList) ResourceList { - multipliers.assertSameResourceListFactory(rl) + assertSameResourceListFactory(rl.factory, multipliers.factory) if rl.IsEmpty() || multipliers.IsEmpty() { return ResourceList{} } @@ -238,6 +238,23 @@ func (rl ResourceList) Multiply(multipliers ResourceFractionList) ResourceList { return ResourceList{factory: rl.factory, resources: result} } +// Divide, return 0 on attempt to divide by 0 +func (rl ResourceList) DivideZeroOnError(other ResourceList) ResourceFractionList { + assertSameResourceListFactory(rl.factory, other.factory) + if rl.IsEmpty() || other.IsEmpty() { + return ResourceFractionList{} + } + + result := make([]float64, len(rl.resources)) + for i, r := range rl.resources { + denom := other.resources[i] + if denom != 0 { + result[i] = float64(r) / float64(denom) + } + } + return ResourceFractionList{factory: rl.factory, fractions: result} +} + func (rl ResourceList) Negate() ResourceList { if rl.IsEmpty() { return rl @@ -274,12 +291,16 @@ func resourcesZeroIfEmpty(resources []int64, factory *ResourceListFactory) []int return resources } -func (rl ResourceList) assertSameResourceListFactory(other ResourceList) { - if rl.factory != nil && other.factory != nil && rl.factory != other.factory { +func assertSameResourceListFactory(a, b *ResourceListFactory) { + if a != nil && b != nil && a != b { panic("mismatched ResourceListFactory") } } func multiplyResource(res int64, multiplier float64) int64 { + if multiplier == 1.0 { + // avoid rounding error in the simple case + return res + } return int64(float64(res) * multiplier) } diff --git a/internal/scheduler/internaltypes/resource_list_test.go b/internal/scheduler/internaltypes/resource_list_test.go index b1942b1d895..216ab626670 100644 --- a/internal/scheduler/internaltypes/resource_list_test.go +++ b/internal/scheduler/internaltypes/resource_list_test.go @@ -258,25 +258,53 @@ func TestMultiply(t *testing.T) { assert.Equal(t, testResourceList(factory, "100", "150Ki"), testResourceList(factory, "400", "200Ki").Multiply( - testResourceFractionList(factory, 0.25, 0.75))) + testResourceFractionList(factory, 0.25, 0.75, 1))) assert.Equal(t, testResourceList(factory, "0", "0"), testResourceList(factory, "0", "200Ki").Multiply( - testResourceFractionList(factory, 0.25, 0))) + testResourceFractionList(factory, 0.25, 0, 1))) + assert.Equal(t, + testResourceList(factory, "2", "100Ki"), + testResourceList(factory, "2", "100Ki").Multiply( + testResourceFractionList(factory, 1, 1, 1))) assert.Equal(t, testResourceList(factory, "-100", "150Ki"), testResourceList(factory, "400", "-200Ki").Multiply( - testResourceFractionList(factory, -0.25, -0.75))) + testResourceFractionList(factory, -0.25, -0.75, 1))) } func TestMultiply_HandlesEmptyCorrectly(t *testing.T) { factory := testFactory() assert.Equal(t, ResourceList{}, ResourceList{}.Multiply(ResourceFractionList{})) - assert.Equal(t, ResourceList{}, ResourceList{}.Multiply(testResourceFractionList(factory, 1, 1))) + assert.Equal(t, ResourceList{}, ResourceList{}.Multiply(testResourceFractionList(factory, 1, 1, 1))) assert.Equal(t, ResourceList{}, testResourceList(factory, "1", "1Ki").Multiply(ResourceFractionList{})) } +func TestDivideZeroOnError(t *testing.T) { + factory := testFactory() + + expected := testResourceFractionList(factory, 0.5, 0.25, 0) + actual := testResourceList(factory, "2", "2Ki").DivideZeroOnError(testResourceList(factory, "4", "8Ki")) + assert.Equal(t, expected, actual) +} + +func TestDDivideZeroOnError_HandlesZeroDenominatorCorrectly(t *testing.T) { + factory := testFactory() + + expected := testResourceFractionList(factory, 2, 0, 0) + actual := testResourceList(factory, "2", "2Ki").DivideZeroOnError(testResourceList(factory, "1", "0Ki")) + assert.Equal(t, expected, actual) +} + +func TestDivideZeroOnError_HandlesEmptyCorrectly(t *testing.T) { + factory := testFactory() + + assert.Equal(t, ResourceFractionList{}, testResourceList(factory, "1", "1Ki").DivideZeroOnError(ResourceList{})) + assert.Equal(t, ResourceFractionList{}, ResourceList{}.DivideZeroOnError(testResourceList(factory, "1", "1Ki"))) + assert.Equal(t, ResourceFractionList{}, ResourceList{}.DivideZeroOnError(ResourceList{})) +} + func TestNegate(t *testing.T) { factory := testFactory() @@ -308,9 +336,9 @@ func testResourceList(factory *ResourceListFactory, cpu string, memory string) R }) } -func testResourceFractionList(factory *ResourceListFactory, cpu float64, memory float64) ResourceFractionList { +func testResourceFractionList(factory *ResourceListFactory, cpu float64, memory float64, defaultValue float64) ResourceFractionList { return factory.MakeResourceFractionList(map[string]float64{ "cpu": cpu, "memory": memory, - }, 1) + }, defaultValue) }