From fd1d8e85bf0ce83ff2f20c73d590c0eea480ccdb Mon Sep 17 00:00:00 2001 From: Cassie Coyle Date: Wed, 3 Jul 2024 15:58:55 -0600 Subject: [PATCH] Distributed Scheduler CLI Changes (#1405) * wip Signed-off-by: Cassandra Coyle * rm scheduleJob logic. keep only init/uninstall logic Signed-off-by: Cassandra Coyle * Fixes install and uninstall of scheduler in standalone mode. Signed-off-by: Artur Souza * Fixing path for Go tools in Darwin. Signed-off-by: Artur Souza * Fix Go bin location for MacOS. Signed-off-by: Artur Souza * Fix min scheduler version to be 1.14.x Signed-off-by: Artur Souza * Use env var to pass scheduler host. Signed-off-by: Artur Souza * Fix CLI build to work with latest MacOS runners from GH Signed-off-by: Artur Souza --------- Signed-off-by: Cassandra Coyle Signed-off-by: Artur Souza Co-authored-by: Artur Souza --- .github/workflows/dapr_cli.yaml | 18 ++- .github/workflows/self_hosted_e2e.yaml | 17 ++- .gitignore | 5 +- CONTRIBUTING.md | 2 +- Makefile | 1 + README.md | 19 ++- cmd/init.go | 2 +- cmd/run.go | 3 + cmd/uninstall.go | 4 +- go.mod | 1 + go.sum | 2 + pkg/standalone/bundle.go | 4 +- pkg/standalone/bundle_test.go | 4 +- pkg/standalone/common.go | 17 ++- pkg/standalone/run.go | 24 ++++ pkg/standalone/standalone.go | 174 +++++++++++++++++++++++-- pkg/standalone/uninstall.go | 18 ++- 17 files changed, 279 insertions(+), 36 deletions(-) diff --git a/.github/workflows/dapr_cli.yaml b/.github/workflows/dapr_cli.yaml index 1983bdfea..a27c13ba6 100644 --- a/.github/workflows/dapr_cli.yaml +++ b/.github/workflows/dapr_cli.yaml @@ -39,7 +39,7 @@ jobs: WIX_BIN_PATH: 'C:/Program Files (x86)/WiX Toolset v3.11/bin' strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest, windows-latest, macOS-latest, macOS-latest-large] target_arch: [arm, arm64, amd64] include: - os: ubuntu-latest @@ -48,6 +48,8 @@ jobs: target_os: windows - os: macOS-latest target_os: darwin + - os: macOS-latest-large + target_os: darwin exclude: - os: windows-latest target_arch: arm @@ -55,7 +57,21 @@ jobs: target_arch: arm64 - os: macOS-latest target_arch: arm + - os: macOS-latest + target_arch: amd64 + - os: macOS-latest-large + target_arch: arm + - os: macOS-latest-large + target_arch: arm64 steps: + - name: Prepare Go's bin location - MacOS + if: matrix.target_os == 'darwin' + run: | + export PATH=$HOME/bin:$PATH + echo "$HOME/bin" >> $GITHUB_PATH + + echo "GOBIN=$HOME/bin" >> $GITHUB_ENV + mkdir -p $HOME/bin - name: Check out code into the Go module directory uses: actions/checkout@v3 - name: Set up Go diff --git a/.github/workflows/self_hosted_e2e.yaml b/.github/workflows/self_hosted_e2e.yaml index 7213ef564..c8c0441ba 100644 --- a/.github/workflows/self_hosted_e2e.yaml +++ b/.github/workflows/self_hosted_e2e.yaml @@ -48,13 +48,14 @@ jobs: # TODO: Remove this when our E2E tests are stable for podman on MacOS. fail-fast: false # Keep running if one leg fails. matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + # See https://github.com/actions/runner-images + os: [macos-latest-large, ubuntu-latest, windows-latest] target_arch: [amd64] dapr_install_mode: [slim, complete] include: - os: ubuntu-latest target_os: linux - - os: macOS-latest + - os: macos-latest-large target_os: darwin - os: windows-latest target_os: windows @@ -62,6 +63,14 @@ jobs: - os: windows-latest dapr_install_mode: complete steps: + - name: Prepare Go's bin location - MacOS + if: matrix.os == 'macos-latest-large' + run: | + export PATH=$HOME/bin:$PATH + echo "$HOME/bin" >> $GITHUB_PATH + + echo "GOBIN=$HOME/bin" >> $GITHUB_ENV + mkdir -p $HOME/bin - name: Check out code into the Go module directory uses: actions/checkout@v3 - name: Set up Go @@ -101,7 +110,7 @@ jobs: ${{ matrix.target_os }}-${{ matrix.target_arch }}-go-${{ steps.setup-go.outputs.go-version }}- - name: Install podman - MacOS timeout-minutes: 15 - if: matrix.os == 'macos-latest' && matrix.dapr_install_mode == 'complete' + if: matrix.os == 'macos-latest-large' && matrix.dapr_install_mode == 'complete' run: | # Install podman curl -sL -o podman.pkg https://github.com/containers/podman/releases/download/v${{ env.PODMAN_VERSION }}/podman-installer-macos-amd64.pkg @@ -149,7 +158,7 @@ jobs: echo "DAPR_DASHBOARD_LATEST_STABLE_VERSION=$LATEST_STABLE_DASHBOARD_VERSION" >> $GITHUB_ENV shell: bash - name: Set the test timeout - MacOS - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-latest-large' run: echo "E2E_SH_TEST_TIMEOUT=30m" >> $GITHUB_ENV - name: Run E2E tests with GHCR # runs every 6hrs diff --git a/.gitignore b/.gitignore index 605a37812..897ccb8b4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ *.dylib cli +# Handy directory to keep local scripts and help files. +.local/ + # Mac's metadata folder .DS_Store @@ -37,4 +40,4 @@ go.work #Wix files *.wixobj *.wixpdb -*.msi \ No newline at end of file +*.msi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6e3afa2a..9dd21609e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ Before you file an issue, make sure you've checked the following: - 👎 down-vote 1. For bugs - Check it's not an environment issue. For example, if running on Kubernetes, make sure prerequisites are in place. (state stores, bindings, etc.) - - You have as much data as possible. This usually comes in the form of logs and/or stacktrace. If running on Kubernetes or other environment, look at the logs of the Dapr services (runtime, operator, placement service). More details on how to get logs can be found [here](https://docs.dapr.io/operations/troubleshooting/logs-troubleshooting/). + - You have as much data as possible. This usually comes in the form of logs and/or stacktrace. If running on Kubernetes or other environment, look at the logs of the Dapr services (runtime, operator, placement, scheduler service). More details on how to get logs can be found [here](https://docs.dapr.io/operations/troubleshooting/logs-troubleshooting/). 1. For proposals - Many changes to the Dapr runtime may require changes to the API. In that case, the best place to discuss the potential feature is the main [Dapr repo](https://github.com/dapr/dapr). - Other examples could include bindings, state stores or entirely new components. diff --git a/Makefile b/Makefile index 592cbc3c3..3bcc674c1 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,7 @@ ifeq ($(LOCAL_OS),Linux) else ifeq ($(LOCAL_OS),Darwin) TARGET_OS_LOCAL = darwin GOLANGCI_LINT:=golangci-lint + PATH := $(PATH):$(HOME)/go/bin/darwin_$(GOARCH) export ARCHIVE_EXT = .tar.gz else TARGET_OS_LOCAL ?= windows diff --git a/README.md b/README.md index dc8aca585..27edc7b88 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Install windows Dapr CLI using MSI package. ### Install Dapr on your local machine (self-hosted) -In self-hosted mode, dapr can be initialized using the CLI with the placement, redis and zipkin containers enabled by default(recommended) or without them which also does not require docker to be available in the environment. +In self-hosted mode, dapr can be initialized using the CLI with the placement, scheduler, redis, and zipkin containers enabled by default(recommended) or without them which also does not require docker to be available in the environment. #### Initialize Dapr @@ -89,6 +89,7 @@ Output should look like so: ✅ Downloaded binaries and completed components set up. ℹ️ daprd binary has been installed to $HOME/.dapr/bin. ℹ️ dapr_placement container is running. +ℹ️ dapr_scheduler container is running. ℹ️ dapr_redis container is running. ℹ️ dapr_zipkin container is running. ℹ️ Use `docker ps` to check running containers. @@ -118,10 +119,11 @@ Output should look like so: ✅ Downloaded binaries and completed components set up. ℹ️ daprd binary has been installed to $HOME/.dapr/bin. ℹ️ placement binary has been installed. +ℹ️ scheduler binary has been installed. ✅ Success! Dapr is up and running. To get started, go here: https://aka.ms/dapr-getting-started ``` ->Note: When initializing Dapr with the `--slim` flag only the Dapr runtime binary and the placement service binary are installed. An empty default components folder is created with no default configuration files. During `dapr run` user should use `--resources-path` (`--components-path` is deprecated and will be removed in future releases) to point to a components directory with custom configurations files or alternatively place these files in the default directory. For Linux/MacOS, the default components directory path is `$HOME/.dapr/components` and for Windows it is `%USERPROFILE%\.dapr\components`. +>Note: When initializing Dapr with the `--slim` flag only the Dapr runtime, placement, and scheduler service binaries are installed. An empty default components folder is created with no default configuration files. During `dapr run` user should use `--resources-path` (`--components-path` is deprecated and will be removed in future releases) to point to a components directory with custom configurations files or alternatively place these files in the default directory. For Linux/MacOS, the default components directory path is `$HOME/.dapr/components` and for Windows it is `%USERPROFILE%\.dapr\components`. #### Install a specific runtime version @@ -171,7 +173,7 @@ Move to the bundle directory and run the following command: > If you are not running the above command from the bundle directory, provide the full path to bundle directory as input. For example, assuming the bundle directory path is $HOME/daprbundle, run `$HOME/daprbundle/dapr init --from-dir $HOME/daprbundle` to have the same behavior. -> Note: Dapr Installer bundle just contains the placement container apart from the binaries and so `zipkin` and `redis` are not enabled by default. You can pull the images locally either from network or private registry and run as follows: +> Note: Dapr Installer bundle just contains the placement and scheduler containers apart from the binaries and so `zipkin` and `redis` are not enabled by default. You can pull the images locally either from network or private registry and run as follows: ```bash docker run --name "dapr_zipkin" --restart always -d -p 9411:9411 openzipkin/zipkin @@ -199,6 +201,9 @@ dapr init --network dapr-network > Note: When installed to a specific Docker network, you will need to add the `--placement-host-address` arguments to `dapr run` commands run in any containers within that network. > The format of `--placement-host-address` argument is either `` or `:`. If the port is omitted, the default port `6050` for Windows and `50005` for Linux/MacOS applies. +> Note: When installed to a specific Docker network, you will need to add the `--scheduler-host-address` arguments to `dapr run` commands run in any containers within that network. +> The format of `--scheduler-host-address` argument is either `` or `:`. If the port is omitted, the default port `6060` for Windows and `50006` for Linux/MacOS applies. + #### Install with a specific container runtime You can install the Dapr runtime using a specific container runtime @@ -228,7 +233,7 @@ For more details, see the docs for dev containers with [Visual Studio Code](http ### Uninstall Dapr in a standalone mode -Uninstalling will remove daprd binary and the placement container (if installed with Docker or the placement binary if not). +Uninstalling will remove daprd binary along with the placement and scheduler containers (if installed with Docker or the placement and scheduler binaries if not). ```bash @@ -237,7 +242,7 @@ dapr uninstall > For Linux users, if you run your docker cmds with sudo, you need to use "**sudo dapr uninstall**" to remove the containers. -The command above won't remove the redis or zipkin containers by default in case you were using it for other purposes. It will also not remove the default dapr folder that was created on `dapr init`. To remove all the containers (placement, redis, zipkin) and also the default dapr folder created on init run: +The command above won't remove the redis or zipkin containers by default in case you were using it for other purposes. It will also not remove the default dapr folder that was created on `dapr init`. To remove all the containers (placement, scheduler, redis, zipkin) and also the default dapr folder created on init run: ```bash dapr uninstall --all @@ -245,7 +250,7 @@ dapr uninstall --all The above command can also be run when Dapr has been installed in a non-docker environment, it will only remove the installed binaries and the default dapr folder in that case. -> NB: The `dapr uninstall` command will always try to remove the placement binary/service and will throw an error is not able to. +> NB: The `dapr uninstall` command will always try to remove the placement and scheduler binaries/services and will throw an error is not able to. **You should always run a `dapr uninstall` before running another `dapr init`.** @@ -407,7 +412,7 @@ dapr init --network dapr-network dapr run --app-id nodeapp --placement-host-address dapr_placement node app.js ``` -> Note: When in a specific Docker network, the Redis, Zipkin and placement service containers are given specific network aliases, `dapr_redis`, `dapr_zipkin` and `dapr_placement`, respectively. The default configuration files reflect the network alias rather than `localhost` when a docker network is specified. +> Note: When in a specific Docker network, the Redis, Zipkin and placement and scheduler service containers are given specific network aliases, `dapr_redis`, `dapr_zipkin`, `dapr_placement`, and `dapr_scheduler`, respectively. The default configuration files reflect the network alias rather than `localhost` when a docker network is specified. ### Use gRPC diff --git a/cmd/init.go b/cmd/init.go index 1b5c638eb..86f785c66 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -210,7 +210,7 @@ func init() { InitCmd.Flags().BoolVarP(&devMode, "dev", "", false, "Use Dev mode. Deploy Redis, Zipkin also in the Kubernetes cluster") InitCmd.Flags().BoolVarP(&wait, "wait", "", false, "Wait for Kubernetes initialization to complete") InitCmd.Flags().UintVarP(&timeout, "timeout", "", 300, "The wait timeout for the Kubernetes installation") - InitCmd.Flags().BoolVarP(&slimMode, "slim", "s", false, "Exclude placement service, Redis and Zipkin containers from self-hosted installation") + InitCmd.Flags().BoolVarP(&slimMode, "slim", "s", false, "Exclude placement service, scheduler service, Redis and Zipkin containers from self-hosted installation") InitCmd.Flags().StringVarP(&runtimeVersion, "runtime-version", "", defaultRuntimeVersion, "The version of the Dapr runtime to install, for example: 1.0.0") InitCmd.Flags().StringVarP(&dashboardVersion, "dashboard-version", "", defaultDashboardVersion, "The version of the Dapr dashboard to install, for example: 0.13.0") InitCmd.Flags().StringVarP(&initNamespace, "namespace", "n", "dapr-system", "The Kubernetes namespace to install Dapr in") diff --git a/cmd/run.go b/cmd/run.go index 773ed0e09..4b5959ec0 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -120,6 +120,7 @@ dapr run --run-file /path/to/directory -k Args: cobra.MinimumNArgs(0), PreRun: func(cmd *cobra.Command, args []string) { viper.BindPFlag("placement-host-address", cmd.Flags().Lookup("placement-host-address")) + viper.BindPFlag("scheduler-host-address", cmd.Flags().Lookup("scheduler-host-address")) }, Run: func(cmd *cobra.Command, args []string) { if len(runFilePath) > 0 { @@ -171,6 +172,7 @@ dapr run --run-file /path/to/directory -k MaxConcurrency: maxConcurrency, AppProtocol: protocol, PlacementHostAddr: viper.GetString("placement-host-address"), + SchedulerHostAddr: viper.GetString("scheduler-host-address"), ComponentsPath: componentsPath, ResourcesPaths: resourcesPaths, AppSSL: appSSL, @@ -454,6 +456,7 @@ func init() { // By marking this as deprecated, the flag will be hidden from the help menu, but will continue to work. It will show a warning message when used. RunCmd.Flags().MarkDeprecated("components-path", "This flag is deprecated and will be removed in the future releases. Use \"resources-path\" flag instead") RunCmd.Flags().String("placement-host-address", "localhost", "The address of the placement service. Format is either for default port or : for custom port") + RunCmd.Flags().String("scheduler-host-address", "localhost", "The address of the scheduler service. Format is either for default port or : for custom port") // TODO: Remove below flag once the flag is removed in runtime in future release. RunCmd.Flags().BoolVar(&appSSL, "app-ssl", false, "Enable https when Dapr invokes the application") RunCmd.Flags().MarkDeprecated("app-ssl", "This flag is deprecated and will be removed in the future releases. Use \"app-protocol\" flag with https or grpcs values instead") diff --git a/cmd/uninstall.go b/cmd/uninstall.go index f451dd781..b8b672a11 100644 --- a/cmd/uninstall.go +++ b/cmd/uninstall.go @@ -43,7 +43,7 @@ var UninstallCmd = &cobra.Command{ # Uninstall from self-hosted mode dapr uninstall -# Uninstall from self-hosted mode and remove .dapr directory, Redis, Placement and Zipkin containers +# Uninstall from self-hosted mode and remove .dapr directory, Redis, Placement, Scheduler, and Zipkin containers dapr uninstall --all # Uninstall from Kubernetes @@ -99,7 +99,7 @@ func init() { UninstallCmd.Flags().BoolVarP(&uninstallKubernetes, "kubernetes", "k", false, "Uninstall Dapr from a Kubernetes cluster") UninstallCmd.Flags().BoolVarP(&uninstallDev, "dev", "", false, "Uninstall Dapr Redis and Zipking installations from Kubernetes cluster") UninstallCmd.Flags().UintVarP(&timeout, "timeout", "", 300, "The timeout for the Kubernetes uninstall") - UninstallCmd.Flags().BoolVar(&uninstallAll, "all", false, "Remove .dapr directory, Redis, Placement and Zipkin containers on local machine, and CRDs on a Kubernetes cluster") + UninstallCmd.Flags().BoolVar(&uninstallAll, "all", false, "Remove .dapr directory, Redis, Placement, Scheduler, and Zipkin containers on local machine, and CRDs on a Kubernetes cluster") UninstallCmd.Flags().String("network", "", "The Docker network from which to remove the Dapr runtime") UninstallCmd.Flags().StringVarP(&uninstallNamespace, "namespace", "n", "dapr-system", "The Kubernetes namespace to uninstall Dapr from") UninstallCmd.Flags().BoolP("help", "h", false, "Print this help message") diff --git a/go.mod b/go.mod index 59ef0a067..ea59e3d7b 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.3 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect diff --git a/go.sum b/go.sum index 74bb86d87..deb41fa5b 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6 github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= diff --git a/pkg/standalone/bundle.go b/pkg/standalone/bundle.go index b1f933233..bef6b8bc6 100644 --- a/pkg/standalone/bundle.go +++ b/pkg/standalone/bundle.go @@ -55,10 +55,10 @@ func isStringNilOrEmpty(val *string) bool { return val == nil || strings.TrimSpace(*val) == "" } -func (b *bundleDetails) getPlacementImageName() string { +func (b *bundleDetails) getDaprImageName() string { return *b.DaprImageName } -func (b *bundleDetails) getPlacementImageFileName() string { +func (b *bundleDetails) getDaprImageFileName() string { return *b.DaprImageFileName } diff --git a/pkg/standalone/bundle_test.go b/pkg/standalone/bundle_test.go index ae3d70eb7..3c5564a50 100644 --- a/pkg/standalone/bundle_test.go +++ b/pkg/standalone/bundle_test.go @@ -44,8 +44,8 @@ func TestParseDetails(t *testing.T) { assert.Equal(t, "0.10.0", *bd.DashboardVersion, "expected versions to match") assert.Equal(t, "dist", *bd.BinarySubDir, "expected value to match") assert.Equal(t, "docker", *bd.ImageSubDir, "expected value to match") - assert.Equal(t, "daprio/dapr:1.7.2", bd.getPlacementImageName(), "expected value to match") - assert.Equal(t, "daprio-dapr-1.7.2.tar.gz", bd.getPlacementImageFileName(), "expected value to match") + assert.Equal(t, "daprio/dapr:1.7.2", bd.getDaprImageName(), "expected value to match") + assert.Equal(t, "daprio-dapr-1.7.2.tar.gz", bd.getDaprImageFileName(), "expected value to match") } func TestParseDetailsMissingDetails(t *testing.T) { diff --git a/pkg/standalone/common.go b/pkg/standalone/common.go index ea077abc8..c47316ac0 100644 --- a/pkg/standalone/common.go +++ b/pkg/standalone/common.go @@ -17,6 +17,7 @@ import ( "os" path_filepath "path/filepath" "runtime" + "strconv" "strings" ) @@ -25,8 +26,10 @@ const ( DefaultConfigFileName = "config.yaml" DefaultResourcesDirName = "resources" - defaultDaprBinDirName = "bin" - defaultComponentsDirName = "components" + defaultDaprBinDirName = "bin" + defaultComponentsDirName = "components" + defaultSchedulerDirName = "scheduler" + defaultSchedulerDataDirName = "data" ) // GetDaprRuntimePath returns the dapr runtime installation path. @@ -58,6 +61,16 @@ func getDaprBinPath(daprDir string) string { return path_filepath.Join(daprDir, defaultDaprBinDirName) } +// getSchedulerDataPath returns the data path of a given instance +// Receiving instanceID allows multiple instances of scheduler to run locally in the future. +func getSchedulerDataPath(daprDir string, instanceID int) string { + return path_filepath.Join( + daprDir, + defaultSchedulerDirName, + defaultSchedulerDataDirName, + strconv.Itoa(instanceID)) +} + func binaryFilePathWithDir(binaryDir string, binaryFilePrefix string) string { binaryPath := path_filepath.Join(binaryDir, binaryFilePrefix) if runtime.GOOS == daprWindowsOS { diff --git a/pkg/standalone/run.go b/pkg/standalone/run.go index 6b4156e5a..6f1f81127 100644 --- a/pkg/standalone/run.go +++ b/pkg/standalone/run.go @@ -70,6 +70,8 @@ type SharedRunConfig struct { MaxConcurrency int `arg:"app-max-concurrency" annotation:"dapr.io/app-max-concurrerncy" yaml:"appMaxConcurrency" default:"-1"` // Speicifcally omitted from annotations similar to config file path above. PlacementHostAddr string `arg:"placement-host-address" yaml:"placementHostAddress"` + // Must use env for scheduler host address because using arg would cause a sidecar crash in older daprd versions. + SchedulerHostAddr string `env:"DAPR_SCHEDULER_HOST_ADDRESS" yaml:"schedulerHostAddress"` // Speicifcally omitted from annotations similar to config file path above. ComponentsPath string `arg:"components-path"` // Deprecated in run template file: use ResourcesPaths instead. // Speicifcally omitted from annotations similar to config file path above. @@ -136,6 +138,22 @@ func (config *RunConfig) validatePlacementHostAddr() error { return nil } +func (config *RunConfig) validateSchedulerHostAddr() error { + schedulerHostAddr := config.SchedulerHostAddr + if len(schedulerHostAddr) == 0 { + schedulerHostAddr = "localhost" + } + if indx := strings.Index(schedulerHostAddr, ":"); indx == -1 { + if runtime.GOOS == daprWindowsOS { + schedulerHostAddr = fmt.Sprintf("%s:6060", schedulerHostAddr) + } else { + schedulerHostAddr = fmt.Sprintf("%s:50006", schedulerHostAddr) + } + } + config.SchedulerHostAddr = schedulerHostAddr + return nil +} + func (config *RunConfig) validatePort(portName string, portPtr *int, meta *DaprMeta) error { if *portPtr <= 0 { port, err := freeport.GetFreePort() @@ -214,6 +232,12 @@ func (config *RunConfig) Validate() error { } err = config.validatePlacementHostAddr() + if err != nil { + return err + } + + err = config.validateSchedulerHostAddr() + if err != nil { return err } diff --git a/pkg/standalone/standalone.go b/pkg/standalone/standalone.go index 64e7e2905..f25bf9bfb 100644 --- a/pkg/standalone/standalone.go +++ b/pkg/standalone/standalone.go @@ -31,6 +31,7 @@ import ( "sync" "time" + "github.com/Masterminds/semver" "github.com/fatih/color" "gopkg.in/yaml.v2" @@ -43,6 +44,7 @@ const ( daprRuntimeFilePrefix = "daprd" dashboardFilePrefix = "dashboard" placementServiceFilePrefix = "placement" + schedulerServiceFilePrefix = "scheduler" daprWindowsOS = "windows" @@ -72,6 +74,8 @@ const ( // DaprPlacementContainerName is the container name of placement service. DaprPlacementContainerName = "dapr_placement" + // DaprSchedulerContainerName is the container name of scheduler service. + DaprSchedulerContainerName = "dapr_scheduler" // DaprRedisContainerName is the container name of redis. DaprRedisContainerName = "dapr_redis" // DaprZipkinContainerName is the container name of zipkin. @@ -81,6 +85,12 @@ const ( healthPort = 58080 metricPort = 59090 + + schedulerHealthPort = 58081 + schedulerMetricPort = 59091 + schedulerEtcdPort = 52379 + + daprVersionsWithScheduler = ">= 1.14.x" ) var ( @@ -154,6 +164,21 @@ func isBinaryInstallationRequired(binaryFilePrefix, binInstallDir string) (bool, return true, nil } +// isSchedulerIncluded returns true if scheduler is included a given version for Dapr. +func isSchedulerIncluded(runtimeVersion string) (bool, error) { + c, err := semver.NewConstraint(daprVersionsWithScheduler) + if err != nil { + return false, err + } + + v, err := semver.NewVersion(runtimeVersion) + if err != nil { + return false, err + } + + return c.Check(v), nil +} + // Init installs Dapr on a local machine using the supplied runtimeVersion. func Init(runtimeVersion, dashboardVersion string, dockerNetwork string, slimMode bool, imageRegistryURL string, fromDir string, containerRuntime string, imageVariant string, daprInstallPath string) error { var err error @@ -240,8 +265,10 @@ func Init(runtimeVersion, dashboardVersion string, dockerNetwork string, slimMod createComponentsAndConfiguration, installDaprRuntime, installPlacement, + installScheduler, installDashboard, runPlacementService, + runSchedulerService, runRedis, runZipkin, } @@ -302,6 +329,7 @@ func Init(runtimeVersion, dashboardVersion string, dockerNetwork string, slimMod if slimMode { // Print info on placement binary only on slim install. print.InfoStatusEvent(os.Stdout, "%s binary has been installed to %s.", placementServiceFilePrefix, daprBinDir) + print.InfoStatusEvent(os.Stdout, "%s binary has been installed to %s.", schedulerServiceFilePrefix, daprBinDir) } else { runtimeCmd := utils.GetContainerRuntimeCmd(info.containerRuntime) dockerContainerNames := []string{DaprPlacementContainerName, DaprRedisContainerName, DaprZipkinContainerName} @@ -309,6 +337,10 @@ func Init(runtimeVersion, dashboardVersion string, dockerNetwork string, slimMod if isAirGapInit { dockerContainerNames = []string{DaprPlacementContainerName} } + hasScheduler, err := isSchedulerIncluded(info.runtimeVersion) + if err == nil && hasScheduler { + dockerContainerNames = append(dockerContainerNames, DaprSchedulerContainerName) + } for _, container := range dockerContainerNames { containerName := utils.CreateContainerName(container, dockerNetwork) ok, err := confirmContainerIsRunningOrExists(containerName, true, runtimeCmd) @@ -489,15 +521,15 @@ func runPlacementService(wg *sync.WaitGroup, errorChan chan<- error, info initIn if isAirGapInit { // if --from-dir flag is given load the image details from the installer-bundle. dir := path_filepath.Join(info.fromDir, *info.bundleDet.ImageSubDir) - image = info.bundleDet.getPlacementImageName() - err = loadContainer(dir, info.bundleDet.getPlacementImageFileName(), info.containerRuntime) + image = info.bundleDet.getDaprImageName() + err = loadContainer(dir, info.bundleDet.getDaprImageFileName(), info.containerRuntime) if err != nil { errorChan <- err return } } else { // otherwise load the image from the specified repository. - image, err = getPlacementImageName(imgInfo, info) + image, err = getDaprImageName(imgInfo, info) if err != nil { errorChan <- err return @@ -545,6 +577,106 @@ func runPlacementService(wg *sync.WaitGroup, errorChan chan<- error, info initIn errorChan <- nil } +func runSchedulerService(wg *sync.WaitGroup, errorChan chan<- error, info initInfo) { + defer wg.Done() + + if info.slimMode { + return + } + + hasScheduler, err := isSchedulerIncluded(info.runtimeVersion) + if err != nil { + errorChan <- err + return + } + if !hasScheduler { + return + } + + runtimeCmd := utils.GetContainerRuntimeCmd(info.containerRuntime) + schedulerContainerName := utils.CreateContainerName(DaprSchedulerContainerName, info.dockerNetwork) + + exists, err := confirmContainerIsRunningOrExists(schedulerContainerName, false, runtimeCmd) + + if err != nil { + errorChan <- err + return + } else if exists { + errorChan <- fmt.Errorf("%s container exists or is running. %s", schedulerContainerName, errInstallTemplate) + return + } + var image string + + imgInfo := daprImageInfo{ + ghcrImageName: daprGhcrImageName, + dockerHubImageName: daprDockerImageName, + imageRegistryURL: info.imageRegistryURL, + imageRegistryName: defaultImageRegistryName, + } + + if isAirGapInit { + // if --from-dir flag is given load the image details from the installer-bundle. + dir := path_filepath.Join(info.fromDir, *info.bundleDet.ImageSubDir) + image = info.bundleDet.getDaprImageName() + err = loadContainer(dir, info.bundleDet.getDaprImageFileName(), info.containerRuntime) + if err != nil { + errorChan <- err + return + } + } else { + // otherwise load the image from the specified repository. + image, err = getDaprImageName(imgInfo, info) + if err != nil { + errorChan <- err + return + } + } + + // instanceID is 0 because we run one instance only for now. + schedulerDataDir := getSchedulerDataPath(info.installDir, 0) + args := []string{ + "run", + "--name", schedulerContainerName, + "--restart", "always", + "-d", + "--entrypoint", "./scheduler", + "--volume", fmt.Sprintf("%v:/data-default-dapr-scheduler-server-0", schedulerDataDir), + } + + if info.dockerNetwork != "" { + args = append(args, + "--network", info.dockerNetwork, + "--network-alias", DaprSchedulerContainerName) + } else { + osPort := 50006 + if runtime.GOOS == daprWindowsOS { + osPort = 6060 + } + + args = append(args, + "-p", fmt.Sprintf("%v:50006", osPort), + "-p", fmt.Sprintf("%v:2379", schedulerEtcdPort), + "-p", fmt.Sprintf("%v:8080", schedulerHealthPort), + "-p", fmt.Sprintf("%v:9090", schedulerMetricPort), + ) + } + + args = append(args, image) + + _, err = utils.RunCmdAndWait(runtimeCmd, args...) + + if err != nil { + runError := isContainerRunError(err) + if !runError { + errorChan <- parseContainerRuntimeError("scheduler service", err) + } else { + errorChan <- fmt.Errorf("%s %s failed with: %w", runtimeCmd, args, err) + } + return + } + errorChan <- nil +} + func moveDashboardFiles(extractedFilePath string, dir string) (string, error) { // Move /release/os/web directory to /web. oldPath := path_filepath.Join(path_filepath.Dir(extractedFilePath), "web") @@ -609,7 +741,29 @@ func installPlacement(wg *sync.WaitGroup, errorChan chan<- error, info initInfo) } } -// installBinary installs the daprd, placement or dashboard binaries and associated files inside the default dapr bin directory. +func installScheduler(wg *sync.WaitGroup, errorChan chan<- error, info initInfo) { + defer wg.Done() + + if !info.slimMode { + return + } + + hasScheduler, err := isSchedulerIncluded(info.runtimeVersion) + if err != nil { + errorChan <- err + return + } + if !hasScheduler { + return + } + + err = installBinary(info.runtimeVersion, schedulerServiceFilePrefix, cli_ver.DaprGitHubRepo, info) + if err != nil { + errorChan <- err + } +} + +// installBinary installs the daprd, placement, scheduler, or dashboard binaries and associated files inside the default dapr bin directory. func installBinary(version, binaryFilePrefix, githubRepo string, info initInfo) error { var ( err error @@ -1189,16 +1343,16 @@ func copyWithTimeout(ctx context.Context, dst io.Writer, src io.Reader) (int64, } } -// getPlacementImageName returns the resolved placement image name for online `dapr init`. +// getDaprImageName returns the resolved Dapr image name for online `dapr init`. // It can either be resolved to the image-registry if given, otherwise GitHub container registry if // selected or fallback to Docker Hub. -func getPlacementImageName(imageInfo daprImageInfo, info initInfo) (string, error) { +func getDaprImageName(imageInfo daprImageInfo, info initInfo) (string, error) { image, err := resolveImageURI(imageInfo) if err != nil { return "", err } - image, err = getPlacementImageWithTag(image, info.runtimeVersion, info.imageVariant) + image, err = getDaprImageWithTag(image, info.runtimeVersion, info.imageVariant) if err != nil { return "", err } @@ -1206,8 +1360,8 @@ func getPlacementImageName(imageInfo daprImageInfo, info initInfo) (string, erro // if default registry is GHCR and the image is not available in or cannot be pulled from GHCR // fallback to using dockerhub. if useGHCR(imageInfo, info.fromDir) && !tryPullImage(image, info.containerRuntime) { - print.InfoStatusEvent(os.Stdout, "Placement image not found in Github container registry, pulling it from Docker Hub") - image, err = getPlacementImageWithTag(daprDockerImageName, info.runtimeVersion, info.imageVariant) + print.InfoStatusEvent(os.Stdout, "Image not found in Github container registry, pulling it from Docker Hub") + image, err = getDaprImageWithTag(daprDockerImageName, info.runtimeVersion, info.imageVariant) if err != nil { return "", err } @@ -1215,7 +1369,7 @@ func getPlacementImageName(imageInfo daprImageInfo, info initInfo) (string, erro return image, nil } -func getPlacementImageWithTag(name, version, imageVariant string) (string, error) { +func getDaprImageWithTag(name, version, imageVariant string) (string, error) { err := utils.ValidateImageVariant(imageVariant) if err != nil { return "", err diff --git a/pkg/standalone/uninstall.go b/pkg/standalone/uninstall.go index c7431f561..0162cbe43 100644 --- a/pkg/standalone/uninstall.go +++ b/pkg/standalone/uninstall.go @@ -24,13 +24,17 @@ import ( "github.com/dapr/cli/utils" ) -func removeContainers(uninstallPlacementContainer, uninstallAll bool, dockerNetwork, runtimeCmd string) []error { +func removeContainers(uninstallPlacementContainer, uninstallSchedulerContainer, uninstallAll bool, dockerNetwork, runtimeCmd string) []error { var containerErrs []error if uninstallPlacementContainer { containerErrs = removeDockerContainer(containerErrs, DaprPlacementContainerName, dockerNetwork, runtimeCmd) } + if uninstallSchedulerContainer { + containerErrs = removeDockerContainer(containerErrs, DaprSchedulerContainerName, dockerNetwork, runtimeCmd) + } + if uninstallAll { containerErrs = removeDockerContainer(containerErrs, DaprRedisContainerName, dockerNetwork, runtimeCmd) containerErrs = removeDockerContainer(containerErrs, DaprZipkinContainerName, dockerNetwork, runtimeCmd) @@ -84,19 +88,27 @@ func Uninstall(uninstallAll bool, dockerNetwork string, containerRuntime string, placementFilePath := binaryFilePathWithDir(daprBinDir, placementServiceFilePrefix) _, placementErr := os.Stat(placementFilePath) // check if the placement binary exists. uninstallPlacementContainer := errors.Is(placementErr, fs.ErrNotExist) + + schedulerFilePath := binaryFilePathWithDir(daprBinDir, schedulerServiceFilePrefix) + _, schedulerErr := os.Stat(schedulerFilePath) // check if the scheduler binary exists. + uninstallSchedulerContainer := errors.Is(schedulerErr, fs.ErrNotExist) + // Remove .dapr/bin. err = removeDir(daprBinDir) if err != nil { print.WarningStatusEvent(os.Stdout, "WARNING: could not delete dapr bin dir: %s", daprBinDir) } + // We don't delete .dapr/scheduler by choice since it holds state. + // To delete .dapr/scheduler, user is expected to use the `--all` flag as it deletes the .dapr folder. + // The same happens for .dapr/components folder. containerRuntime = strings.TrimSpace(containerRuntime) runtimeCmd := utils.GetContainerRuntimeCmd(containerRuntime) containerRuntimeAvailable := false containerRuntimeAvailable = utils.IsContainerRuntimeInstalled(containerRuntime) if containerRuntimeAvailable { - containerErrs = removeContainers(uninstallPlacementContainer, uninstallAll, dockerNetwork, runtimeCmd) - } else if uninstallPlacementContainer || uninstallAll { + containerErrs = removeContainers(uninstallPlacementContainer, uninstallSchedulerContainer, uninstallAll, dockerNetwork, runtimeCmd) + } else if uninstallSchedulerContainer || uninstallPlacementContainer || uninstallAll { print.WarningStatusEvent(os.Stdout, "WARNING: could not delete supporting containers as container runtime is not installed or running") }