Skip to content

Commit

Permalink
chrooted bash executor
Browse files Browse the repository at this point in the history
Signed-off-by: Mikhail Scherba <mikhail.scherba@flant.com>
  • Loading branch information
miklezzzz committed Feb 18, 2025
1 parent 31473d9 commit 4aa6ba0
Show file tree
Hide file tree
Showing 17 changed files with 653 additions and 67 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/dominikbraun/graph v0.23.0
github.com/ettle/strcase v0.2.0
github.com/flant/kube-client v1.2.2
github.com/flant/shell-operator v1.5.4-0.20250205135215-f632bb655900
github.com/flant/shell-operator v1.5.4-0.20250217150012-87c1974516fb
github.com/go-chi/chi/v5 v5.2.0
github.com/go-openapi/loads v0.19.5
github.com/go-openapi/spec v0.19.8
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -138,8 +138,8 @@ github.com/flant/kube-client v1.2.2 h1:27LBs+PKJEFnkQXjPU9eIps7a7iyI13AKcSYj897D
github.com/flant/kube-client v1.2.2/go.mod h1:eMa3aJ6V1PRWSQ/RCROkObDpY4S74uM84SJS4G/LINg=
github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee h1:evii83J+/6QGNvyf6tjQ/p27DPY9iftxIBb37ALJRTg=
github.com/flant/libjq-go v1.6.3-0.20201126171326-c46a40ff22ee/go.mod h1:f+REaGl/+pZR97rbTcwHEka/MAipoQQ2Mc0iQUj4ak0=
github.com/flant/shell-operator v1.5.4-0.20250205135215-f632bb655900 h1:CLG+boH2YkiJykuXZEGUncGjGYk7WgFMJHe1gq9Jdbk=
github.com/flant/shell-operator v1.5.4-0.20250205135215-f632bb655900/go.mod h1:pyR9mte3tgcocQJPgyTH2wTzm6JsQQOuRElrd92O2Ks=
github.com/flant/shell-operator v1.5.4-0.20250217150012-87c1974516fb h1:gyfUDNxEp1/8zuApwPXKds02/NspEpXMyvLWpbKBfM8=
github.com/flant/shell-operator v1.5.4-0.20250217150012-87c1974516fb/go.mod h1:pyR9mte3tgcocQJPgyTH2wTzm6JsQQOuRElrd92O2Ks=
github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI=
Expand Down
7 changes: 6 additions & 1 deletion pkg/addon-operator/admission_http_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package addon_operator

import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
Expand Down Expand Up @@ -57,7 +58,11 @@ func (as *AdmissionServer) start(ctx context.Context) {
cert := path.Join(as.certsDir, "tls.crt")
key := path.Join(as.certsDir, "tls.key")
if err := srv.ListenAndServeTLS(cert, key); err != nil {
log.Fatal("admission server listen and serve tls", log.Err(err))
if errors.Is(err, http.ErrServerClosed) {
log.Info("admission server stopped")
} else {
log.Fatal("admission server listen and serve tls", log.Err(err))
}
}
}()

Expand Down
1 change: 1 addition & 0 deletions pkg/addon-operator/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func (op *AddonOperator) SetupModuleManager(modulesDir string, globalHooksDir st
ModulesDir: modulesDir,
GlobalHooksDir: globalHooksDir,
TempDir: tempDir,
ChrootDir: app.ShellChrootDir,
}
deps := module_manager.ModuleManagerDependencies{
KubeObjectPatcher: op.engine.ObjectPatcher,
Expand Down
17 changes: 10 additions & 7 deletions pkg/addon-operator/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -832,14 +832,17 @@ func (op *AddonOperator) HandleConvergeModules(t sh_task.Task, logLabels map[str
enabledModules[enabledModule] = struct{}{}
}

for _, moduleName := range op.ModuleManager.GetModuleNames() {
if _, enabled := enabledModules[moduleName]; !enabled {
op.ModuleManager.SendModuleEvent(events.ModuleEvent{
ModuleName: moduleName,
EventType: events.ModuleDisabled,
})
logEntry.Debug("ConvergeModules: send module disabled events")
go func() {
for _, moduleName := range op.ModuleManager.GetModuleNames() {
if _, enabled := enabledModules[moduleName]; !enabled {
op.ModuleManager.SendModuleEvent(events.ModuleEvent{
ModuleName: moduleName,
EventType: events.ModuleDisabled,
})
}
}
}
}()
}
tasks := op.CreateConvergeModulesTasks(state, t.GetLogLabels(), string(taskEvent))

Expand Down
6 changes: 6 additions & 0 deletions pkg/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ var (

GlobalHooksDir = "global-hooks"
ModulesDir = "modules"
ShellChrootDir = ""

UnnumberedModuleOrder = 1

Expand Down Expand Up @@ -166,6 +167,11 @@ func DefineStartCommandFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause)
Default(CRDsFilters).
StringVar(&CRDsFilters)

cmd.Flag("shell-chroot-dir", "Defines the path where shell scripts (shell hooks and enabled scripts) will be chrooted to.").
Envar("ADDON_OPERATOR_SHELL_CHROOT_DIR").
Default("").
StringVar(&ShellChrootDir)

shapp.DefineKubeClientFlags(cmd)
shapp.DefineJqFlags(cmd)
shapp.DefineLoggingFlags(cmd)
Expand Down
225 changes: 225 additions & 0 deletions pkg/module_manager/environment_manager/evironment_manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,225 @@
package environment_manager

import (
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"sync"
"syscall"

"github.com/deckhouse/deckhouse/pkg/log"

"github.com/flant/addon-operator/pkg/utils"
)

type (
Type string
Environment int
)

const (
Mount Type = "mount"
File Type = "file"
DevNull Type = "devNull"
)

const (
NoEnvironment Environment = iota
EnabledScriptEnvironment
ShellHookEnvironment
)

const testsEnv = "ADDON_OPERATOR_IS_TESTS_ENVIRONMENT"

type ObjectDescriptor struct {
Source string
Target string
Flags uintptr
Type Type
TargetEnvironment Environment
}

type Manager struct {
objects map[string]ObjectDescriptor
chroot string

l sync.Mutex
preparedEnvironments map[string]Environment

logger *log.Logger
}

func NewManager(chroot string, logger *log.Logger) *Manager {
return &Manager{
preparedEnvironments: make(map[string]Environment),
chroot: chroot,
objects: make(map[string]ObjectDescriptor),
logger: logger,
}
}

func (m *Manager) AddObjectsToEnvironment(objects ...ObjectDescriptor) {
m.l.Lock()
for _, object := range objects {
m.objects[object.Source] = object
}
m.l.Unlock()
}

func makedev(majorNumber int64, minorNumber int64) int {
return int((majorNumber << 8) | (minorNumber & 0xff) | ((minorNumber & 0xfff00) << 12))
}

func (m *Manager) DisassembleEnvironmentForModule(moduleName, modulePath string, targetEnvironment Environment) error {
logEntry := utils.EnrichLoggerWithLabels(m.logger, map[string]string{
"operator.component": "EnvironmentManager.DisassembleEnvironmentForModule",
})
m.l.Lock()
defer m.l.Unlock()

currentEnvironment := m.preparedEnvironments[moduleName]
if currentEnvironment == NoEnvironment || currentEnvironment == targetEnvironment {
return nil
}

logEntry.Debug("Disassembling environment",
slog.String("module", moduleName),
slog.Any("currentEnvironment", currentEnvironment),
slog.Any("targetEnvironment", targetEnvironment))

chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName)
for _, properties := range m.objects {
if properties.TargetEnvironment > targetEnvironment && properties.TargetEnvironment <= currentEnvironment {
var chrootedObjectPath string
if len(properties.Target) > 0 {
chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Target)
} else {
chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Source)
}

switch properties.Type {
case File, DevNull:
if err := os.Remove(chrootedObjectPath); err != nil {
return fmt.Errorf("delete file %q: %w", chrootedObjectPath, err)
}

case Mount:
if err := syscall.Unmount(chrootedObjectPath, 0); err != nil {
return fmt.Errorf("unmount folder %q: %w", chrootedObjectPath, err)
}
}
}
}

if targetEnvironment == NoEnvironment {
if os.Getenv(testsEnv) != "true" {
chrootedModuleDir := filepath.Join(chrootedModuleEnvPath, modulePath)
if err := syscall.Unmount(chrootedModuleDir, 0); err != nil {
return fmt.Errorf("unmount %q module's dir: %w", modulePath, err)
}
}

delete(m.preparedEnvironments, moduleName)
} else {
m.preparedEnvironments[moduleName] = targetEnvironment
}

return nil
}

func (m *Manager) AssembleEnvironmentForModule(moduleName, modulePath string, targetEnvironment Environment) error {
logEntry := utils.EnrichLoggerWithLabels(m.logger, map[string]string{
"operator.component": "EnvironmentManager.AssembleEnvironmentForModule",
})

m.l.Lock()
defer m.l.Unlock()

currentEnvironment := m.preparedEnvironments[moduleName]
if currentEnvironment >= targetEnvironment {
return nil
}

logEntry.Debug("Preparing environment",
slog.String("module", moduleName),
slog.Any("currentEnvironment", currentEnvironment),
slog.Any("targetEnvironment", targetEnvironment))

chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName)

if currentEnvironment == NoEnvironment {
logEntry.Debug("Preparing environment - creating the module's directory",
slog.String("module", moduleName),
slog.Any("currentEnvironment", currentEnvironment),
slog.Any("targetEnvironment", targetEnvironment))

chrootedModuleDir := filepath.Join(chrootedModuleEnvPath, modulePath)
if err := os.MkdirAll(chrootedModuleDir, 0o755); err != nil {
return fmt.Errorf("make %q module's dir: %w", modulePath, err)
}

if os.Getenv(testsEnv) != "true" {
if err := syscall.Mount(modulePath, chrootedModuleDir, "", syscall.MS_BIND|syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("mount %q module's dir: %w", modulePath, err)
}
}
}

for _, properties := range m.objects {
if properties.TargetEnvironment != currentEnvironment && properties.TargetEnvironment <= targetEnvironment {
var chrootedObjectPath string
if len(properties.Target) > 0 {
chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Target)
} else {
chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Source)
}

switch properties.Type {
case File:
if err := os.MkdirAll(filepath.Dir(chrootedObjectPath), 0o755); err != nil {
return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err)
}

bytesRead, err := os.ReadFile(properties.Source)
if err != nil {
return fmt.Errorf("read from file %q: %w", properties.Source, err)
}

if err = os.WriteFile(chrootedObjectPath, bytesRead, 0o644); err != nil {
return fmt.Errorf("write to file %q: %w", chrootedObjectPath, err)
}

case DevNull:
if err := os.MkdirAll(filepath.Dir(chrootedObjectPath), 0o755); err != nil {
return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err)
}

if err := syscall.Mknod(chrootedObjectPath, syscall.S_IFCHR|0o666, makedev(1, 3)); err != nil {
if errors.Is(err, os.ErrExist) {
continue
}
return fmt.Errorf("create null file: %w", err)
}

if err := os.Chmod(chrootedObjectPath, 0o666); err != nil {
return fmt.Errorf("chmod %q file: %w", chrootedObjectPath, err)
}

case Mount:
if err := os.MkdirAll(chrootedObjectPath, 0o755); err != nil {
return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err)
}

if err := syscall.Mount(properties.Source, chrootedObjectPath, "", properties.Flags, ""); err != nil {
return fmt.Errorf("mount folder %q: %w", chrootedObjectPath, err)
}
}
}
}

m.preparedEnvironments[moduleName] = targetEnvironment

return nil
}
Loading

0 comments on commit 4aa6ba0

Please sign in to comment.