From 2081d13f47485debe243dfb6a69cdaab155afa23 Mon Sep 17 00:00:00 2001 From: Jmnote Date: Sun, 6 Oct 2024 05:18:20 +0900 Subject: [PATCH] main & tester (#120) * main & tester --- app/app.go | 57 +++ app/app_test.go | 108 +++++ cmd/lethe/main.go | 63 +-- cmd/lethe/main_test.go | 64 +-- config/config.go | 164 +++++--- config/config_test.go | 187 +++++---- etc/lethe.example.yaml | 4 +- go.mod | 6 +- go.sum | 9 +- letheql/engine.go | 30 +- letheql/engine_test.go | 157 ++++---- letheql/evaluator_test.go | 396 ++++++++++--------- letheql/init_test.go | 47 --- rotator/rotator.go | 14 +- rotator/rotator_test.go | 144 ++++--- router/init_test.go | 46 --- router/metadata_test.go | 10 +- router/probe_test.go | 16 +- router/query.go | 11 +- router/query_test.go | 34 +- router/router.go | 3 +- router/router_test.go | 36 +- router/targets_test.go | 9 +- storage/driver/filesystem/driver_test.go | 342 ++++++++-------- storage/driver/filesystem/fileinfo_test.go | 41 +- storage/driver/filesystem/filewriter_test.go | 36 +- storage/driver/filesystem/init_test.go | 10 - storage/fileservice/clean_test.go | 45 +-- storage/fileservice/delete.go | 14 +- storage/fileservice/delete_test.go | 320 ++++++++------- storage/fileservice/fileservice.go | 10 +- storage/fileservice/fileservice_test.go | 50 +-- storage/fileservice/init_test.go | 26 -- storage/fileservice/list.go | 2 +- storage/fileservice/list_test.go | 111 ++++-- storage/logservice/init_test.go | 29 -- storage/logservice/logservice.go | 16 +- storage/logservice/logservice_test.go | 11 +- storage/querier/init_test.go | 12 - storage/querier/querier_test.go | 44 +++ storage/queryservice/init_test.go | 31 -- storage/queryservice/queryservice.go | 5 +- storage/queryservice/queryservice_test.go | 33 +- testdata/etc/lethe.error1.yaml | 4 +- testdata/etc/lethe.error2.yaml | 4 +- testdata/etc/lethe.error3.yaml | 4 +- testdata/etc/lethe.error4.yaml | 8 + testdata/etc/lethe.legacy.yaml | 6 + testdata/etc/lethe.main.yaml | 6 +- testdata/etc/lethe.ok1.yaml | 4 +- testdata/etc/lethe.ok2.yaml | 4 +- util/string.go | 47 --- util/string_test.go | 49 --- util/testutil/testutil.go | 102 ----- util/time.go | 56 --- util/time_test.go | 32 -- 56 files changed, 1530 insertions(+), 1599 deletions(-) create mode 100644 app/app.go create mode 100644 app/app_test.go delete mode 100644 letheql/init_test.go delete mode 100644 router/init_test.go delete mode 100644 storage/driver/filesystem/init_test.go delete mode 100644 storage/fileservice/init_test.go delete mode 100644 storage/logservice/init_test.go delete mode 100644 storage/querier/init_test.go create mode 100644 storage/querier/querier_test.go delete mode 100644 storage/queryservice/init_test.go create mode 100644 testdata/etc/lethe.error4.yaml create mode 100644 testdata/etc/lethe.legacy.yaml delete mode 100644 util/string.go delete mode 100644 util/string_test.go delete mode 100644 util/testutil/testutil.go delete mode 100644 util/time.go delete mode 100644 util/time_test.go diff --git a/app/app.go b/app/app.go new file mode 100644 index 0000000..c7d8a6a --- /dev/null +++ b/app/app.go @@ -0,0 +1,57 @@ +package app + +import ( + "fmt" + + "github.com/kuoss/common/logger" + "github.com/kuoss/lethe/config" + "github.com/kuoss/lethe/rotator" + "github.com/kuoss/lethe/router" + "github.com/kuoss/lethe/storage/fileservice" + "github.com/kuoss/lethe/storage/logservice" + "github.com/kuoss/lethe/storage/queryservice" +) + +type IApp interface { + New(version string) error + Run() error +} + +type App struct { + Config *config.Config + rotator *rotator.Rotator + router *router.Router +} + +// New creates a new App instance +func New(version string) (*App, error) { + // Load configuration + cfg, err := config.New(version) + if err != nil { + return nil, fmt.Errorf("failed to load configuration: %w", err) + } + logger.Infof("Loaded configuration: %v", cfg) + + // Initialize services + fileService, err := fileservice.New(cfg) + if err != nil { + return nil, fmt.Errorf("new fileservice err: %w", err) + } + logService := logservice.New(cfg, fileService) + queryService := queryservice.New(cfg, logService) + + // Create rotator & router + myRotator := rotator.New(cfg, fileService) + myRouter := router.New(cfg, fileService, queryService) + + return &App{ + Config: cfg, + rotator: myRotator, + router: myRouter, + }, nil +} + +func (a App) Run() error { + a.rotator.Start() + return a.router.Run() +} diff --git a/app/app_test.go b/app/app_test.go new file mode 100644 index 0000000..0e10fea --- /dev/null +++ b/app/app_test.go @@ -0,0 +1,108 @@ +package app + +import ( + "context" + "testing" + "time" + + "github.com/kuoss/common/tester" + "github.com/kuoss/lethe/config" + "github.com/stretchr/testify/require" +) + +func TestNew(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{}) + defer cleanup() + + wantConfig := &config.Config{ + Version: "test", + Query: config.Query{ + Limit: 1000, + Timeout: 20000000000, + }, + Retention: config.Retention{ + Size: 0, + Time: 15 * 24 * time.Hour, // 15d + SizingStrategy: "file", + RotationInterval: 20 * time.Second, + }, + Storage: config.Storage{LogDataPath: "data/log"}, + Web: config.Web{ + ListenAddress: ":6060", + GinMode: "release", + }, + } + + app, err := New("test") + require.NoError(t, err) + require.NotEmpty(t, app) + require.NotEmpty(t, app.rotator) + require.NotEmpty(t, app.router) + require.Equal(t, wantConfig, app.Config) +} + +func TestNew_error1(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/etc/lethe.error1.yaml": "etc/lethe.yaml", + }) + defer cleanup() + + app, err := New("test") + require.Error(t, err) + require.Nil(t, app) +} + +func TestNew_error2(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/etc/lethe.error1.yaml": "etc/lethe.yaml", + }) + defer cleanup() + + app, err := New("test") + require.Error(t, err) + require.Nil(t, app) +} + +func TestRun_error4(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/etc/lethe.error4.yaml": "etc/lethe.yaml", + }) + defer cleanup() + + app, err := New("test") + require.NoError(t, err) + + err = app.Run() + require.EqualError(t, err, "listen tcp: address foo: missing port in address") +} + +func TestRun_smokeTest(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{}) + defer cleanup() + + app, err := New("test") + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.TODO(), time.Second) + defer cancel() + panicChan := make(chan interface{}, 1) + go func() { + defer func() { + if r := recover(); r != nil { + panicChan <- r + } + }() + err := app.Run() + require.NoError(t, err) + close(panicChan) + }() + + var done bool + select { + case <-ctx.Done(): + done = true + case p := <-panicChan: + t.Fatalf("panic occurred: %v", p) + } + require.True(t, done) +} diff --git a/cmd/lethe/main.go b/cmd/lethe/main.go index 8e5cb6c..aecba0e 100644 --- a/cmd/lethe/main.go +++ b/cmd/lethe/main.go @@ -1,66 +1,27 @@ package main import ( - "fmt" - "time" + "os" "github.com/kuoss/common/logger" - "github.com/kuoss/lethe/config" - "github.com/kuoss/lethe/rotator" - "github.com/kuoss/lethe/router" - "github.com/kuoss/lethe/storage/fileservice" - "github.com/kuoss/lethe/storage/logservice" - "github.com/kuoss/lethe/storage/queryservice" + "github.com/kuoss/lethe/app" ) var ( Version = "development" + + myApp app.IApp + exit = os.Exit ) func main() { - logger.Infof("💧 Lethe starting... version: %s", Version) - - // Load configuration - cfg := mustConfig(Version) - - // Initialize services - fileService := mustFileService(cfg) - logService := logservice.New(fileService) - queryService := queryservice.New(logService) - - // Start the rotator - startRotator(cfg, fileService) - - // Run the router - runRouter(cfg, fileService, queryService) -} - -func mustConfig(version string) *config.Config { - cfg, err := config.New(version) - if err != nil { - panic(fmt.Errorf("failed to create new config: %v", err)) + logger.Infof("Starting Lethe, version: %s", Version) + if err := myApp.New(Version); err != nil { + logger.Errorf("Failed to create app: %s", err.Error()) + exit(1) } - logger.Infof("Loaded configuration: %v", cfg) - return cfg -} - -func mustFileService(cfg *config.Config) *fileservice.FileService { - fileService, err := fileservice.New(cfg) - if err != nil { - logger.Fatalf("Failed to create new file service: %s", err.Error()) - } - return fileService -} - -func startRotator(cfg *config.Config, fileService *fileservice.FileService) { - rotator := rotator.New(cfg, fileService) - rotator.Start(20 * time.Minute) - logger.Infof("Log rotator started with interval: %v", 20*time.Minute) -} - -func runRouter(cfg *config.Config, fileService *fileservice.FileService, queryService *queryservice.QueryService) { - r := router.New(cfg, fileService, queryService) - if err := r.Run(); err != nil { - logger.Fatalf("Failed to run router: %s", err.Error()) + if err := myApp.Run(); err != nil { + logger.Errorf("Failed to run app: %s", err.Error()) + exit(1) } } diff --git a/cmd/lethe/main_test.go b/cmd/lethe/main_test.go index ea47505..67bd775 100644 --- a/cmd/lethe/main_test.go +++ b/cmd/lethe/main_test.go @@ -1,40 +1,52 @@ package main import ( + "errors" "testing" - "github.com/kuoss/common/tester" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestMustConfig(t *testing.T) { - version := "test" - cfg := mustConfig(version) - assert.NotEmpty(t, cfg) - assert.Equal(t, version, cfg.Version) +type MockApp struct { + newErr error + runErr error } -func TestMustFileService(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ - "@/testdata/etc/lethe.main.yaml": "etc/lethe.yaml", - }) - defer cleanup() +func (m *MockApp) New(version string) error { + return m.newErr +} - cfg := mustConfig("test") - fileService := mustFileService(cfg) - assert.NotNil(t, fileService, "Expected file service to be non-nil") +func (m *MockApp) Run() error { + return m.runErr } -func TestStartRotator(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ - "@/testdata/etc/lethe.main.yaml": "etc/lethe.yaml", +func TestMainFunc(t *testing.T) { + originalApp := myApp + originalExit := exit + defer func() { + myApp = originalApp + exit = originalExit + }() + + var gotExitCode int + exit = func(code int) { + gotExitCode = code + } + + t.Run("ok", func(t *testing.T) { + myApp = &MockApp{} + main() + require.Equal(t, 0, gotExitCode) + }) + t.Run("new error", func(t *testing.T) { + myApp = &MockApp{newErr: errors.New("fake new error")} + main() + require.Equal(t, 1, gotExitCode) + }) + + t.Run("run error", func(t *testing.T) { + myApp = &MockApp{runErr: errors.New("fake run error")} + main() + require.Equal(t, 1, gotExitCode) }) - defer cleanup() - - version := "test-version" - cfg := mustConfig(version) - fileService := mustFileService(cfg) - assert.NotPanics(t, func() { - startRotator(cfg, fileService) - }, "Expected startRotator to not panic") } diff --git a/config/config.go b/config/config.go index a490193..795d7a1 100644 --- a/config/config.go +++ b/config/config.go @@ -2,25 +2,48 @@ package config import ( "fmt" + "os" "path/filepath" + "strings" "time" + "github.com/alecthomas/units" + "github.com/gin-gonic/gin" + "github.com/prometheus/common/model" + "github.com/spf13/viper" + "github.com/kuoss/common/logger" _ "github.com/kuoss/lethe/storage/driver/filesystem" - "github.com/kuoss/lethe/util" - "github.com/spf13/viper" ) type Config struct { Version string - limit int - logDataPath string - retentionSize int - retentionTime time.Duration - retentionSizingStrategy string - timeout time.Duration - webListenAddress string + Query Query + Retention Retention + Storage Storage + Web Web +} + +type Query struct { + Limit int + Timeout time.Duration +} + +type Retention struct { + Size int64 + Time time.Duration + SizingStrategy string + RotationInterval time.Duration +} + +type Storage struct { + LogDataPath string +} + +type Web struct { + ListenAddress string + GinMode string } func New(version string) (*Config, error) { @@ -31,80 +54,91 @@ func New(version string) (*Config, error) { v.AddConfigPath(filepath.Join("..", "etc")) // Set default values - v.SetDefault("version", "test") - v.SetDefault("limit", 1000) - v.SetDefault("storage.log_data_path", "/var/data/log") - v.SetDefault("retention.sizingStrategy", "file") - v.SetDefault("web.listen_address", ":6060") - v.SetDefault("timeout", 20*time.Second) + v.SetDefault("query.limit", 1000) + v.SetDefault("query.timeout", "20s") + v.SetDefault("retention.size", 0) v.SetDefault("retention.time", "15d") - v.SetDefault("retention.size", "1000g") + v.SetDefault("retention.sizing_strategy", "file") + v.SetDefault("retention.rotation_interval", "20s") + v.SetDefault("storage.log_data_path", "data/log") + v.SetDefault("web.listen_address", ":6060") + v.SetDefault("web.gin_mode", gin.ReleaseMode) // "release" + // Read the configuration file, if it exists if err := v.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - logger.Warnf("Configuration lethe.yaml is not provided\n") - } else { - // Config file was found but another error was produced - return nil, fmt.Errorf("readInConfig err: %w", err) + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + _ = showFileContent(v.ConfigFileUsed()) + // Config file was found but another error occurred + return nil, fmt.Errorf("Config file has error: %w", err) } + logger.Warnf("Config file not found; continuing with defaults") + } else { + _ = showFileContent(v.ConfigFileUsed()) } - - retentionSize, err := util.StringToBytes(v.GetString("retention.size")) + // units + retentionSizeString := v.GetString("retention.size") + retentionSize, err := parseSize(retentionSizeString) if err != nil { - return nil, fmt.Errorf("stringToBytes err: %w", err) + return nil, fmt.Errorf("parse retention size err: %w, size: %s", err, retentionSizeString) } retentionTimeString := v.GetString("retention.time") - retentionTime, err := util.GetDurationFromAge(retentionTimeString) + retentionTime, err := model.ParseDuration(retentionTimeString) if err != nil { - return nil, fmt.Errorf("getDurationFromAge err: %w", err) + return nil, fmt.Errorf("parse retention time err: %w, time: %s", err, retentionTimeString) } return &Config{ - Version: version, - limit: 1000, - logDataPath: v.GetString("storage.log_data_path"), - retentionSize: retentionSize, // 1000g - retentionTime: retentionTime, // 15d - retentionSizingStrategy: v.GetString("retention.sizingStrategy"), - timeout: v.GetDuration("timeout"), - webListenAddress: v.GetString("web.listen_address"), + Version: version, + Query: Query{ + Limit: v.GetInt("query.limit"), + Timeout: v.GetDuration("query.timeout"), + }, + Retention: Retention{ + Size: int64(retentionSize), + Time: time.Duration(retentionTime), + SizingStrategy: v.GetString("retention.sizing_strategy"), + RotationInterval: v.GetDuration("retention.rotation_interval"), + }, + Storage: Storage{ + LogDataPath: v.GetString("storage.log_data_path"), + }, + Web: Web{ + ListenAddress: v.GetString("web.listen_address"), + GinMode: v.GetString("web.gin_mode"), + }, }, nil } -func (c *Config) Limit() int { - return c.limit -} - -func (c *Config) LogDataPath() string { - return c.logDataPath -} -func (c *Config) SetLogDataPath(logDataPath string) { - c.logDataPath = logDataPath -} - -func (c *Config) RetentionSize() int { - return c.retentionSize -} -func (c *Config) SetRetentionSize(retentionSize int) { - c.retentionSize = retentionSize -} - -func (c *Config) RetentionTime() time.Duration { - return c.retentionTime -} -func (c *Config) SetRetentionTime(retentionTime time.Duration) { - c.retentionTime = retentionTime +func showFileContent(filePath string) error { + logger.Infof("Read config file: %s", filePath) + b, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("read file err: %w", err) + } + fmt.Println(string(b)) + return nil } -func (c *Config) RetentionSizingStrategy() string { - return c.retentionSizingStrategy -} +func parseSize(sizeString string) (int64, error) { + // Handle legacy units and ensure the size string is uppercase for parsing + unitReplacements := map[string]string{ + "k": "KB", + "m": "MB", + "g": "GB", + } -func (c *Config) Timeout() time.Duration { - return c.timeout -} + // Replace legacy units + for oldUnit, newUnit := range unitReplacements { + if strings.HasSuffix(sizeString, oldUnit) { + sizeString = strings.ReplaceAll(sizeString, oldUnit, newUnit) + } + } -func (c *Config) WebListenAddress() string { - return c.webListenAddress + // Parse the size string using units package + size, err := units.ParseBase2Bytes(sizeString) + if err != nil { + return 0, fmt.Errorf("failed to parse size '%s': %w", sizeString, err) + } + return int64(size), nil } diff --git a/config/config_test.go b/config/config_test.go index 7700e1c..7ca062b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -5,150 +5,171 @@ import ( "time" "github.com/kuoss/common/tester" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestNew_default(t *testing.T) { +func TestNew(t *testing.T) { want := &Config{ - Version: "test", - limit: 1000, - logDataPath: "/var/data/log", - retentionSize: 1000 * 1024 * 1024 * 1024, - retentionTime: 15 * 24 * time.Hour, - retentionSizingStrategy: "file", - timeout: 20 * time.Second, - webListenAddress: ":6060", + Version: "test", + Query: Query{ + Limit: 1000, + Timeout: 20 * time.Second, + }, + Retention: Retention{ + Size: 0, + Time: 15 * 24 * time.Hour, + SizingStrategy: "file", + RotationInterval: 20 * time.Second, + }, + Storage: Storage{LogDataPath: "data/log"}, + Web: Web{ + ListenAddress: ":6060", + GinMode: "release", + }, } got, err := New("test") - assert.NoError(t, err) - assert.Equal(t, want, got) + require.NoError(t, err) + require.Equal(t, want, got) } func TestNew_example(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ + _, cleanup := tester.SetupDir(t, map[string]string{ "@/etc/lethe.example.yaml": "etc/lethe.yaml", }) defer cleanup() want := &Config{ - Version: "example", - limit: 1000, - logDataPath: "/data/log", - retentionSize: 100 * 1024 * 1024, - retentionTime: 24 * time.Hour, - retentionSizingStrategy: "file", - timeout: 20 * time.Second, - webListenAddress: ":6060", + Version: "example", + Query: Query{Limit: 1000, Timeout: 20 * time.Second}, + Retention: Retention{ + Size: 1000 * 1024 * 1024 * 1024, // 1000GB + Time: 15 * 24 * time.Hour, // 15d + SizingStrategy: "file", + RotationInterval: 20 * time.Second, + }, + Storage: Storage{LogDataPath: "/data/log"}, + Web: Web{ + ListenAddress: ":6060", + GinMode: "release", + }, } got, err := New("example") - assert.NoError(t, err) - assert.Equal(t, want, got) + require.NoError(t, err) + require.Equal(t, want, got) } func TestNew_ok1(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ + _, cleanup := tester.SetupDir(t, map[string]string{ "@/testdata/etc/lethe.ok1.yaml": "etc/lethe.yaml", }) defer cleanup() want := &Config{ - Version: "ok1", - limit: 1000, - logDataPath: "/data/log", - retentionSize: 200 * 1024 * 1024, - retentionTime: 24 * time.Hour, - retentionSizingStrategy: "file", - timeout: 20 * time.Second, - webListenAddress: ":6060", + Version: "ok1", + Query: Query{Limit: 1000, Timeout: 20 * time.Second}, + Retention: Retention{ + Size: 200 * 1024 * 1024 * 1024, // 200GB + Time: 24 * time.Hour, // 1d + SizingStrategy: "file", + RotationInterval: 20 * time.Second, + }, + Storage: Storage{LogDataPath: "/tmp/log"}, + Web: Web{ + ListenAddress: ":6060", + GinMode: "release", + }, } got, err := New("ok1") - assert.NoError(t, err) - assert.Equal(t, want, got) + require.NoError(t, err) + require.Equal(t, want, got) } func TestNew_ok2(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ + _, cleanup := tester.SetupDir(t, map[string]string{ "@/testdata/etc/lethe.ok2.yaml": "etc/lethe.yaml", }) defer cleanup() want := &Config{ - Version: "test2", - limit: 1000, - logDataPath: "/var/data/log", - retentionSize: 300 * 1024 * 1024, - retentionTime: 15 * 24 * time.Hour, // Only retentionTime feild is initialized with default value - retentionSizingStrategy: "file", - timeout: 20 * time.Second, - webListenAddress: ":6060", + Version: "test2", + Query: Query{Limit: 1000, Timeout: 20 * time.Second}, + Retention: Retention{ + Size: 300 * 1024 * 1024 * 1024, // 300GB + Time: 15 * 24 * time.Hour, // 15d + SizingStrategy: "file", + RotationInterval: 20 * time.Second, + }, + Storage: Storage{LogDataPath: "/tmp/log"}, + Web: Web{ + ListenAddress: ":6060", + GinMode: "release", + }, } got, err := New("test2") - assert.NoError(t, err) - assert.Equal(t, want, got) + require.NoError(t, err) + require.Equal(t, want, got) +} + +func TestNew_legacy(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/etc/lethe.legacy.yaml": "etc/lethe.yaml", + }) + defer cleanup() + + want := &Config{ + Version: "test2", + Query: Query{Limit: 1000, Timeout: 20 * time.Second}, + Retention: Retention{ + Size: 100 * 1024 * 1024 * 1024, // 100GB + Time: 15 * 24 * time.Hour, // 15d + SizingStrategy: "file", + RotationInterval: 20 * time.Second, + }, + Storage: Storage{LogDataPath: "/tmp/log"}, + Web: Web{ + ListenAddress: ":6060", + GinMode: "release", + }, + } + + got, err := New("test2") + require.NoError(t, err) + require.Equal(t, want, got) } func TestNew_error1(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ + _, cleanup := tester.SetupDir(t, map[string]string{ "@/testdata/etc/lethe.error1.yaml": "etc/lethe.yaml", }) defer cleanup() got, err := New("error") - assert.EqualError(t, err, `readInConfig err: While parsing config: yaml: did not find expected key`) - assert.Nil(t, got) + require.Error(t, err) + require.Nil(t, got) } func TestNew_error2(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ + _, cleanup := tester.SetupDir(t, map[string]string{ "@/testdata/etc/lethe.error2.yaml": "etc/lethe.yaml", }) defer cleanup() got, err := New("error") - assert.EqualError(t, err, "stringToBytes err: cannot accept unit '0' in '200''. allowed units: [k, m, g]") - assert.Nil(t, got) + require.Error(t, err) + require.Nil(t, got) } func TestNew_error3(t *testing.T) { - _, cleanup := tester.MustSetupDir(t, map[string]string{ + _, cleanup := tester.SetupDir(t, map[string]string{ "@/testdata/etc/lethe.error3.yaml": "etc/lethe.yaml", }) defer cleanup() got, err := New("error") - assert.EqualError(t, err, "getDurationFromAge err: not a valid duration string: \"1\"") - assert.Nil(t, got) -} - -func TestFunctions(t *testing.T) { - cfg, err := New("test") - assert.NoError(t, err) - - assert.Equal(t, "test", cfg.Version) - - assert.Equal(t, 1000, cfg.Limit()) - - assert.Equal(t, "/var/data/log", cfg.LogDataPath()) - cfg.SetLogDataPath("tmp/hello") - assert.Equal(t, "tmp/hello", cfg.LogDataPath()) - cfg.SetLogDataPath("/data/log") - - assert.Equal(t, 1000*1024*1024*1024, cfg.RetentionSize()) // 1000 GiB - cfg.SetRetentionSize(1 * 1024 * 1024) // 1 MiB - assert.Equal(t, 1*1024*1024, cfg.RetentionSize()) // 1 MiB - cfg.SetRetentionSize(100 * 1024 * 1024) // 100 MiB - - assert.Equal(t, 15*24*time.Hour, cfg.RetentionTime()) // 15 day - cfg.SetRetentionTime(1 * 24 * time.Hour) // 1 days - assert.Equal(t, 1*24*time.Hour, cfg.RetentionTime()) // 1 days - cfg.SetRetentionTime(15 * 24 * time.Hour) // 15 day - - assert.Equal(t, 20*time.Second, cfg.Timeout()) - - assert.Equal(t, "file", cfg.RetentionSizingStrategy()) - assert.Equal(t, ":6060", cfg.WebListenAddress()) + require.Error(t, err) + require.Nil(t, got) } diff --git a/etc/lethe.example.yaml b/etc/lethe.example.yaml index 147e1d7..1deb159 100644 --- a/etc/lethe.example.yaml +++ b/etc/lethe.example.yaml @@ -1,6 +1,6 @@ retention: - time: 1d - size: 100m + time: 15d + size: 1000GB storage: driver: filesystem log_data_path: /data/log diff --git a/go.mod b/go.mod index 430c916..b668b5d 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/kuoss/lethe go 1.20 require ( + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/gin-gonic/gin v1.9.1 github.com/google/fscrypt v0.3.4 - github.com/kuoss/common v0.1.6 + github.com/kuoss/common v0.1.7 github.com/prometheus/common v0.43.0 github.com/prometheus/prometheus v0.43.0 - github.com/spf13/afero v1.8.2 github.com/spf13/cast v1.4.1 github.com/spf13/viper v1.11.0 github.com/stretchr/testify v1.9.0 @@ -38,6 +38,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-isatty v0.0.19 // indirect @@ -53,6 +54,7 @@ require ( github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect + github.com/spf13/afero v1.8.2 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect diff --git a/go.sum b/go.sum index 0b791b6..df1379f 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aws/aws-sdk-go v1.44.245 h1:KtY2s4q31/kn33AdV63R5t77mdxsI7rq3YT7Mgo805M= github.com/aws/aws-sdk-go v1.44.245/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -61,6 +62,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -186,10 +188,11 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kuoss/common v0.1.6 h1:RDq25zD5mrup5ezDijR4xz93LpyHDklSqSu9nn+9ox0= -github.com/kuoss/common v0.1.6/go.mod h1:u/JgnK5aSk4hv1aqy4/JCCKz3PZ1rkB7AN27cnTd95M= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kuoss/common v0.1.7 h1:2ErvqIOMxp8IenXULaX7PiAiFHVFJwcBnr8rhtt5sUQ= +github.com/kuoss/common v0.1.7/go.mod h1:u/JgnK5aSk4hv1aqy4/JCCKz3PZ1rkB7AN27cnTd95M= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= diff --git a/letheql/engine.go b/letheql/engine.go index 920be4c..13431a0 100644 --- a/letheql/engine.go +++ b/letheql/engine.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kuoss/common/logger" + "github.com/kuoss/lethe/config" "github.com/kuoss/lethe/letheql/model" "github.com/kuoss/lethe/letheql/parser" "github.com/kuoss/lethe/storage/logservice" @@ -16,19 +17,20 @@ import ( ) type Engine struct { + cfg *config.Config logService *logservice.LogService } -func NewEngine(logService *logservice.LogService) *Engine { - return &Engine{logService} +func NewEngine(cfg *config.Config, logService *logservice.LogService) *Engine { + return &Engine{cfg, logService} } -func (ng *Engine) NewInstantQuery(_ context.Context, q storage.Queryable, qs string, ts time.Time) (Query, error) { +func (en *Engine) NewInstantQuery(_ context.Context, q storage.Queryable, qs string, ts time.Time) (Query, error) { expr, err := parser.ParseExpr(qs) if err != nil { return nil, err } - qry, err := ng.newQuery(q, expr, ts, ts, 0) + qry, err := en.newQuery(q, expr, ts, ts, 0) if err != nil { return nil, err } @@ -37,13 +39,13 @@ func (ng *Engine) NewInstantQuery(_ context.Context, q storage.Queryable, qs str return qry, nil } -func (ng *Engine) NewRangeQuery(_ context.Context, q storage.Queryable, qs string, start, end time.Time, interval time.Duration) (Query, error) { +func (en *Engine) NewRangeQuery(_ context.Context, q storage.Queryable, qs string, start, end time.Time, interval time.Duration) (Query, error) { logger.Infof("newRangeQuery qs: %s", qs) expr, err := parser.ParseExpr(qs) if err != nil { return nil, err } - qry, err := ng.newQuery(q, expr, start, end, interval) + qry, err := en.newQuery(q, expr, start, end, interval) if err != nil { return nil, err } @@ -52,7 +54,7 @@ func (ng *Engine) NewRangeQuery(_ context.Context, q storage.Queryable, qs strin return qry, nil } -func (ng *Engine) newQuery(q storage.Queryable, expr parser.Expr, start, end time.Time, interval time.Duration) (*query, error) { +func (en *Engine) newQuery(q storage.Queryable, expr parser.Expr, start, end time.Time, interval time.Duration) (*query, error) { es := &parser.EvalStmt{ Expr: expr, Start: start, @@ -61,27 +63,27 @@ func (ng *Engine) newQuery(q storage.Queryable, expr parser.Expr, start, end tim } qry := &query{ stmt: es, - ng: ng, + ng: en, queryable: q, } return qry, nil } -func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws model.Warnings, err error) { - ctx, cancel := context.WithTimeout(ctx, ng.logService.Config().Timeout()) +func (en *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws model.Warnings, err error) { + ctx, cancel := context.WithTimeout(ctx, en.cfg.Query.Timeout) q.cancel = cancel defer q.cancel() switch s := q.Statement().(type) { case *parser.EvalStmt: - return ng.execEvalStmt(ctx, q, s) + return en.execEvalStmt(ctx, q, s) case parser.TestStmt: return nil, nil, s(ctx) } panic(fmt.Errorf("letheql.exec: unhandled statement of type %T", q.Statement())) } -func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, model.Warnings, error) { - mint, maxt := ng.findMinMaxTime(s) +func (en *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.EvalStmt) (parser.Value, model.Warnings, error) { + mint, maxt := en.findMinMaxTime(s) querier, err := query.queryable.Querier(ctx, mint, maxt) if err != nil { return nil, nil, fmt.Errorf("querier err: %w", err) @@ -90,7 +92,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval // Range evaluation. evaluator := &evaluator{ - logService: ng.logService, + logService: en.logService, start: s.Start, end: s.End, startTimestamp: timeMilliseconds(s.Start), diff --git a/letheql/engine_test.go b/letheql/engine_test.go index 160bd1e..40bee00 100644 --- a/letheql/engine_test.go +++ b/letheql/engine_test.go @@ -2,74 +2,89 @@ package letheql import ( "context" - "fmt" "testing" "time" + "github.com/kuoss/common/tester" "github.com/kuoss/lethe/clock" + "github.com/kuoss/lethe/config" "github.com/kuoss/lethe/letheql/model" + "github.com/kuoss/lethe/storage/fileservice" + "github.com/kuoss/lethe/storage/logservice" "github.com/kuoss/lethe/storage/logservice/logmodel" "github.com/kuoss/lethe/storage/querier" "github.com/prometheus/prometheus/storage" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestNewEngine(t *testing.T) { - assert.NotNil(t, engine1) -} +func newEngine(t *testing.T) *Engine { + cfg, err := config.New("test") + require.NoError(t, err) -func TestNewInstantQuery(t *testing.T) { + fileService, err := fileservice.New(cfg) + require.NoError(t, err) - t.Run("QueryableFunc", func(t *testing.T) { - queryable := storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return nil, nil - }) - ctx, cancelCtx := context.WithCancel(context.Background()) - defer cancelCtx() + logService := logservice.New(cfg, fileService) + return NewEngine(cfg, logService) +} - qry, err := engine1.NewInstantQuery(ctx, queryable, "pod", time.Unix(1, 0)) - assert.NoError(t, err) - assert.NotEmpty(t, qry) +func TestNewInstantQuery_QueryableFunc(t *testing.T) { + queryable := storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return nil, nil }) - t.Run("LetheQueryable", func(t *testing.T) { - // LetheQueryable - queryable := &querier.LetheQueryable{LetheQuerier: &querier.LetheQuerier{}} - testCases := []struct { - qs string - wantError string - want *Result - }{ - { - `pod`, - "getTargets err: target matcher err: not found label 'namespace' for logType 'pod'", - &Result{}, - }, - { - `pod{namespace="namespace01"}`, - "", - &Result{Value: model.Log{Name: "pod", Lines: []model.LogLine{}}, Warnings: model.Warnings(nil)}, - }, - } - for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - qry, err := engine1.NewInstantQuery(context.TODO(), queryable, tc.qs, clock.Now()) - assert.NoError(t, err) - got := qry.Exec(context.TODO()) - err = got.Err - got.Err = nil - if tc.wantError == "" { - assert.NoError(t, err) - } else { - assert.EqualError(t, err, tc.wantError) - } - assert.Equal(t, tc.want, got) + en := newEngine(t) + qry, err := en.NewInstantQuery(context.TODO(), queryable, "pod", time.Unix(1, 0)) + require.NoError(t, err) + require.NotEmpty(t, qry) +} + +func TestNewInstantQuery_LetheQueryable(t *testing.T) { + queryable := &querier.LetheQueryable{LetheQuerier: &querier.LetheQuerier{}} + testCases := []struct { + qs string + wantError bool + want *Result + }{ + { + qs: `pod`, + wantError: true, + want: &Result{}, + }, + { + qs: `pod{namespace="namespace01"}`, + want: &Result{Value: model.Log{Name: "pod", Lines: []model.LogLine{}}, Warnings: model.Warnings(nil)}, + }, + } + for i, tc := range testCases { + t.Run(tester.CaseName(i, tc.qs), func(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", }) - } - }) + defer cleanup() + + en := newEngine(t) + qry, err := en.NewInstantQuery(context.TODO(), queryable, tc.qs, clock.Now()) + require.NoError(t, err) + got := qry.Exec(context.TODO()) + err = got.Err + got.Err = nil + if tc.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.want, got) + }) + } } func TestNewRangeQuery(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) ago10d := clock.Now().Add(-240 * time.Hour) ago2m := clock.Now().Add(-2 * time.Minute) @@ -80,18 +95,21 @@ func TestNewRangeQuery(t *testing.T) { qs string start time.Time end time.Time - wantError string + wantError bool want *Result }{ { - `pod`, ago10d, now, - "getTargets err: target matcher err: not found label 'namespace' for logType 'pod'", - &Result{}, + qs: `pod`, + start: ago10d, + end: now, + wantError: true, + want: &Result{}, }, { - `pod{namespace="namespace01"}`, ago2m, now, - "", - &Result{Err: error(nil), Value: model.Log{Name: "pod", Lines: []model.LogLine{ + qs: `pod{namespace="namespace01"}`, + start: ago2m, + end: now, + want: &Result{Err: error(nil), Value: model.Log{Name: "pod", Lines: []model.LogLine{ logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "lerom ipsum"}, logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "lerom from sidecar"}, @@ -99,9 +117,10 @@ func TestNewRangeQuery(t *testing.T) { logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}}}, Warnings: model.Warnings(nil)}, }, { - `pod{namespace="namespace01"}`, ago10d, now, - "", - &Result{Err: error(nil), Value: model.Log{Name: "pod", Lines: []model.LogLine{ + qs: `pod{namespace="namespace01"}`, + start: ago10d, + end: now, + want: &Result{Err: error(nil), Value: model.Log{Name: "pod", Lines: []model.LogLine{ logmodel.PodLog{Time: "2009-11-10T21:00:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T21:01:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T21:02:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, @@ -119,19 +138,25 @@ func TestNewRangeQuery(t *testing.T) { }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + t.Run(tester.CaseName(i), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + ctx := context.TODO() - qry, err := engine1.NewRangeQuery(ctx, queryable, tc.qs, tc.start, tc.end, 0) - assert.NoError(t, err) + engine := newEngine(t) + qry, err := engine.NewRangeQuery(ctx, queryable, tc.qs, tc.start, tc.end, 0) + require.NoError(t, err) got := qry.Exec(ctx) err = got.Err got.Err = nil - if tc.wantError == "" { - assert.NoError(t, err) + if tc.wantError { + require.Error(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.NoError(t, err) } - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } diff --git a/letheql/evaluator_test.go b/letheql/evaluator_test.go index 975783c..ac94e56 100644 --- a/letheql/evaluator_test.go +++ b/letheql/evaluator_test.go @@ -6,75 +6,112 @@ import ( "testing" "time" + "github.com/kuoss/common/tester" + "github.com/kuoss/lethe/clock" + "github.com/kuoss/lethe/config" "github.com/kuoss/lethe/letheql/model" "github.com/kuoss/lethe/letheql/parser" + "github.com/kuoss/lethe/storage/fileservice" "github.com/kuoss/lethe/storage/logservice" "github.com/kuoss/lethe/storage/logservice/logmodel" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func newLogService(t *testing.T) *logservice.LogService { + cfg, err := config.New("test") + require.NoError(t, err) + + fileService, err := fileservice.New(cfg) + require.NoError(t, err) + + return logservice.New(cfg, fileService) +} + func TestNewEvaluator(t *testing.T) { - logService1 := &logservice.LogService{} - contextBackground := context.Background() - contextTODO := context.TODO() + logService := newLogService(t) testCases := []struct { - logService *logservice.LogService - ctx context.Context startTimestamp int64 endTimestamp int64 interval int64 want evaluator }{ { - logService1, contextBackground, 0, 0, 0, - evaluator{logService1, contextBackground, 0, 0, 0, time.Time{}, time.Time{}}, + 0, 0, 0, + evaluator{ + logService, + context.TODO(), + 0, + 0, + 0, + time.Time{}, + time.Time{}, + }, }, { - logService1, contextTODO, 0, 0, 0, - evaluator{logService1, contextTODO, 0, 0, 0, time.Time{}, time.Time{}}, + 0, 0, 0, + evaluator{logService, context.TODO(), 0, 0, 0, time.Time{}, time.Time{}}, }, } for i, tc := range testCases { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got := evaluator{tc.logService, tc.ctx, tc.startTimestamp, tc.endTimestamp, tc.interval, time.Time{}, time.Time{}} - assert.Equal(t, tc.want, got) + got := evaluator{logService, context.TODO(), tc.startTimestamp, tc.endTimestamp, tc.interval, time.Time{}, time.Time{}} + require.Equal(t, tc.want, got) }) } - assert.NotNil(t, evaluator1) } func TestEval(t *testing.T) { testCases := []struct { input string - wantParseError string - wantError string + wantParseError bool + wantEvalError bool wantWarnings model.Warnings want parser.Value }{ + // error + { + input: ``, + wantParseError: true, + }, + { + input: `pod{namespace|="namespace01"}`, + wantParseError: true, + }, + { + input: `pod{namespace="namespace01",pod~="nginx-.*"}`, + wantParseError: true, + }, + { + input: `pod`, + wantEvalError: true, + }, + { + input: `node`, + wantEvalError: true, + }, { - ``, - "1:1: parse error: no expression found in input", "", nil, nil, + input: `node{namespace="namespace01"}`, + wantEvalError: true, }, { - `pod`, - "", "getTargets err: target matcher err: not found label 'namespace' for logType 'pod'", nil, nil, + input: `node{node="node01",process!="kubelet"} != "sidecar" |~ "*" `, + wantEvalError: true, }, { - `"hello"`, - "", "", nil, String{T: 0, V: "hello"}, + input: `"hello"`, + want: String{T: 0, V: "hello"}, }, { - `pod{namespace="hello"}`, - "", "", nil, model.Log{Name: "pod", Lines: []model.LogLine{}}, + input: `pod{namespace="hello"}`, + want: model.Log{Name: "pod", Lines: []model.LogLine{}}, }, { - `pod{namespace|="namespace01"}`, - "1:14: parse error: unexpected character inside braces: '|'", "", nil, nil, + input: `pod{namespace="namespace01",pod="nginx-.*"}`, + want: model.Log{Name: "pod", Lines: []model.LogLine{}}, }, { - `pod{namespace="namespace01"}`, - "", "", nil, - model.Log{Name: "pod", Lines: []model.LogLine{ + input: `pod{namespace="namespace01"}`, + want: model.Log{Name: "pod", Lines: []model.LogLine{ logmodel.PodLog{Time: "2009-11-10T21:00:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T21:01:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T21:02:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, @@ -91,10 +128,108 @@ func TestEval(t *testing.T) { logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}}}, }, { - `pod{namespace!="namespace01"}`, - "", "", - model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, - model.Log{Name: "pod", Lines: []model.LogLine{ + input: `pod{namespace="namespace01",pod=~"nginx-.*"}`, + want: model.Log{Name: "pod", Lines: []model.LogLine{ + logmodel.PodLog{Time: "2009-11-10T21:00:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, + logmodel.PodLog{Time: "2009-11-10T21:01:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, + logmodel.PodLog{Time: "2009-11-10T21:02:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, + logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "lerom ipsum"}, + logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "lerom from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}}}, + }, + { + input: `pod{namespace="namespace01",pod!~"nginx-.*"}`, + want: model.Log{Name: "pod", Lines: []model.LogLine{ + logmodel.PodLog{Time: "2009-11-10T22:56:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:56:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:56:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:57:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:57:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:57:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}}}, + }, + { + input: `pod{namespace="namespace01",pod=~"nginx-.*",container="sidecar"}`, + want: model.Log{Name: "pod", Lines: []model.LogLine{ + logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "lerom from sidecar"}, + logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}}}, + }, + { + input: `node{node="node01"}`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "lerom ipsum"}, + logmodel.NodeLog{Time: "2009-11-10T22:57:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:57:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:57:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "lerom from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + { + input: `node{node="node01",process!="kubelet"}`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "lerom ipsum"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "lerom from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + { + input: `node{node="node01",process!="kubelet"} |= "hello"`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + { + input: `node{node="node01",process!="kubelet"} |= "hello" |= "sidecar"`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, + logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}}}, + }, + { + input: `node{node="node01",process!="kubelet"} |= "hello" != "sidecar"`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + { + input: `node{node="node01",process!="kubelet"} |~ "ll.*" !~ "car.*"`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + { + input: `node{node="node01",process!="kubelet"} |~ "ll.*" !~ "car.*"`, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + { + input: `node{node="node01",process!="kubelet"} != "sidecar" |~ "d$" `, + want: model.Log{Name: "node", Lines: []model.LogLine{ + logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, + logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, + }, + // warnings + { + input: `node{node!~"node.*"}`, + wantWarnings: model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, + want: model.Log{Name: "node", Lines: []model.LogLine{}}, + }, + { + input: `pod{namespace!="namespace01"}`, + wantWarnings: model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, + want: model.Log{Name: "pod", Lines: []model.LogLine{ logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace02", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace02", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "lerom ipsum"}, logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace02", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, @@ -109,10 +244,9 @@ func TestEval(t *testing.T) { logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace02", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}}}, }, { - `pod{namespace=~"namespace.*"}`, - "", "", - model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, - model.Log{Name: "pod", Lines: []model.LogLine{ + input: `pod{namespace=~"namespace.*"}`, + wantWarnings: model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, + want: model.Log{Name: "pod", Lines: []model.LogLine{ logmodel.PodLog{Time: "2009-11-10T21:00:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T21:01:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, logmodel.PodLog{Time: "2009-11-10T21:02:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, @@ -141,76 +275,9 @@ func TestEval(t *testing.T) { logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace02", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}}}, }, { - `pod{namespace="namespace01",pod="nginx-.*"}`, - "", "", nil, - model.Log{Name: "pod", Lines: []model.LogLine{}}, - }, - { - `pod{namespace="namespace01",pod~="nginx-.*"}`, - "1:32: parse error: unexpected character inside braces: '~'", "", nil, nil, - }, - { - `pod{namespace="namespace01",pod=~"nginx-.*"}`, - "", "", nil, - model.Log{Name: "pod", Lines: []model.LogLine{ - logmodel.PodLog{Time: "2009-11-10T21:00:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, - logmodel.PodLog{Time: "2009-11-10T21:01:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, - logmodel.PodLog{Time: "2009-11-10T21:02:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}, - logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "lerom ipsum"}, - logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "lerom from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:59:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "nginx", Log: "hello world"}}}, - }, - { - `pod{namespace="namespace01",pod!~"nginx-.*"}`, - "", "", nil, - model.Log{Name: "pod", Lines: []model.LogLine{ - logmodel.PodLog{Time: "2009-11-10T22:56:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:56:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:56:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:57:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:57:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:57:00.000000Z", Namespace: "namespace01", Pod: "apache-75675f5897-7ci7o", Container: "httpd", Log: "hello from sidecar"}}}, - }, - { - `pod{namespace="namespace01",pod=~"nginx-.*",container="sidecar"}`, - "", "", nil, - model.Log{Name: "pod", Lines: []model.LogLine{ - logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "lerom from sidecar"}, - logmodel.PodLog{Time: "2009-11-10T22:58:00.000000Z", Namespace: "namespace01", Pod: "nginx-deployment-75675f5897-7ci7o", Container: "sidecar", Log: "hello from sidecar"}}}, - }, - { - `node`, - "", "getTargets err: target matcher err: not found label 'node' for logType 'node'", nil, nil, - }, - { - `node{namespace="namespace01"}`, - "", "getTargets err: target matcher err: not found label 'node' for logType 'node'", nil, nil, - }, - { - `node{node="node01"}`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "lerom ipsum"}, - logmodel.NodeLog{Time: "2009-11-10T22:57:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:57:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:57:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "lerom from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node=~"node.*"}`, - "", "", - model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, - model.Log{Name: "node", Lines: []model.LogLine{ + input: `node{node=~"node.*"}`, + wantWarnings: model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, + want: model.Log{Name: "node", Lines: []model.LogLine{ logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, @@ -237,10 +304,9 @@ func TestEval(t *testing.T) { logmodel.NodeLog{Time: "2009-11-10T21:58:00.000000Z", Node: "node02", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}}}, }, { - `node{node=~"node.*"}`, - "", "", - model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, - model.Log{Name: "node", Lines: []model.LogLine{ + input: `node{node=~"node.*"}`, + wantWarnings: model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, + want: model.Log{Name: "node", Lines: []model.LogLine{ logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, logmodel.NodeLog{Time: "2009-11-10T22:56:00.000000Z", Node: "node01", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, @@ -266,91 +332,47 @@ func TestEval(t *testing.T) { logmodel.NodeLog{Time: "2009-11-10T21:58:00.000000Z", Node: "node02", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}, logmodel.NodeLog{Time: "2009-11-10T21:58:00.000000Z", Node: "node02", Process: "kubelet", Log: "I0525 20:00:45.752587 17221 scope.go:110] \"RemoveContainer\" hello from sidecar"}}}, }, - { - `node{node!~"node.*"}`, - "", "", - model.Warnings{fmt.Errorf("warnMultiTargets: use operator '=' for selecting target")}, - model.Log{Name: "node", Lines: []model.LogLine{}}, - }, - { - `node{node="node01",process!="kubelet"}`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "lerom ipsum"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "lerom from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node="node01",process!="kubelet"} |= "hello"`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node="node01",process!="kubelet"} |= "hello" |= "sidecar"`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}, - logmodel.NodeLog{Time: "2009-11-10T22:58:00.000000Z", Node: "node01", Process: "dockerd", Log: "hello from sidecar"}}}, - }, - { - `node{node="node01",process!="kubelet"} |= "hello" != "sidecar"`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node="node01",process!="kubelet"} |~ "ll.*" !~ "car.*"`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node="node01",process!="kubelet"} |~ "ll.*" !~ "car.*"`, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node="node01",process!="kubelet"} != "sidecar" |~ "d$" `, - "", "", nil, - model.Log{Name: "node", Lines: []model.LogLine{ - logmodel.NodeLog{Time: "2009-11-10T22:59:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}, - logmodel.NodeLog{Time: "2009-11-10T23:00:00.000000Z", Node: "node01", Process: "containerd", Log: "hello world"}}}, - }, - { - `node{node="node01",process!="kubelet"} != "sidecar" |~ "*" `, - "", "getMatchFuncSet err: getLineMatchFuncs err: getLineMatchFunc err: error parsing regexp: missing argument to repetition operator: `*`", nil, nil, - }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + t.Run(tester.CaseName(i, tc.input), func(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + logService := newLogService(t) var got parser.Value var ws model.Warnings - var err error + var evalErr error expr, parseErr := parser.ParseExpr(tc.input) - if tc.wantParseError != "" { - assert.EqualError(t, parseErr, tc.wantParseError) + if tc.wantParseError { + require.Error(t, parseErr) } else { - assert.NoError(t, parseErr) - got, ws, err = evaluator1.Eval(expr) + require.NoError(t, parseErr) + + now := clock.Now() + ev := evaluator{ + logService: logService, + ctx: context.TODO(), + startTimestamp: 0, + endTimestamp: 0, + interval: 0, + start: now.Add(-4 * time.Hour), + end: now, + } + got, ws, evalErr = ev.Eval(expr) } - if tc.wantError != "" { - assert.EqualError(t, err, tc.wantError) + if tc.wantEvalError { + require.Error(t, evalErr) } else { - assert.NoError(t, err) + require.NoError(t, evalErr) } - assert.Equal(t, tc.want, got) - assert.Equal(t, tc.wantWarnings, ws) - }) + require.Equal(t, tc.want, got) + require.Equal(t, tc.wantWarnings, ws) + }, + ) } } diff --git a/letheql/init_test.go b/letheql/init_test.go deleted file mode 100644 index aa6d904..0000000 --- a/letheql/init_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package letheql - -import ( - "context" - "time" - - "github.com/kuoss/lethe/clock" - "github.com/kuoss/lethe/config" - _ "github.com/kuoss/lethe/storage/driver/filesystem" - "github.com/kuoss/lethe/storage/fileservice" - "github.com/kuoss/lethe/storage/logservice" - "github.com/kuoss/lethe/util/testutil" -) - -var ( - engine1 *Engine - evaluator1 *evaluator -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/init") - fileService, err := fileservice.New(cfg) - if err != nil { - panic(err) - } - logService := logservice.New(fileService) - engine1 = NewEngine(logService) - - now := clock.Now() - evaluator1 = &evaluator{ - logService: logService, - ctx: context.TODO(), - start: now.Add(-4 * time.Hour), - end: now, - startTimestamp: 0, - endTimestamp: 0, - interval: 0, - } -} diff --git a/rotator/rotator.go b/rotator/rotator.go index 4cf5aeb..54a08fd 100644 --- a/rotator/rotator.go +++ b/rotator/rotator.go @@ -9,23 +9,23 @@ import ( ) type Rotator struct { - config *config.Config + interval time.Duration fileService *fileservice.FileService } func New(cfg *config.Config, fileService *fileservice.FileService) *Rotator { - return &Rotator{cfg, fileService} + return &Rotator{cfg.Retention.RotationInterval, fileService} } -func (r *Rotator) Start(interval time.Duration) { - go r.routineLoop(interval) +func (r *Rotator) Start() { + go r.routineLoop() } -func (r *Rotator) routineLoop(interval time.Duration) { +func (r *Rotator) routineLoop() { for { r.RunOnce() - logger.Infof("routineLoop... sleep %s", interval) - time.Sleep(interval) + logger.Infof("routineLoop... sleep %s", r.interval) + time.Sleep(r.interval) } } diff --git a/rotator/rotator_test.go b/rotator/rotator_test.go index bd6e422..c13a9a8 100644 --- a/rotator/rotator_test.go +++ b/rotator/rotator_test.go @@ -1,103 +1,101 @@ package rotator import ( - "fmt" "testing" "time" + "github.com/kuoss/common/tester" "github.com/kuoss/lethe/clock" "github.com/kuoss/lethe/config" "github.com/kuoss/lethe/storage/fileservice" - "github.com/kuoss/lethe/util/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -var ( - rotator1 *Rotator -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/rotator_rotator_test") - fileService, err := fileservice.New(cfg) - if err != nil { - panic(err) - } - rotator1 = New(cfg, fileService) -} - func TestRunOnce(t *testing.T) { testCases := []struct { - retentionSize int + retentionSize int64 retentionTime time.Duration - want []fileservice.LogFile + want []string }{ { - 0, - 0, - []fileservice.LogFile{ - {Fullpath: "tmp/rotator_rotator_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/rotator_rotator_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/rotator_rotator_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/rotator_rotator_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2000-01-01_00.log", Subpath: "pod/namespace01/2000-01-01_00.log", LogType: "pod", Target: "namespace01", Name: "2000-01-01_00.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace02/0000-00-00_00.log", Subpath: "pod/namespace02/0000-00-00_00.log", LogType: "pod", Target: "namespace02", Name: "0000-00-00_00.log", Extension: ".log", Size: 12}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + retentionSize: 0, + retentionTime: 0, + want: []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2000-01-01_00.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/0000-00-00_00.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 10, // 10 bytes - 100 * time.Hour, // 100 hours - []fileservice.LogFile{}, + retentionSize: 10, // 10 bytes + retentionTime: 100 * time.Hour, // 100 hours + want: []string{}, }, { - 2 * 1024, // 2 KiB - 100 * time.Hour, // 100 hours - []fileservice.LogFile{ - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + retentionSize: 2 * 1024, // 2 KiB + retentionTime: 100 * time.Hour, // 100 hours + want: []string{"pod/namespace01/2029-11-10_23.log", "pod/namespace02/2009-11-10_22.log"}, }, { - 3 * 1024, // 3 KiB - 100 * time.Hour, // 100 hours - []fileservice.LogFile{ - {Fullpath: "tmp/rotator_rotator_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + retentionSize: 3 * 1024, // 3 KiB + retentionTime: 100 * time.Hour, // 100 hours + want: []string{ + "node/node01/2009-11-10_22.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 100 * 1024 * 1024 * 1024, // 100 GiB - 2 * 24 * time.Hour, // 2 days - []fileservice.LogFile{ - {Fullpath: "tmp/rotator_rotator_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/rotator_rotator_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/rotator_rotator_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/rotator_rotator_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + retentionSize: 100 * 1024 * 1024 * 1024, // 100 GiB + retentionTime: 2 * 24 * time.Hour, // 2 days + want: []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - rotator1.config.SetRetentionSize(tc.retentionSize) - rotator1.config.SetRetentionTime(tc.retentionTime) - rotator1.RunOnce() - got, err := rotator1.fileService.ListFiles() - assert.NoError(t, err) - assert.Equal(t, tc.want, got) - testutil.ResetLogData() + t.Run(tester.CaseName(i, tc.retentionSize, tc.retentionTime), func(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + cfg, err := config.New("test") + require.NoError(t, err) + cfg.Retention.Size = tc.retentionSize + cfg.Retention.Time = tc.retentionTime + + fileService, err := fileservice.New(cfg) + require.NoError(t, err) + rotator := New(cfg, fileService) + + rotator.RunOnce() + + list, err := rotator.fileService.ListFiles() + subpaths := []string{} + for _, item := range list { + subpaths = append(subpaths, item.Subpath) + } + require.NoError(t, err) + require.Equal(t, tc.want, subpaths) }) } } diff --git a/router/init_test.go b/router/init_test.go deleted file mode 100644 index a0353ab..0000000 --- a/router/init_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package router - -import ( - "net/http" - "net/http/httptest" - - "github.com/kuoss/lethe/clock" - "github.com/kuoss/lethe/config" - "github.com/kuoss/lethe/storage/fileservice" - "github.com/kuoss/lethe/storage/logservice" - "github.com/kuoss/lethe/storage/queryservice" - "github.com/kuoss/lethe/util/testutil" -) - -var ( - router1 *Router -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/init") - fileService, err := fileservice.New(cfg) - if err != nil { - panic(err) - } - logService := logservice.New(fileService) - queryService := queryservice.New(logService) - router1 = New(cfg, fileService, queryService) -} - -func testGET(url string) (code int, body string) { - w := httptest.NewRecorder() - req, err := http.NewRequest("GET", url, nil) - if err != nil { - panic(err) - } - router1.ginRouter.ServeHTTP(w, req) - return w.Code, w.Body.String() -} diff --git a/router/metadata_test.go b/router/metadata_test.go index 1b66e88..8e9702c 100644 --- a/router/metadata_test.go +++ b/router/metadata_test.go @@ -3,13 +3,15 @@ package router import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMetadata(t *testing.T) { - code, body := testGET("/api/v1/metadata") - assert.Equal(t, 200, code) - assert.JSONEq(t, `{ + code, body, cleanup := testGET(t, "/api/v1/metadata") + defer cleanup() + + require.Equal(t, 200, code) + require.JSONEq(t, `{ "data": { "targets": [ "node{node=\"node01\"}", diff --git a/router/probe_test.go b/router/probe_test.go index aa727fd..a920d26 100644 --- a/router/probe_test.go +++ b/router/probe_test.go @@ -3,17 +3,19 @@ package router import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestHealthy(t *testing.T) { - code, body := testGET("/-/healthy") - assert.Equal(t, 200, code) - assert.Equal(t, "Venti is Healthy.\n", body) + code, body, cleanup := testGET(t, "/-/healthy") + defer cleanup() + require.Equal(t, 200, code) + require.Equal(t, "Venti is Healthy.\n", body) } func TestReady(t *testing.T) { - code, body := testGET("/-/ready") - assert.Equal(t, 200, code) - assert.Equal(t, "Venti is Ready.\n", body) + code, body, cleanup := testGET(t, "/-/ready") + defer cleanup() + require.Equal(t, 200, code) + require.Equal(t, "Venti is Ready.\n", body) } diff --git a/router/query.go b/router/query.go index 33f8ae9..7f562dc 100644 --- a/router/query.go +++ b/router/query.go @@ -1,11 +1,13 @@ package router import ( + "math" "net/http" + "time" "github.com/gin-gonic/gin" "github.com/kuoss/lethe/letheql/model" - "github.com/kuoss/lethe/util" + "github.com/spf13/cast" ) func (r *Router) Query(c *gin.Context) { @@ -17,7 +19,7 @@ func (r *Router) QueryRange(c *gin.Context) { qs := c.Query("query") start := c.Query("start") end := c.Query("end") - r.query(c, qs, model.TimeRange{Start: util.FloatStringToTime(start), End: util.FloatStringToTime(end)}) + r.query(c, qs, model.TimeRange{Start: floatStringToTime(start), End: floatStringToTime(end)}) } func (r *Router) query(c *gin.Context, qs string, tr model.TimeRange) { @@ -69,3 +71,8 @@ func (r *Router) query(c *gin.Context, qs string, tr model.TimeRange) { } c.JSON(http.StatusOK, obj) } + +func floatStringToTime(timeFloat string) time.Time { + sec, dec := math.Modf(cast.ToFloat64(timeFloat)) + return time.Unix(int64(sec), int64(dec*(1e9))) +} diff --git a/router/query_test.go b/router/query_test.go index 012a1d1..0a48f15 100644 --- a/router/query_test.go +++ b/router/query_test.go @@ -6,8 +6,9 @@ import ( "testing" "time" + "github.com/kuoss/common/tester" "github.com/kuoss/lethe/clock" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestQuery(t *testing.T) { @@ -42,17 +43,24 @@ func TestQuery(t *testing.T) { }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + t.Run(tester.CaseName(i, tc.qs), func(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + v := url.Values{} v.Add("query", tc.qs) - code, body := testGET("/api/v1/query?" + v.Encode()) - assert.Equal(t, tc.wantCode, code) - assert.Equal(t, tc.wantBody, body) + code, body, cleanup := testGET(t, "/api/v1/query?"+v.Encode()) + defer cleanup() + require.Equal(t, tc.wantCode, code) + require.Equal(t, tc.wantBody, body) }) } } func TestQueryRange(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + now := clock.Now() ago10d := now.Add(-240 * time.Hour) testCases := []struct { @@ -88,14 +96,16 @@ func TestQueryRange(t *testing.T) { }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + t.Run(tester.CaseName(i), func(t *testing.T) { + v := url.Values{} v.Add("query", tc.qs) v.Add("start", time2string(tc.start)) v.Add("end", time2string(tc.end)) - code, body := testGET("/api/v1/query_range?" + v.Encode()) - assert.Equal(t, tc.wantCode, code) - assert.Equal(t, tc.wantBody, body) + code, body, cleanup := testGET(t, "/api/v1/query_range?"+v.Encode()) + defer cleanup() + require.Equal(t, tc.wantCode, code) + require.Equal(t, tc.wantBody, body) }) } } @@ -103,3 +113,9 @@ func TestQueryRange(t *testing.T) { func time2string(t time.Time) string { return fmt.Sprintf("%d", t.Unix()) } + +func Test_FloatStringToTime(t *testing.T) { + want := time.Date(2015, time.July, 1, 20, 10, 51, 780999898, time.UTC) + got := floatStringToTime("1435781451.781") + require.True(t, want.Equal(got)) +} diff --git a/router/router.go b/router/router.go index 6f93817..766f6ac 100644 --- a/router/router.go +++ b/router/router.go @@ -25,6 +25,7 @@ func New(cfg *config.Config, fileService *fileservice.FileService, queryService } func (r *Router) setupRouter() { + // gin.SetMode(r.config.Web.GinMode) ginRouter := gin.Default() ginRouter.GET("/-/healthy", r.Healthy) @@ -39,5 +40,5 @@ func (r *Router) setupRouter() { } func (r *Router) Run() error { - return r.ginRouter.Run(r.config.WebListenAddress()) + return r.ginRouter.Run(r.config.Web.ListenAddress) } diff --git a/router/router_test.go b/router/router_test.go index 4aa79d3..d8402b9 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -1,11 +1,43 @@ package router import ( + "net/http" + "net/http/httptest" "testing" - "github.com/stretchr/testify/assert" + "github.com/kuoss/common/tester" + "github.com/kuoss/lethe/config" + "github.com/kuoss/lethe/storage/fileservice" + "github.com/kuoss/lethe/storage/logservice" + "github.com/kuoss/lethe/storage/queryservice" + "github.com/stretchr/testify/require" ) +func newRouter(t *testing.T) (*Router, func()) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := fileservice.New(cfg) + require.NoError(t, err) + logService := logservice.New(cfg, fileService) + queryService := queryservice.New(cfg, logService) + return New(cfg, fileService, queryService), cleanup +} + +func testGET(t *testing.T, url string) (code int, body string, cleanup func()) { + r, cleanup := newRouter(t) + w := httptest.NewRecorder() + req, err := http.NewRequest("GET", url, nil) + require.NoError(t, err) + r.ginRouter.ServeHTTP(w, req) + return w.Code, w.Body.String(), cleanup +} + func TestNew(t *testing.T) { - assert.NotEmpty(t, router1) + want := &Router{} + got, cleanup := newRouter(t) + defer cleanup() + require.NotEqual(t, want, got) } diff --git a/router/targets_test.go b/router/targets_test.go index 7a58786..241c0cd 100644 --- a/router/targets_test.go +++ b/router/targets_test.go @@ -3,13 +3,14 @@ package router import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestTarget(t *testing.T) { - code, body := testGET("/api/v1/targets") - assert.Equal(t, 200, code) - assert.JSONEq(t, `{"data":{"activeTargets":[ + code, body, cleanup := testGET(t, "/api/v1/targets") + defer cleanup() + require.Equal(t, 200, code) + require.JSONEq(t, `{"data":{"activeTargets":[ {"discoveredLabels":{"__meta_kubernetes_node_name":"node01","job":"node"},"health":"up","lastScrape":"2009-11-10T23:00:00Z"}, {"discoveredLabels":{"__meta_kubernetes_node_name":"node02","job":"node"},"health":"up","lastScrape":"2009-11-10T21:58:00Z"}, {"discoveredLabels":{"__meta_kubernetes_namespace":"namespace01","job":"pod"},"health":"up","lastScrape":"2009-11-10T23:00:00Z"}, diff --git a/storage/driver/filesystem/driver_test.go b/storage/driver/filesystem/driver_test.go index 050eae1..1c33748 100644 --- a/storage/driver/filesystem/driver_test.go +++ b/storage/driver/filesystem/driver_test.go @@ -5,22 +5,12 @@ import ( "sort" "testing" - storagedriver "github.com/kuoss/lethe/storage/driver" - "github.com/kuoss/lethe/util/testutil" - "github.com/stretchr/testify/assert" -) + // storagedriver "github.com/kuoss/lethe/storage/driver" -var ( - driver1 storagedriver.Driver - logDataPath_driver = "tmp/storage_driver_filesystem_driver_test" + "github.com/kuoss/common/tester" + "github.com/stretchr/testify/require" ) -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - driver1 = New(Params{RootDirectory: logDataPath_driver}) -} - func TestNew(t *testing.T) { testCases := []struct { params Params @@ -42,109 +32,105 @@ func TestNew(t *testing.T) { for i, tc := range testCases { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { got := New(tc.params) - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } } -func TestRootDirectory(t *testing.T) { - got := driver1.RootDirectory() - assert.Equal(t, logDataPath_driver, got) -} - -func TestName(t *testing.T) { - got := driver1.Name() - assert.Equal(t, "filesystem", got) -} - func TestGetContent(t *testing.T) { testCases := []struct { path string want string - wantError string + wantError bool }{ { - "", - "", - "read tmp/storage_driver_filesystem_driver_test: is a directory", + path: "", + want: "", + wantError: true, }, { - "node", - "", - "read tmp/storage_driver_filesystem_driver_test/node: is a directory", + path: "node", + want: "", + wantError: true, }, { - "pod/namespace01/2009-11-10_21.log", - "2009-11-10T21:00:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:01:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:02:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n", - "", + path: "pod/namespace01/2009-11-10_21.log", + want: "2009-11-10T21:00:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:01:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:02:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n", }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got, err := driver1.GetContent(tc.path) - if tc.wantError == "" { - assert.NoError(t, err) + t.Run(tester.CaseName(i), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + got, err := d.GetContent(tc.path) + if tc.wantError { + require.Error(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.NoError(t, err) } - assert.Equal(t, tc.want, string(got)) + require.Equal(t, tc.want, string(got)) }) } } func TestPutContent(t *testing.T) { testCases := []struct { - path string - content string - wantError string - wantError2 string - want string + path string + content string + wantError bool + want string }{ { - "", "", - "open tmp/storage_driver_filesystem_driver_test: is a directory", - "", - "", + path: "", + content: "", + wantError: true, + want: "", }, { - "node", "", - "open tmp/storage_driver_filesystem_driver_test/node: is a directory", - "", - "", + path: "node", + content: "", + wantError: true, + want: "", }, { - "pod/namespace01/test1.log", "hello", - "", - "", - "hello", + path: "pod/namespace01/test1.log", + content: "hello", + want: "hello", }, { - "pod/namespace01/2009-11-10_21.log", "hello", - "", - "", - "hello11-10T21:00:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:01:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:02:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n", + path: "pod/namespace01/2009-11-10_21.log", + content: "hello", + want: "hello11-10T21:00:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:01:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n2009-11-10T21:02:00.000000Z[namespace01|nginx-deployment-75675f5897-7ci7o|nginx] hello world\n", }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - err := driver1.PutContent(tc.path, ([]byte)(tc.content)) - if tc.wantError == "" { - assert.NoError(t, err) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + err := d.PutContent(tc.path, ([]byte)(tc.content)) + if tc.wantError { + require.Error(t, err) } else { - assert.EqualError(t, err, tc.wantError) - return + require.NoError(t, err) } - got, err := driver1.GetContent(tc.path) - if tc.wantError2 == "" { - assert.NoError(t, err) + + got, err := d.GetContent(tc.path) + if tc.wantError { + require.Error(t, err) } else { - assert.EqualError(t, err, tc.wantError2) - return + require.NoError(t, err) } - assert.Equal(t, tc.want, string(got)) + require.Equal(t, tc.want, string(got)) }) } - testutil.ResetLogData() } func TestReader(t *testing.T) { @@ -166,14 +152,20 @@ func TestReader(t *testing.T) { }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got, err := driver1.Reader(tc.path) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + got, err := d.Reader(tc.path) if tc.wantError == "" { - assert.NoError(t, err) - assert.NotEmpty(t, got) + require.NoError(t, err) + require.NotEmpty(t, got) } else { - assert.EqualError(t, err, tc.wantError) - assert.Nil(t, got) + require.EqualError(t, err, tc.wantError) + require.Nil(t, got) } }) } @@ -182,38 +174,42 @@ func TestReader(t *testing.T) { func TestWriter(t *testing.T) { testCases := []struct { path string - wantError string + wantError bool + want string }{ { - "", - "open tmp/storage_driver_filesystem_driver_test: is a directory", + path: "", + wantError: true, }, { - "hello", - "", + path: "node", + wantError: true, }, { - "node", - "open tmp/storage_driver_filesystem_driver_test/node: is a directory", + path: "hello", }, { - "pod/namespace01/2009-11-10_21.log", - "", + path: "pod/namespace01/2009-11-10_21.log", }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got, err := driver1.Writer(tc.path) - if tc.wantError == "" { - assert.NoError(t, err) - assert.NotEmpty(t, got) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + got, err := d.Writer(tc.path) + if tc.wantError { + require.Error(t, err) + require.Nil(t, got) } else { - assert.EqualError(t, err, tc.wantError) - assert.Nil(t, got) + require.NoError(t, err) + require.NotEmpty(t, got) } }) } - testutil.ResetLogData() } func TestStat(t *testing.T) { @@ -224,50 +220,56 @@ func TestStat(t *testing.T) { }{ { "", - "tmp/storage_driver_filesystem_driver_test", + "data/log", "", }, { "hello", "", - "Path not found: hello, err: stat err: stat tmp/storage_driver_filesystem_driver_test/hello: no such file or directory", + "Path not found: hello, err: stat err: stat data/log/hello: no such file or directory", }, { "node", - "tmp/storage_driver_filesystem_driver_test/node", + "data/log/node", "", }, { "pod", - "tmp/storage_driver_filesystem_driver_test/pod", + "data/log/pod", "", }, { "pod/namespace01", - "tmp/storage_driver_filesystem_driver_test/pod/namespace01", + "data/log/pod/namespace01", "", }, { "pod/namespace01/hello.log", "", - "Path not found: pod/namespace01/hello.log, err: stat err: stat tmp/storage_driver_filesystem_driver_test/pod/namespace01/hello.log: no such file or directory", + "Path not found: pod/namespace01/hello.log, err: stat err: stat data/log/pod/namespace01/hello.log: no such file or directory", }, { "pod/namespace01/2009-11-10_21.log", - "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log", + "data/log/pod/namespace01/2009-11-10_21.log", "", }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - fi, err := driver1.Stat(tc.path) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + fi, err := d.Stat(tc.path) if tc.wantError != "" { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) return } - assert.NoError(t, err) + require.NoError(t, err) got := fi.Fullpath() - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } } @@ -291,7 +293,7 @@ func TestList(t *testing.T) { { "hello", nil, - "Path not found: hello, err: open err: open tmp/storage_driver_filesystem_driver_test/hello: no such file or directory", + "Path not found: hello, err: open err: open data/log/hello: no such file or directory", }, { "pod", @@ -305,15 +307,21 @@ func TestList(t *testing.T) { }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got, err := driver1.List(tc.path) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + got, err := d.List(tc.path) if tc.wantError == "" { - assert.NoError(t, err) + require.NoError(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } sort.Strings(got) - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } } @@ -326,44 +334,45 @@ func TestMove(t *testing.T) { }{ { "", "", - "rename tmp/storage_driver_filesystem_driver_test tmp/storage_driver_filesystem_driver_test: file exists", + "rename data/log data/log: file exists", }, { "hello", "", - "Path not found: hello, err: stat err: stat tmp/storage_driver_filesystem_driver_test/hello: no such file or directory", + "Path not found: hello, err: stat err: stat data/log/hello: no such file or directory", }, { "", "hello", - "rename tmp/storage_driver_filesystem_driver_test tmp/storage_driver_filesystem_driver_test/hello: invalid argument", + "rename data/log data/log/hello: invalid argument", }, { "hello", "hello", - "Path not found: hello, err: stat err: stat tmp/storage_driver_filesystem_driver_test/hello: no such file or directory", + "Path not found: hello, err: stat err: stat data/log/hello: no such file or directory", }, { "pod/namespace01/hello.log", "pod/namespace01/hello.log", - "Path not found: pod/namespace01/hello.log, err: stat err: stat tmp/storage_driver_filesystem_driver_test/pod/namespace01/hello.log: no such file or directory", + "Path not found: pod/namespace01/hello.log, err: stat err: stat data/log/pod/namespace01/hello.log: no such file or directory", }, { "pod/namespace01/2009-11-10_21.log", "pod/namespace01/2009-11-10_00.log", // move "", }, - { - "pod/namespace01/2009-11-10_21.log", "pod/namespace01/2009-11-10_00.log", // duplicate - "Path not found: pod/namespace01/2009-11-10_21.log, err: stat err: stat tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log: no such file or directory", - }, } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - err := driver1.Move(tc.a, tc.b) + for i, tc := range testCases { + t.Run(tester.CaseName(i, tc.a, tc.b), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + err := d.Move(tc.a, tc.b) if tc.wantError == "" { - assert.NoError(t, err) + require.NoError(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } }) } - testutil.ResetLogData() } func TestDelete(t *testing.T) { @@ -387,22 +396,23 @@ func TestDelete(t *testing.T) { "pod/namespace01/2009-11-10_21.log", // delete "", }, - { - "pod/namespace01/2009-11-10_21.log", // duplicate - "Path not found: pod/namespace01/2009-11-10_21.log, err: stat err: stat tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log: no such file or directory", - }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - err := driver1.Delete(tc.path) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + err := d.Delete(tc.path) if tc.wantError == "" { - assert.NoError(t, err) + require.NoError(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } }) } - testutil.ResetLogData() } func TestWalk(t *testing.T) { @@ -415,52 +425,58 @@ func TestWalk(t *testing.T) { { "hello", nil, - "walk err: walkFunc err: lstat tmp/storage_driver_filesystem_driver_test/hello: no such file or directory", + "walk err: walkFunc err: lstat data/log/hello: no such file or directory", }, { - "tmp/storage_driver_filesystem_driver_test", + "data/log", nil, - "walk err: walkFunc err: lstat tmp/storage_driver_filesystem_driver_test/tmp/storage_driver_filesystem_driver_test: no such file or directory", + "walk err: walkFunc err: lstat data/log/data/log: no such file or directory", }, // ok { "", - []string{"tmp/storage_driver_filesystem_driver_test/node/node01/2009-11-10_21.log", "tmp/storage_driver_filesystem_driver_test/node/node01/2009-11-10_22.log", "tmp/storage_driver_filesystem_driver_test/node/node02/2009-11-01_00.log", "tmp/storage_driver_filesystem_driver_test/node/node02/2009-11-10_21.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2000-01-01_00.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_22.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2029-11-10_23.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace02/0000-00-00_00.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace02/2009-11-10_22.log"}, + []string{"data/log/node/node01/2009-11-10_21.log", "data/log/node/node01/2009-11-10_22.log", "data/log/node/node02/2009-11-01_00.log", "data/log/node/node02/2009-11-10_21.log", "data/log/pod/namespace01/2000-01-01_00.log", "data/log/pod/namespace01/2009-11-10_21.log", "data/log/pod/namespace01/2009-11-10_22.log", "data/log/pod/namespace01/2029-11-10_23.log", "data/log/pod/namespace02/0000-00-00_00.log", "data/log/pod/namespace02/2009-11-10_22.log"}, "", }, { "node", - []string{"tmp/storage_driver_filesystem_driver_test/node/node01/2009-11-10_21.log", "tmp/storage_driver_filesystem_driver_test/node/node01/2009-11-10_22.log", "tmp/storage_driver_filesystem_driver_test/node/node02/2009-11-01_00.log", "tmp/storage_driver_filesystem_driver_test/node/node02/2009-11-10_21.log"}, + []string{"data/log/node/node01/2009-11-10_21.log", "data/log/node/node01/2009-11-10_22.log", "data/log/node/node02/2009-11-01_00.log", "data/log/node/node02/2009-11-10_21.log"}, "", }, { "pod", - []string{"tmp/storage_driver_filesystem_driver_test/pod/namespace01/2000-01-01_00.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_22.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2029-11-10_23.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace02/0000-00-00_00.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace02/2009-11-10_22.log"}, + []string{"data/log/pod/namespace01/2000-01-01_00.log", "data/log/pod/namespace01/2009-11-10_21.log", "data/log/pod/namespace01/2009-11-10_22.log", "data/log/pod/namespace01/2029-11-10_23.log", "data/log/pod/namespace02/0000-00-00_00.log", "data/log/pod/namespace02/2009-11-10_22.log"}, "", }, { "pod/namespace01", - []string{"tmp/storage_driver_filesystem_driver_test/pod/namespace01/2000-01-01_00.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_22.log", "tmp/storage_driver_filesystem_driver_test/pod/namespace01/2029-11-10_23.log"}, + []string{"data/log/pod/namespace01/2000-01-01_00.log", "data/log/pod/namespace01/2009-11-10_21.log", "data/log/pod/namespace01/2009-11-10_22.log", "data/log/pod/namespace01/2029-11-10_23.log"}, "", }, { "pod/namespace01/2009-11-10_21.log", - []string{"tmp/storage_driver_filesystem_driver_test/pod/namespace01/2009-11-10_21.log"}, + []string{"data/log/pod/namespace01/2009-11-10_21.log"}, "", }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - infos, err := driver1.Walk(tc.path) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + infos, err := d.Walk(tc.path) if tc.wantError == "" { - assert.NoError(t, err) + require.NoError(t, err) paths := []string{} for _, info := range infos { paths = append(paths, info.Fullpath()) } - assert.Equal(t, tc.want, paths) + require.Equal(t, tc.want, paths) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } }) } @@ -476,12 +492,12 @@ func TestWalkDir(t *testing.T) { { "hello", nil, - "walkdir err: walkDirFunc err: lstat tmp/storage_driver_filesystem_driver_test/hello: no such file or directory", + "walkdir err: walkDirFunc err: lstat data/log/hello: no such file or directory", }, { - "tmp/storage_driver_filesystem_driver_test", + "data/log", nil, - "walkdir err: walkDirFunc err: lstat tmp/storage_driver_filesystem_driver_test/tmp/storage_driver_filesystem_driver_test: no such file or directory", + "walkdir err: walkDirFunc err: lstat data/log/data/log: no such file or directory", }, // ok { @@ -511,13 +527,19 @@ func TestWalkDir(t *testing.T) { }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got, err := driver1.WalkDir(tc.path) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + d := New(Params{RootDirectory: "data/log"}) + got, err := d.WalkDir(tc.path) if tc.wantError == "" { - assert.NoError(t, err) - assert.Equal(t, tc.want, got) + require.NoError(t, err) + require.Equal(t, tc.want, got) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } }) } diff --git a/storage/driver/filesystem/fileinfo_test.go b/storage/driver/filesystem/fileinfo_test.go index 697ff9e..0c0bff5 100644 --- a/storage/driver/filesystem/fileinfo_test.go +++ b/storage/driver/filesystem/fileinfo_test.go @@ -1,16 +1,16 @@ package filesystem import ( - "fmt" "os" "testing" - "github.com/stretchr/testify/assert" + "github.com/kuoss/common/tester" + "github.com/stretchr/testify/require" ) func TestFileInfo(t *testing.T) { testCases := []struct { - fullpath string + path string wantError string wantFullpath string wantSize int64 @@ -26,31 +26,36 @@ func TestFileInfo(t *testing.T) { "", 0, 2023, false, }, { - "tmp/init/pod", "", - "tmp/init/pod", 0, 2023, true, + "data/log/pod", "", + "data/log/pod", 0, 2023, true, }, { - "tmp/init/pod/namespace01", "", - "tmp/init/pod/namespace01", 0, 2023, true, + "data/log/pod/namespace01", "", + "data/log/pod/namespace01", 0, 2023, true, }, { - "tmp/init/pod/namespace01/2009-11-10_21.log", "", - "tmp/init/pod/namespace01/2009-11-10_21.log", 279, 2023, false, + "data/log/pod/namespace01/2009-11-10_21.log", "", + "data/log/pod/namespace01/2009-11-10_21.log", 279, 2023, false, }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - fi, err := os.Stat(tc.fullpath) + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + fi, err := os.Stat(tc.path) if tc.wantError != "" { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } else { - assert.NoError(t, err) + require.NoError(t, err) - got := FileInfo{fi, tc.fullpath} - assert.Equal(t, tc.wantFullpath, got.Fullpath()) - assert.Equal(t, tc.wantSize, got.Size()) - assert.GreaterOrEqual(t, got.ModTime().Year(), tc.wantModTimeYear) - assert.Equal(t, tc.wantIsDir, got.IsDir()) + got := FileInfo{fi, tc.path} + require.Equal(t, tc.wantFullpath, got.Fullpath()) + require.Equal(t, tc.wantSize, got.Size()) + require.GreaterOrEqual(t, got.ModTime().Year(), tc.wantModTimeYear) + require.Equal(t, tc.wantIsDir, got.IsDir()) } }) } diff --git a/storage/driver/filesystem/filewriter_test.go b/storage/driver/filesystem/filewriter_test.go index 8983324..f36b76f 100644 --- a/storage/driver/filesystem/filewriter_test.go +++ b/storage/driver/filesystem/filewriter_test.go @@ -5,27 +5,19 @@ import ( "os" "testing" + "github.com/kuoss/common/tester" storagedriver "github.com/kuoss/lethe/storage/driver" - "github.com/kuoss/lethe/util/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func init() { - dir := "tmp/storage_driver_filesystem_filewriter_test" - testutil.ChdirRoot() - err := os.RemoveAll(dir) - if err != nil { - panic(err) - } - err = os.Mkdir(dir, 0755) - if err != nil { - panic(err) - } -} - func TestFileWriter(t *testing.T) { - f, err := os.Create("tmp/storage_driver_filesystem_filewriter_test/greet.txt") - assert.NoError(t, err) + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + f, err := os.Create("data/log/greet.txt") + require.NoError(t, err) bw := bufio.NewWriter(f) var fw storagedriver.FileWriter = &fileWriter{ @@ -38,11 +30,11 @@ func TestFileWriter(t *testing.T) { } s := "hello" n, err := fw.Write([]byte(s)) - assert.NoError(t, err) - assert.Equal(t, len(s), n) + require.NoError(t, err) + require.Equal(t, len(s), n) fw.Close() - content, err := os.ReadFile("tmp/storage_driver_filesystem_filewriter_test/greet.txt") - assert.NoError(t, err) - assert.Equal(t, "hello", string(content)) + content, err := os.ReadFile("data/log/greet.txt") + require.NoError(t, err) + require.Equal(t, "hello", string(content)) } diff --git a/storage/driver/filesystem/init_test.go b/storage/driver/filesystem/init_test.go deleted file mode 100644 index 02f8e8e..0000000 --- a/storage/driver/filesystem/init_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package filesystem - -import ( - "github.com/kuoss/lethe/util/testutil" -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() -} diff --git a/storage/fileservice/clean_test.go b/storage/fileservice/clean_test.go index ba1c5fb..e9cf57e 100644 --- a/storage/fileservice/clean_test.go +++ b/storage/fileservice/clean_test.go @@ -4,42 +4,31 @@ import ( "os" "testing" + "github.com/kuoss/common/tester" "github.com/kuoss/lethe/config" - "github.com/kuoss/lethe/util/testutil" "github.com/stretchr/testify/require" ) -var ( - fileService_clean *FileService - logDataPath_clean string = "tmp/storage_fileservice_clean_test" -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() +func TestClean(t *testing.T) { + tmpDir, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath(logDataPath_clean) - fileService_clean, err = New(cfg) - if err != nil { - panic(err) - } -} + require.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) -func TestClean(t *testing.T) { - // setup - var err error - err = os.WriteFile("tmp/storage_fileservice_clean_test/kube.1", []byte("hello"), 0644) + // // setup + err = os.WriteFile(tmpDir+"/data/log/kube.1", []byte("hello"), 0644) require.NoError(t, err) - err = os.WriteFile("tmp/storage_fileservice_clean_test/host.1", []byte("hello"), 0644) + err = os.WriteFile(tmpDir+"/data/log/host.1", []byte("hello"), 0644) require.NoError(t, err) - require.FileExists(t, "tmp/storage_fileservice_clean_test/kube.1") - require.FileExists(t, "tmp/storage_fileservice_clean_test/host.1") - fileService_clean.Clean() - require.NoFileExists(t, "tmp/storage_fileservice_clean_test/kube.1") - require.NoFileExists(t, "tmp/storage_fileservice_clean_test/host.1") + require.FileExists(t, tmpDir+"/data/log/kube.1") + require.FileExists(t, tmpDir+"/data/log/host.1") + fileService.Clean() + require.NoFileExists(t, tmpDir+"/data/log/kube.1") + require.NoFileExists(t, tmpDir+"/data/log/host.1") } diff --git a/storage/fileservice/delete.go b/storage/fileservice/delete.go index b06e9b9..7d65441 100644 --- a/storage/fileservice/delete.go +++ b/storage/fileservice/delete.go @@ -11,7 +11,7 @@ import ( ) func (s *FileService) DeleteByAge() error { - retentionTime := s.config.RetentionTime() + retentionTime := s.Config.Retention.Time if retentionTime == 0 { logger.Infof("retentionTime is 0 (DeleteByAge skipped)") return nil @@ -44,7 +44,7 @@ func (s *FileService) DeleteByAge() error { } func (s *FileService) DeleteBySize() error { - retentionSize := s.config.RetentionSize() + retentionSize := s.Config.Retention.Size if retentionSize == 0 { logger.Infof("retentionSize is 0 (DeleteBySize skipped)") return nil @@ -60,19 +60,19 @@ func (s *FileService) DeleteBySize() error { }) // calculate sum of all files size - var used int = 0 + var used int64 = 0 for _, file := range files { - used += int(file.Size) + used += file.Size } - var deleteSize int = 0 + var deleteSize int64 = 0 var deleteFiles []LogFile for _, file := range files { if used-deleteSize < retentionSize { break } deleteFiles = append(deleteFiles, file) - deleteSize += int(file.Size) + deleteSize += file.Size } logger.Infof("DeleteBySize: try to flush %d files, %d bytes", len(deleteFiles), deleteSize) for _, file := range deleteFiles { @@ -87,7 +87,7 @@ func (s *FileService) DeleteBySize() error { } func (s *FileService) GetUsedBytes(subpath string) (int, error) { - if s.config.RetentionSizingStrategy() == "disk" { + if s.Config.Retention.SizingStrategy == "disk" { return s.GetUsedBytesFromDisk(subpath) } return s.GetUsedBytesFromFiles(subpath) diff --git a/storage/fileservice/delete_test.go b/storage/fileservice/delete_test.go index 637d8ec..652963d 100644 --- a/storage/fileservice/delete_test.go +++ b/storage/fileservice/delete_test.go @@ -1,215 +1,237 @@ package fileservice import ( - "fmt" "testing" "time" + "github.com/kuoss/common/tester" "github.com/kuoss/lethe/clock" "github.com/kuoss/lethe/config" - "github.com/kuoss/lethe/util/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -var ( - fileService2 *FileService -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/storage_fileservice_delete_test") - fileService2, err = New(cfg) - if err != nil { - panic(err) - } -} - func TestDeleteByAge(t *testing.T) { testCases := []struct { retentionTime time.Duration - want []LogFile + want []string }{ { 0, // disabled - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2000-01-01_00.log", Subpath: "pod/namespace01/2000-01-01_00.log", LogType: "pod", Target: "namespace01", Name: "2000-01-01_00.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/0000-00-00_00.log", Subpath: "pod/namespace02/0000-00-00_00.log", LogType: "pod", Target: "namespace02", Name: "0000-00-00_00.log", Extension: ".log", Size: 12}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2000-01-01_00.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/0000-00-00_00.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 1 * time.Second, // 1 second - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + 1 * time.Second, // 1s + []string{ + "node/node01/2009-11-10_22.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 1 * time.Hour, // 1 hour - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + 1 * time.Hour, // 1h + []string{ + "node/node01/2009-11-10_22.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 2 * time.Hour, // 2 hours - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + 2 * time.Hour, // 2h + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 1 * 24 * time.Hour, // 1 day - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + 1 * 24 * time.Hour, // 1d + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 10 * 24 * time.Hour, // 10 days - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + 10 * 24 * time.Hour, // 10d + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { - 100 * 365 * 24 * time.Hour, // 100 years - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2000-01-01_00.log", Subpath: "pod/namespace01/2000-01-01_00.log", LogType: "pod", Target: "namespace01", Name: "2000-01-01_00.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + 100 * 365 * 24 * time.Hour, // 100y + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2000-01-01_00.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d__%s", i, tc.retentionTime), func(t *testing.T) { - fileService2.config.SetRetentionTime(tc.retentionTime) - err := fileService2.DeleteByAge() - assert.NoError(t, err) + t.Run(tester.CaseName(i, tc.retentionTime), func(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() - got, err := fileService2.ListFiles() - assert.NoError(t, err) - assert.Equal(t, tc.want, got) - testutil.ResetLogData() + cfg, err := config.New("test") + require.NoError(t, err) + cfg.Retention.Time = tc.retentionTime + + fileService, err := New(cfg) + require.NoError(t, err) + + err = fileService.DeleteByAge() + require.NoError(t, err) + + listFiles, err := fileService.ListFiles() + require.NoError(t, err) + + subPaths := []string{} + for _, f := range listFiles { + subPaths = append(subPaths, f.Subpath) + } + require.Equal(t, tc.want, subPaths) }) } } func TestDeleteBySize(t *testing.T) { testCases := []struct { - retentionSize int - want []LogFile + retentionSize int64 + want []string }{ { 0, // disabled - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2000-01-01_00.log", Subpath: "pod/namespace01/2000-01-01_00.log", LogType: "pod", Target: "namespace01", Name: "2000-01-01_00.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/0000-00-00_00.log", Subpath: "pod/namespace02/0000-00-00_00.log", LogType: "pod", Target: "namespace02", Name: "0000-00-00_00.log", Extension: ".log", Size: 12}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2000-01-01_00.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/0000-00-00_00.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { 10, // 10 bytes - []LogFile{}, + []string{}, }, { 1 * 1024, // 1 KiB - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}}, + []string{"pod/namespace01/2029-11-10_23.log"}, }, { 2 * 1024, // 2 KiB - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + []string{ + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { 3 * 1024, // 3 KiB - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + []string{ + "node/node01/2009-11-10_22.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { 1 * 1024 * 1024, // 1 MiB - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2000-01-01_00.log", Subpath: "pod/namespace01/2000-01-01_00.log", LogType: "pod", Target: "namespace01", Name: "2000-01-01_00.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/0000-00-00_00.log", Subpath: "pod/namespace02/0000-00-00_00.log", LogType: "pod", Target: "namespace02", Name: "0000-00-00_00.log", Extension: ".log", Size: 12}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2000-01-01_00.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/0000-00-00_00.log", + "pod/namespace02/2009-11-10_22.log", + }, }, { 999999999 * 1024 * 1024 * 1024, // 999,999,999 GiB - []LogFile{ - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_21.log", Subpath: "node/node01/2009-11-10_21.log", LogType: "node", Target: "node01", Name: "2009-11-10_21.log", Extension: ".log", Size: 1057}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node01/2009-11-10_22.log", Subpath: "node/node01/2009-11-10_22.log", LogType: "node", Target: "node01", Name: "2009-11-10_22.log", Extension: ".log", Size: 177}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-01_00.log", Subpath: "node/node02/2009-11-01_00.log", LogType: "node", Target: "node02", Name: "2009-11-01_00.log", Extension: ".log", Size: 0}, - {Fullpath: "tmp/storage_fileservice_delete_test/node/node02/2009-11-10_21.log", Subpath: "node/node02/2009-11-10_21.log", LogType: "node", Target: "node02", Name: "2009-11-10_21.log", Extension: ".log", Size: 1116}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2000-01-01_00.log", Subpath: "pod/namespace01/2000-01-01_00.log", LogType: "pod", Target: "namespace01", Name: "2000-01-01_00.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_21.log", Subpath: "pod/namespace01/2009-11-10_21.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_21.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2009-11-10_22.log", Subpath: "pod/namespace01/2009-11-10_22.log", LogType: "pod", Target: "namespace01", Name: "2009-11-10_22.log", Extension: ".log", Size: 1031}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace01/2029-11-10_23.log", Subpath: "pod/namespace01/2029-11-10_23.log", LogType: "pod", Target: "namespace01", Name: "2029-11-10_23.log", Extension: ".log", Size: 279}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/0000-00-00_00.log", Subpath: "pod/namespace02/0000-00-00_00.log", LogType: "pod", Target: "namespace02", Name: "0000-00-00_00.log", Extension: ".log", Size: 12}, - {Fullpath: "tmp/storage_fileservice_delete_test/pod/namespace02/2009-11-10_22.log", Subpath: "pod/namespace02/2009-11-10_22.log", LogType: "pod", Target: "namespace02", Name: "2009-11-10_22.log", Extension: ".log", Size: 1125}}, + []string{ + "node/node01/2009-11-10_21.log", + "node/node01/2009-11-10_22.log", + "node/node02/2009-11-01_00.log", + "node/node02/2009-11-10_21.log", + "pod/namespace01/2000-01-01_00.log", + "pod/namespace01/2009-11-10_21.log", + "pod/namespace01/2009-11-10_22.log", + "pod/namespace01/2029-11-10_23.log", + "pod/namespace02/0000-00-00_00.log", + "pod/namespace02/2009-11-10_22.log", + }, }, } for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d__%d", i, tc.retentionSize), func(t *testing.T) { - fileService2.config.SetRetentionSize(tc.retentionSize) - err := fileService2.DeleteBySize() - assert.NoError(t, err) + t.Run(tester.CaseName(i, tc.retentionSize), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + cfg, err := config.New("test") + require.NoError(t, err) + cfg.Retention.Size = tc.retentionSize + + fileService, err := New(cfg) + require.NoError(t, err) + + err = fileService.DeleteBySize() + require.NoError(t, err) - got, err := fileService2.ListFiles() - assert.NoError(t, err) - assert.Equal(t, tc.want, got) - testutil.ResetLogData() + listFiles, err := fileService.ListFiles() + require.NoError(t, err) + subPaths := []string{} + for _, f := range listFiles { + subPaths = append(subPaths, f.Subpath) + } + require.Equal(t, tc.want, subPaths) }) } } diff --git a/storage/fileservice/fileservice.go b/storage/fileservice/fileservice.go index fdcb0b4..622d3dd 100644 --- a/storage/fileservice/fileservice.go +++ b/storage/fileservice/fileservice.go @@ -10,23 +10,19 @@ import ( ) type FileService struct { - config *config.Config + Config *config.Config driver storagedriver.Driver } func New(cfg *config.Config) (*FileService, error) { - driver, err := factory.Get("filesystem", map[string]any{"RootDirectory": cfg.LogDataPath()}) + driver, err := factory.Get("filesystem", map[string]any{"RootDirectory": cfg.Storage.LogDataPath}) if err != nil { return nil, fmt.Errorf("factory.Get err: %w", err) } - if err := os.MkdirAll(cfg.LogDataPath(), os.ModePerm); err != nil { + if err := os.MkdirAll(cfg.Storage.LogDataPath, os.ModePerm); err != nil { return nil, fmt.Errorf("os.MkdirAll err: %w", err) } return &FileService{cfg, driver}, nil } - -func (s *FileService) Config() *config.Config { - return s.config -} diff --git a/storage/fileservice/fileservice_test.go b/storage/fileservice/fileservice_test.go index 626f7ab..79da197 100644 --- a/storage/fileservice/fileservice_test.go +++ b/storage/fileservice/fileservice_test.go @@ -1,56 +1,18 @@ package fileservice import ( - "os" - "path/filepath" "testing" "github.com/kuoss/lethe/config" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestNew(t *testing.T) { - assert.NotEmpty(t, fileService) - assert.Equal(t, "filesystem", fileService.driver.Name()) - assert.Equal(t, "tmp/init", fileService.driver.RootDirectory()) -} - -func TestNewTemp(t *testing.T) { - cfg, err := config.New("test") - assert.NoError(t, err) - assert.NotEmpty(t, cfg) - - dataPath := "tmp/temp" // caution: RemoveAll() - - cfg.SetLogDataPath(dataPath) - tempFileService, err := New(cfg) - assert.NoError(t, err) - assert.NotEmpty(t, tempFileService) - assert.DirExists(t, dataPath) // exists - - // duplicate ok - cfg.SetLogDataPath(dataPath) - tempFileService2, err := New(cfg) - assert.NoError(t, err) - assert.NotEmpty(t, tempFileService2) - assert.DirExists(t, dataPath) // exists - - // clean up - err = os.RemoveAll(dataPath) - assert.NoError(t, err) -} - -func TestNewLogDataPath(t *testing.T) { cfg, err := config.New("test") - assert.NoError(t, err) - - cfg.SetLogDataPath(filepath.Join(".", "tmp", "writer")) - - _, err = New(cfg) - - assert.NoError(t, err) - assert.DirExists(t, filepath.Join(".", "tmp", "writer")) + require.NoError(t, err) - err = os.RemoveAll(filepath.Join(".", "tmp", "writer")) - assert.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) + require.NotEmpty(t, fileService) + require.Equal(t, cfg, fileService.Config) } diff --git a/storage/fileservice/init_test.go b/storage/fileservice/init_test.go deleted file mode 100644 index 0700554..0000000 --- a/storage/fileservice/init_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package fileservice - -import ( - "github.com/kuoss/lethe/config" - _ "github.com/kuoss/lethe/storage/driver/filesystem" - "github.com/kuoss/lethe/util/testutil" -) - -var ( - fileService *FileService -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/init") - fileService, err = New(cfg) - if err != nil { - panic(err) - } -} diff --git a/storage/fileservice/list.go b/storage/fileservice/list.go index 0c8bd70..4612d6c 100644 --- a/storage/fileservice/list.go +++ b/storage/fileservice/list.go @@ -117,7 +117,7 @@ func (s *FileService) ListFiles() ([]LogFile, error) { logFiles := []LogFile{} rootDir := s.driver.RootDirectory() for _, fileInfo := range fileInfos { - logPath := storagedriver.LogPath{RootDirectory: rootDir, Subpath: fullpath2subpath(s.config.LogDataPath(), fileInfo.Fullpath())} + logPath := storagedriver.LogPath{RootDirectory: rootDir, Subpath: fullpath2subpath(s.Config.Storage.LogDataPath, fileInfo.Fullpath())} if logPath.Depth() == storagedriver.DepthFile { logFiles = append(logFiles, LogFile{ Fullpath: logPath.Fullpath(), diff --git a/storage/fileservice/list_test.go b/storage/fileservice/list_test.go index cbff686..79726e0 100644 --- a/storage/fileservice/list_test.go +++ b/storage/fileservice/list_test.go @@ -5,7 +5,9 @@ import ( "sort" "testing" - "github.com/stretchr/testify/assert" + "github.com/kuoss/common/tester" + "github.com/kuoss/lethe/config" + "github.com/stretchr/testify/require" ) func TestFullpath2subpath(t *testing.T) { @@ -17,14 +19,14 @@ func TestFullpath2subpath(t *testing.T) { {"", "", "."}, {"hello", "", "../."}, {"", "hello", "hello"}, - {"tmp/init", "tmp/init/pod", "pod"}, - {"tmp/init", "tmp/init/pod/ns1", "pod/ns1"}, - {"tmp/init", "tmp/init/pod/ns1/2023", "pod/ns1/2023"}, + {"data/log", "data/log/pod", "pod"}, + {"data/log", "data/log/pod/ns1", "pod/ns1"}, + {"data/log", "data/log/pod/ns1/2023", "pod/ns1/2023"}, } for i, tc := range testCases { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { got := fullpath2subpath(tc.rootDir, tc.fullpath) - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } } @@ -36,7 +38,7 @@ func TestDirSize(t *testing.T) { wantError string }{ {"", 0, ""}, - {"hello", 0, "Path not found: hello, err: open err: open tmp/init/hello: no such file or directory"}, + {"hello", 0, "Path not found: hello, err: open err: open data/log/hello: no such file or directory"}, {"node", 0, ""}, {"pod", 0, ""}, {"node/node01", 1234, ""}, @@ -44,15 +46,24 @@ func TestDirSize(t *testing.T) { {"pod/namespace01", 2620, ""}, {"pod/namespace02", 1137, ""}, } - for _, tc := range testCases { - t.Run("", func(t *testing.T) { + for i, tc := range testCases { + t.Run(tester.CaseName(i, tc.path), func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) + got, err := fileService.dirSize(tc.path) if tc.wantError == "" { - assert.NoError(t, err) + require.NoError(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } } @@ -76,7 +87,7 @@ func TestList(t *testing.T) { { "hello", nil, - "list err: Path not found: hello, err: open err: open tmp/init/hello: no such file or directory", + "list err: Path not found: hello, err: open err: open data/log/hello: no such file or directory", }, { "node", @@ -96,51 +107,85 @@ func TestList(t *testing.T) { { "pod/namespace01/2029-11-10_23.log", nil, - "list err: readdirnames err: readdirent tmp/init/pod/namespace01/2029-11-10_23.log: not a directory", + "list err: readdirnames err: readdirent data/log/pod/namespace01/2029-11-10_23.log: not a directory", }, } for _, tc := range testCases { t.Run("", func(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) + got, err := fileService.List(tc.subpath) if tc.wantError == "" { - assert.NoError(t, err) + require.NoError(t, err) } else { - assert.EqualError(t, err, tc.wantError) + require.EqualError(t, err, tc.wantError) } sort.Strings(got) - assert.Equal(t, tc.want, got) + require.Equal(t, tc.want, got) }) } } func TestListLogDirs(t *testing.T) { - // TODO: if runtime.GOOS == "windows" + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) + want := []LogDir{ - {Fullpath: "tmp/init/node/node01", Subpath: "node/node01", LogType: "node", Target: "node01", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}, - {Fullpath: "tmp/init/node/node02", Subpath: "node/node02", LogType: "node", Target: "node02", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}, - {Fullpath: "tmp/init/pod/namespace01", Subpath: "pod/namespace01", LogType: "pod", Target: "namespace01", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}, - {Fullpath: "tmp/init/pod/namespace02", Subpath: "pod/namespace02", LogType: "pod", Target: "namespace02", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}} + {Fullpath: "data/log/node/node01", Subpath: "node/node01", LogType: "node", Target: "node01", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}, + {Fullpath: "data/log/node/node02", Subpath: "node/node02", LogType: "node", Target: "node02", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}, + {Fullpath: "data/log/pod/namespace01", Subpath: "pod/namespace01", LogType: "pod", Target: "namespace01", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}, + {Fullpath: "data/log/pod/namespace02", Subpath: "pod/namespace02", LogType: "pod", Target: "namespace02", FileCount: 0, FirstFile: "", LastFile: "", Size: 0, LastForward: ""}} got := fileService.ListLogDirs() - assert.Equal(t, want, got) + require.Equal(t, want, got) } func TestListLogDirsWithSize(t *testing.T) { + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) + want := []LogDir{ - {Fullpath: "tmp/init/node/node01", Subpath: "node/node01", LogType: "node", Target: "node01", FileCount: 2, FirstFile: "2009-11-10_21.log", LastFile: "2009-11-10_22.log", Size: 1234, LastForward: ""}, - {Fullpath: "tmp/init/node/node02", Subpath: "node/node02", LogType: "node", Target: "node02", FileCount: 2, FirstFile: "2009-11-01_00.log", LastFile: "2009-11-10_21.log", Size: 1116, LastForward: ""}, - {Fullpath: "tmp/init/pod/namespace01", Subpath: "pod/namespace01", LogType: "pod", Target: "namespace01", FileCount: 4, FirstFile: "2000-01-01_00.log", LastFile: "2029-11-10_23.log", Size: 2620, LastForward: ""}, - {Fullpath: "tmp/init/pod/namespace02", Subpath: "pod/namespace02", LogType: "pod", Target: "namespace02", FileCount: 2, FirstFile: "0000-00-00_00.log", LastFile: "2009-11-10_22.log", Size: 1137, LastForward: ""}} + {Fullpath: "data/log/node/node01", Subpath: "node/node01", LogType: "node", Target: "node01", FileCount: 2, FirstFile: "2009-11-10_21.log", LastFile: "2009-11-10_22.log", Size: 1234, LastForward: ""}, + {Fullpath: "data/log/node/node02", Subpath: "node/node02", LogType: "node", Target: "node02", FileCount: 2, FirstFile: "2009-11-01_00.log", LastFile: "2009-11-10_21.log", Size: 1116, LastForward: ""}, + {Fullpath: "data/log/pod/namespace01", Subpath: "pod/namespace01", LogType: "pod", Target: "namespace01", FileCount: 4, FirstFile: "2000-01-01_00.log", LastFile: "2029-11-10_23.log", Size: 2620, LastForward: ""}, + {Fullpath: "data/log/pod/namespace02", Subpath: "pod/namespace02", LogType: "pod", Target: "namespace02", FileCount: 2, FirstFile: "0000-00-00_00.log", LastFile: "2009-11-10_22.log", Size: 1137, LastForward: ""}} got := fileService.ListLogDirsWithSize() - assert.Equal(t, want, got) + require.Equal(t, want, got) } func TestListTargets(t *testing.T) { - // TODO: if runtime.GOOS == "windows" + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := New(cfg) + require.NoError(t, err) + want := []LogDir{ - {Fullpath: "tmp/init/node/node01", Subpath: "node/node01", LogType: "node", Target: "node01", FileCount: 2, FirstFile: "2009-11-10_21.log", LastFile: "2009-11-10_22.log", Size: 1234, LastForward: "2009-11-10T23:00:00Z"}, - {Fullpath: "tmp/init/node/node02", Subpath: "node/node02", LogType: "node", Target: "node02", FileCount: 2, FirstFile: "2009-11-01_00.log", LastFile: "2009-11-10_21.log", Size: 1116, LastForward: "2009-11-10T21:58:00Z"}, - {Fullpath: "tmp/init/pod/namespace01", Subpath: "pod/namespace01", LogType: "pod", Target: "namespace01", FileCount: 4, FirstFile: "2000-01-01_00.log", LastFile: "2029-11-10_23.log", Size: 2620, LastForward: "2009-11-10T23:00:00Z"}, - {Fullpath: "tmp/init/pod/namespace02", Subpath: "pod/namespace02", LogType: "pod", Target: "namespace02", FileCount: 2, FirstFile: "0000-00-00_00.log", LastFile: "2009-11-10_22.log", Size: 1137, LastForward: "2009-11-10T22:58:00Z"}} + {Fullpath: "data/log/node/node01", Subpath: "node/node01", LogType: "node", Target: "node01", FileCount: 2, FirstFile: "2009-11-10_21.log", LastFile: "2009-11-10_22.log", Size: 1234, LastForward: "2009-11-10T23:00:00Z"}, + {Fullpath: "data/log/node/node02", Subpath: "node/node02", LogType: "node", Target: "node02", FileCount: 2, FirstFile: "2009-11-01_00.log", LastFile: "2009-11-10_21.log", Size: 1116, LastForward: "2009-11-10T21:58:00Z"}, + {Fullpath: "data/log/pod/namespace01", Subpath: "pod/namespace01", LogType: "pod", Target: "namespace01", FileCount: 4, FirstFile: "2000-01-01_00.log", LastFile: "2029-11-10_23.log", Size: 2620, LastForward: "2009-11-10T23:00:00Z"}, + {Fullpath: "data/log/pod/namespace02", Subpath: "pod/namespace02", LogType: "pod", Target: "namespace02", FileCount: 2, FirstFile: "0000-00-00_00.log", LastFile: "2009-11-10_22.log", Size: 1137, LastForward: "2009-11-10T22:58:00Z"}} got := fileService.ListTargets() - assert.Equal(t, want, got) + require.Equal(t, want, got) } diff --git a/storage/logservice/init_test.go b/storage/logservice/init_test.go deleted file mode 100644 index 15ed4c3..0000000 --- a/storage/logservice/init_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package logservice - -import ( - "github.com/kuoss/lethe/clock" - "github.com/kuoss/lethe/config" - "github.com/kuoss/lethe/storage/fileservice" - "github.com/kuoss/lethe/util/testutil" -) - -var ( - logService1 *LogService -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/init") - fileService, err := fileservice.New(cfg) - if err != nil { - panic(err) - } - logService1 = New(fileService) -} diff --git a/storage/logservice/logservice.go b/storage/logservice/logservice.go index 12c64e1..06db6d9 100644 --- a/storage/logservice/logservice.go +++ b/storage/logservice/logservice.go @@ -15,15 +15,12 @@ import ( ) type LogService struct { + config *config.Config fileService *fileservice.FileService } -func New(fileService *fileservice.FileService) *LogService { - return &LogService{fileService} -} - -func (s *LogService) Config() *config.Config { - return s.fileService.Config() +func New(cfg *config.Config, fileService *fileservice.FileService) *LogService { + return &LogService{cfg, fileService} } func (s *LogService) SelectLog(sel *model.LogSelector) (log model.Log, warnings model.Warnings, err error) { @@ -53,9 +50,6 @@ func (s *LogService) SelectLog(sel *model.LogSelector) (log model.Log, warnings // log log = s.getLogFromFiles(sel, files, mfs, &warnings) - if err != nil { - return log, warnings, fmt.Errorf("getTargets err: %w", err) - } return log, warnings, nil } @@ -110,7 +104,7 @@ func isFileInTimeRange(file string, tr *model.TimeRange) bool { // log func (s *LogService) getLogFromFiles(sel *model.LogSelector, files []string, mfs *match.MatchFuncSet, warnings *model.Warnings) model.Log { - limit := s.Config().Limit() + limit := s.config.Query.Limit logLines := []model.LogLine{} sort.Strings(files) for _, file := range files { @@ -126,7 +120,7 @@ func (s *LogService) getLogFromFiles(sel *model.LogSelector, files []string, mfs } func (s *LogService) addLogLinesFromFile(sel *model.LogSelector, logLines *[]model.LogLine, file string, mfs *match.MatchFuncSet, warnings *model.Warnings) { - limit := s.Config().Limit() + limit := s.config.Query.Limit sc, err := s.fileService.Scanner(file) if err != nil { *warnings = append(*warnings, err) diff --git a/storage/logservice/logservice_test.go b/storage/logservice/logservice_test.go index 235d634..b9034ac 100644 --- a/storage/logservice/logservice_test.go +++ b/storage/logservice/logservice_test.go @@ -3,9 +3,16 @@ package logservice import ( "testing" - "github.com/stretchr/testify/assert" + "github.com/kuoss/lethe/config" + "github.com/kuoss/lethe/storage/fileservice" + "github.com/stretchr/testify/require" ) func TestNew(t *testing.T) { - assert.NotEmpty(t, logService1) + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := fileservice.New(cfg) + require.NoError(t, err) + logService := New(cfg, fileService) + require.NotEmpty(t, logService) } diff --git a/storage/querier/init_test.go b/storage/querier/init_test.go deleted file mode 100644 index 2332e9f..0000000 --- a/storage/querier/init_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package querier - -import ( - "github.com/kuoss/lethe/clock" - "github.com/kuoss/lethe/util/testutil" -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) -} diff --git a/storage/querier/querier_test.go b/storage/querier/querier_test.go new file mode 100644 index 0000000..5a9284f --- /dev/null +++ b/storage/querier/querier_test.go @@ -0,0 +1,44 @@ +package querier + +import ( + "context" + "testing" + + "github.com/prometheus/prometheus/model/labels" + promstorage "github.com/prometheus/prometheus/storage" + "github.com/stretchr/testify/require" +) + +// MockSeriesSet is a mock implementation of the promstorage.SeriesSet interface for testing purposes. +type MockSeriesSet struct{} + +func (m *MockSeriesSet) Next() bool { return false } +func (m *MockSeriesSet) At() promstorage.Series { return nil } +func (m *MockSeriesSet) Err() error { return nil } +func (m *MockSeriesSet) Warnings() promstorage.Warnings { return nil } + +func TestLetheQueryable_Querier(t *testing.T) { + mockSeriesSet := &MockSeriesSet{} + mockSelectFunction := func(sortSeries bool, hints *promstorage.SelectHints, matchers ...*labels.Matcher) promstorage.SeriesSet { + return mockSeriesSet + } + + letheQuerier := &LetheQuerier{ + SelectLetheFunction: mockSelectFunction, + } + + letheQueryable := &LetheQueryable{ + LetheQuerier: letheQuerier, + } + + // Create a new querier using the LetheQueryable + querier, err := letheQueryable.Querier(context.Background(), 0, 0) + require.NoError(t, err, "Expected no error from LetheQueryable.Querier") + + // Verify the returned querier is the same as the one we set up + require.Equal(t, letheQuerier, querier, "Expected the querier to match the LetheQuerier") + + // Test the Select method of LetheQuerier + result := querier.Select(false, nil) + require.Equal(t, mockSeriesSet, result, "Expected the SeriesSet returned by Select to match the mockSeriesSet") +} diff --git a/storage/queryservice/init_test.go b/storage/queryservice/init_test.go deleted file mode 100644 index e3a4da0..0000000 --- a/storage/queryservice/init_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package queryservice - -import ( - "github.com/kuoss/lethe/clock" - "github.com/kuoss/lethe/config" - "github.com/kuoss/lethe/storage/fileservice" - "github.com/kuoss/lethe/storage/logservice" - "github.com/kuoss/lethe/util/testutil" -) - -var ( - queryService *QueryService -) - -func init() { - testutil.ChdirRoot() - testutil.ResetLogData() - clock.SetPlaygroundMode(true) - - cfg, err := config.New("test") - if err != nil { - panic(err) - } - cfg.SetLogDataPath("tmp/init") - fileService, err := fileservice.New(cfg) - if err != nil { - panic(err) - } - logService := logservice.New(fileService) - queryService = New(logService) -} diff --git a/storage/queryservice/queryservice.go b/storage/queryservice/queryservice.go index 9b4dd64..299e04a 100644 --- a/storage/queryservice/queryservice.go +++ b/storage/queryservice/queryservice.go @@ -7,6 +7,7 @@ import ( "github.com/kuoss/common/logger" "github.com/kuoss/lethe/clock" + "github.com/kuoss/lethe/config" "github.com/kuoss/lethe/letheql" "github.com/kuoss/lethe/letheql/model" "github.com/kuoss/lethe/storage/logservice" @@ -18,9 +19,9 @@ type QueryService struct { queryable *querier.LetheQueryable } -func New(logService *logservice.LogService) *QueryService { +func New(cfg *config.Config, logService *logservice.LogService) *QueryService { return &QueryService{ - engine: letheql.NewEngine(logService), + engine: letheql.NewEngine(cfg, logService), queryable: &querier.LetheQueryable{ LetheQuerier: &querier.LetheQuerier{}, }, diff --git a/storage/queryservice/queryservice_test.go b/storage/queryservice/queryservice_test.go index 204c8f1..74abc0e 100644 --- a/storage/queryservice/queryservice_test.go +++ b/storage/queryservice/queryservice_test.go @@ -5,16 +5,17 @@ import ( "fmt" "testing" + "github.com/kuoss/common/tester" + "github.com/kuoss/lethe/clock" + "github.com/kuoss/lethe/config" "github.com/kuoss/lethe/letheql" "github.com/kuoss/lethe/letheql/model" + "github.com/kuoss/lethe/storage/fileservice" + "github.com/kuoss/lethe/storage/logservice" "github.com/kuoss/lethe/storage/logservice/logmodel" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestNew(t *testing.T) { - assert.NotEmpty(t, queryService) -} - func TestQuery(t *testing.T) { testCases := []struct { qs string @@ -39,14 +40,30 @@ func TestQuery(t *testing.T) { } for i, tc := range testCases { t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + clock.SetPlaygroundMode(true) + defer clock.SetPlaygroundMode(false) + + _, cleanup := tester.SetupDir(t, map[string]string{ + "@/testdata/log": "data/log", + }) + defer cleanup() + + cfg, err := config.New("test") + require.NoError(t, err) + fileService, err := fileservice.New(cfg) + require.NoError(t, err) + logService := logservice.New(cfg, fileService) + queryService := New(cfg, logService) + require.NotEmpty(t, queryService) + res := queryService.Query(context.TODO(), tc.qs, tc.tr) if tc.wantError == "" { - assert.NoError(t, res.Err) + require.NoError(t, res.Err) } else { - assert.EqualError(t, res.Err, tc.wantError) + require.EqualError(t, res.Err, tc.wantError) } res.Err = nil - assert.Equal(t, tc.want, res) + require.Equal(t, tc.want, res) }) } } diff --git a/testdata/etc/lethe.error1.yaml b/testdata/etc/lethe.error1.yaml index a22da18..dbff829 100644 --- a/testdata/etc/lethe.error1.yaml +++ b/testdata/etc/lethe.error1.yaml @@ -1,6 +1,6 @@ : - size: 200m + size: 200MB time: 1d storage: driver: filesystem - log_data_path: /data/log \ No newline at end of file + log_data_path: /data/log diff --git a/testdata/etc/lethe.error2.yaml b/testdata/etc/lethe.error2.yaml index a9fb923..cefcb65 100644 --- a/testdata/etc/lethe.error2.yaml +++ b/testdata/etc/lethe.error2.yaml @@ -1,6 +1,6 @@ retention: - size: 200 + size: 200x time: 1d storage: driver: filesystem - log_data_path: /data/log \ No newline at end of file + log_data_path: /data/log diff --git a/testdata/etc/lethe.error3.yaml b/testdata/etc/lethe.error3.yaml index e6c7d64..b0307a1 100644 --- a/testdata/etc/lethe.error3.yaml +++ b/testdata/etc/lethe.error3.yaml @@ -1,6 +1,6 @@ retention: - size: 200m + size: 200MB time: 1 storage: driver: filesystem - log_data_path: /data/log \ No newline at end of file + log_data_path: /data/log diff --git a/testdata/etc/lethe.error4.yaml b/testdata/etc/lethe.error4.yaml new file mode 100644 index 0000000..a64cab4 --- /dev/null +++ b/testdata/etc/lethe.error4.yaml @@ -0,0 +1,8 @@ +retention: + size: 200GB + time: 1d +storage: + driver: filesystem + log_data_path: /tmp/log +web: + listen_address: foo diff --git a/testdata/etc/lethe.legacy.yaml b/testdata/etc/lethe.legacy.yaml new file mode 100644 index 0000000..8622815 --- /dev/null +++ b/testdata/etc/lethe.legacy.yaml @@ -0,0 +1,6 @@ +retention: + size: 100g + time: 15d +storage: + driver: filesystem + log_data_path: /tmp/log diff --git a/testdata/etc/lethe.main.yaml b/testdata/etc/lethe.main.yaml index 3a75911..ffbc35b 100644 --- a/testdata/etc/lethe.main.yaml +++ b/testdata/etc/lethe.main.yaml @@ -1,6 +1,8 @@ retention: - size: 200m + size: 200GB time: 1d storage: driver: filesystem - log_data_path: /tmp + log_data_path: /tmp/log +web: + gin_mode: debug diff --git a/testdata/etc/lethe.ok1.yaml b/testdata/etc/lethe.ok1.yaml index 59f1b9e..e08a0fe 100644 --- a/testdata/etc/lethe.ok1.yaml +++ b/testdata/etc/lethe.ok1.yaml @@ -1,6 +1,6 @@ retention: - size: 200m + size: 200GB time: 1d storage: driver: filesystem - log_data_path: /data/log \ No newline at end of file + log_data_path: /tmp/log diff --git a/testdata/etc/lethe.ok2.yaml b/testdata/etc/lethe.ok2.yaml index 438a129..af9f787 100644 --- a/testdata/etc/lethe.ok2.yaml +++ b/testdata/etc/lethe.ok2.yaml @@ -1,5 +1,5 @@ retention: - size: 300m + size: 300GB storage: driver: filesystem - log_data_path: /var/data/log + log_data_path: /tmp/log diff --git a/util/string.go b/util/string.go deleted file mode 100644 index 45a93ad..0000000 --- a/util/string.go +++ /dev/null @@ -1,47 +0,0 @@ -package util - -import ( - "fmt" - "strconv" - "strings" -) - -func SubstrAfter(haystack string, needle string) string { - pos := strings.Index(haystack, needle) - if pos == -1 { - return haystack - } - return haystack[pos+len(needle):] -} -func SubstrBefore(haystack string, needle string) string { - pos := strings.Index(haystack, needle) - if pos == -1 { - return haystack - } - return haystack[:pos] -} - -func CountNewlines(s string) string { - n := strings.Count(s, "\n") - if len(s) > 0 && !strings.HasSuffix(s, "\n") { - n++ - } - return strconv.Itoa(n) -} - -func StringToBytes(str string) (int, error) { - unit := str[len(str)-1:] - num, err := strconv.Atoi(str[:len(str)-1]) - if err != nil { - return 0, err - } - switch unit { - case "k": - return num * 1024, nil - case "m": - return num * 1024 * 1024, nil - case "g": - return num * 1024 * 1024 * 1024, nil - } - return 0, fmt.Errorf("cannot accept unit '%s' in '%s''. allowed units: [k, m, g]", unit, str) -} diff --git a/util/string_test.go b/util/string_test.go deleted file mode 100644 index f2a3675..0000000 --- a/util/string_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package util - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSubstrAfter(t *testing.T) { - testCases := []struct { - haystack string - needle string - want string - }{ - {"hello world", "w", "orld"}, - {"hello world", " ", "world"}, - {"hello world", "l", "lo world"}, - } - for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - got := SubstrAfter(tc.haystack, tc.needle) - assert.Equal(t, tc.want, got) - }) - } -} - -func TestSubstrBefore(t *testing.T) { - assert.Equal(t, "hello ", SubstrBefore("hello world", "w")) - assert.Equal(t, "hello", SubstrBefore("hello world", " ")) - assert.Equal(t, "he", SubstrBefore("hello world", "l")) -} - -func TestCountNewlines(t *testing.T) { - assert.Equal(t, "1", CountNewlines("hello world")) - assert.Equal(t, "2", CountNewlines("hello\nworld")) - assert.Equal(t, "2", CountNewlines("hello\nworld\n")) - -} - -func TestStringToBytes(t *testing.T) { - var out int - out, _ = StringToBytes("100k") - assert.Equal(t, 102400, out) - out, _ = StringToBytes("100m") - assert.Equal(t, 104857600, out) - out, _ = StringToBytes("100g") - assert.Equal(t, 107374182400, out) -} diff --git a/util/testutil/testutil.go b/util/testutil/testutil.go deleted file mode 100644 index 854edd5..0000000 --- a/util/testutil/testutil.go +++ /dev/null @@ -1,102 +0,0 @@ -package testutil - -import ( - "fmt" - "os" - "path" - "path/filepath" - "runtime" - "strings" -) - -var root string - -func ChdirRoot() { - if root != "" { - fmt.Printf("ChdirRoot: %s (skipped)\n", root) - return - } - - _, filename, _, _ := runtime.Caller(0) - root = path.Join(path.Dir(filename), "../..") - fmt.Printf("ChdirRoot: %s\n", root) - err := os.Chdir(root) - if err != nil { - panic(err) - } -} - -func getTestID() string { - _, filename, _, _ := runtime.Caller(2) - if strings.HasSuffix(filename, "/init_test.go") { - return "init" - } - rel, err := filepath.Rel(root, filename) - if err != nil { - panic(err) - } - rel = strings.ReplaceAll(rel, ".go", "") - rel = strings.ReplaceAll(rel, string(os.PathSeparator), "_") - return rel -} - -func ResetLogData() { - testID := getTestID() - logDataPath := "tmp/" + testID - if testID != "init" { - fmt.Printf("remove logDataPath: %s\n", logDataPath) - err := os.RemoveAll(logDataPath) - if err != nil { - panic(err) - } - } - fmt.Printf("fill logDataPath: %s\n", logDataPath) - err := copyRecursively("./testdata/log", logDataPath) - if err != nil { - panic(err) - } -} - -func copyRecursively(src string, dest string) error { - f, err := os.Open(src) - if err != nil { - return fmt.Errorf("open err: %w", err) - } - file, err := f.Stat() - if err != nil { - return fmt.Errorf("stat err: %w", err) - } - if !file.IsDir() { - return fmt.Errorf("not dir: %s", file.Name()) - } - err = os.MkdirAll(dest, 0755) - if err != nil { - return fmt.Errorf("mkdirAll err: %w", err) - } - files, err := os.ReadDir(src) - if err != nil { - return fmt.Errorf("readDir err: %w", err) - } - for _, f := range files { - srcFile := src + "/" + f.Name() - destFile := dest + "/" + f.Name() - // dir - if f.IsDir() { - err := copyRecursively(srcFile, destFile) - if err != nil { - return fmt.Errorf("copyRecursively err: %w", err) - } - continue - } - // file - content, err := os.ReadFile(srcFile) - if err != nil { - return fmt.Errorf("readFile err: %w", err) - } - err = os.WriteFile(destFile, content, 0755) - if err != nil { - return fmt.Errorf("writeFile err: %w", err) - } - } - return nil -} diff --git a/util/time.go b/util/time.go deleted file mode 100644 index 695c0d9..0000000 --- a/util/time.go +++ /dev/null @@ -1,56 +0,0 @@ -package util - -import ( - "errors" - "fmt" - "math" - "regexp" - "strconv" - "time" - - "github.com/spf13/cast" -) - -var durationRE = regexp.MustCompile("^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$") - -func GetDurationFromAge(durationStr string) (time.Duration, error) { - switch durationStr { - case "0": - return 0, nil - case "": - return 0, errors.New("empty duration string") - } - matches := durationRE.FindStringSubmatch(durationStr) - if matches == nil { - return 0, fmt.Errorf("not a valid duration string: %q", durationStr) - } - var dur time.Duration - var overflowErr error - m := func(pos int, mult time.Duration) { - if matches[pos] == "" { - return - } - n, _ := strconv.Atoi(matches[pos]) - if n > int((1<<63-1)/mult/time.Millisecond) { - overflowErr = errors.New("duration out of range") - } - d := time.Duration(n) * time.Millisecond - dur += d * mult - if dur < 0 { - overflowErr = errors.New("duration out of range") - } - } - m(2, 1000*60*60*24*365) // y - m(4, 1000*60*60*24*7) // w - m(6, 1000*60*60*24) // d - m(8, 1000*60*60) // h - m(10, 1000*60) // m - m(12, 1000) // s - m(14, 1) // ms - return dur, overflowErr -} - -func FloatStringToTime(timeFloat string) time.Time { - sec, dec := math.Modf(cast.ToFloat64(timeFloat)) - return time.Unix(int64(sec), int64(dec*(1e9))) -} diff --git a/util/time_test.go b/util/time_test.go deleted file mode 100644 index c3b8c8e..0000000 --- a/util/time_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package util - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func Test_GetDurationFromAge(t *testing.T) { - var d, d2 time.Duration - - d, _ = GetDurationFromAge("2m") - assert.Equal(t, time.Duration(120000000000), d) - - d, _ = GetDurationFromAge("2h") - assert.Equal(t, time.Duration(7200000000000), d) - - d, _ = GetDurationFromAge("2d") - d2, _ = time.ParseDuration("48h") - assert.Equal(t, d2, d) - - d, _ = GetDurationFromAge("100d") - d2, _ = time.ParseDuration("2400h") - assert.Equal(t, d2, d) -} - -func Test_FloatStringToTime(t *testing.T) { - want := time.Date(2015, time.July, 1, 20, 10, 51, 780999898, time.UTC) - got := FloatStringToTime("1435781451.781") - assert.True(t, want.Equal(got)) -}