From 4e1f71586e34372e35acccf92df6fcee9fc3403b Mon Sep 17 00:00:00 2001 From: Vladislav Tropnikov Date: Sun, 9 Feb 2025 11:20:13 +0100 Subject: [PATCH] Add support for multiple S3 storage classes in metrics (#24) --- controllers/s3talker.go | 117 ++++++++++++++++++--------- controllers/s3talker_test.go | 34 ++++---- e2e/tests/test_s3_bucket_exporter.py | 35 +++++--- go.mod | 32 ++++---- go.sum | 64 +++++++-------- main.go | 23 +++--- main_test.go | 40 +++++---- 7 files changed, 206 insertions(+), 139 deletions(-) diff --git a/controllers/s3talker.go b/controllers/s3talker.go index 4c0f2f8..8707340 100644 --- a/controllers/s3talker.go +++ b/controllers/s3talker.go @@ -14,12 +14,17 @@ import ( log "github.com/sirupsen/logrus" ) +// StorageClassMetrics +type StorageClassMetrics struct { + Size float64 `json:"size"` + ObjectNumber float64 `json:"objectNumber"` +} + // Bucket - information per bucket type Bucket struct { - BucketName string `json:"bucketName"` - BucketSize float64 `json:"bucketSize"` - BucketObjectNumber float64 `json:"bucketObjectNumber"` - ListDuration time.Duration `json:"listDuration"` + BucketName string `json:"bucketName"` + StorageClasses map[string]StorageClassMetrics `json:"storageClasses"` + ListDuration time.Duration `json:"listDuration"` } // Buckets - list of Bucket objects @@ -27,11 +32,10 @@ type Buckets []Bucket // S3Summary - one JSON struct to rule them all type S3Summary struct { - S3Status bool `json:"s3Status"` - S3Size float64 `json:"s3Size"` - S3ObjectNumber float64 `json:"s3ObjectNumber"` - S3Buckets Buckets `json:"s3Buckets"` - TotalListDuration time.Duration `json:"totalListDuration"` + S3Status bool `json:"s3Status"` + StorageClasses map[string]StorageClassMetrics `json:"storageClasses"` + S3Buckets Buckets `json:"s3Buckets"` + TotalListDuration time.Duration `json:"totalListDuration"` } // S3Conn struct - keeps information about remote S3 @@ -85,6 +89,24 @@ func getS3Client(cfg aws.Config, s3Conn S3Conn) S3ClientInterface { return s3.NewFromConfig(cfg, options) } +// distinct - removes duplicates from a slice of strings +func distinct(input []string) []string { + seen := make(map[string]struct{}) + result := []string{} + + for _, val := range input { + val = strings.TrimSpace(val) + if val != "" { + if _, exists := seen[val]; !exists { + seen[val] = struct{}{} + result = append(result, val) + } + } + } + + return result +} + // S3UsageInfo - gets S3 connection details and returns S3Summary func S3UsageInfo(s3Conn S3Conn, s3BucketNames string) (S3Summary, error) { summary := S3Summary{S3Status: false} @@ -106,7 +128,7 @@ func fetchBucketData(s3BucketNames string, s3Client S3ClientInterface, s3Region if s3BucketNames != "" { // If specific buckets are provided, use them - bucketNames = strings.Split(s3BucketNames, ",") + bucketNames = distinct(strings.Split(s3BucketNames, ",")) } else { // Otherwise, fetch all buckets result, err := s3Client.ListBuckets(context.TODO(), &s3.ListBucketsInput{BucketRegion: aws.String(s3Region)}) @@ -122,10 +144,30 @@ func fetchBucketData(s3BucketNames string, s3Client S3ClientInterface, s3Region log.Debugf("List of buckets in %s region: %v", s3Region, bucketNames) - resultsChan := make(chan Bucket, len(bucketNames)) - errorChan := make(chan error, len(bucketNames)) - var wg sync.WaitGroup + var summaryMutex sync.Mutex + + summaryMutex.Lock() + summary.StorageClasses = make(map[string]StorageClassMetrics) + summary.S3Buckets = make(Buckets, 0, len(bucketNames)) + summaryMutex.Unlock() + + processBucketResult := func(bucket Bucket) { + summaryMutex.Lock() + defer summaryMutex.Unlock() + + summary.S3Buckets = append(summary.S3Buckets, bucket) + for storageClass, metrics := range bucket.StorageClasses { + summaryMetrics := summary.StorageClasses[storageClass] + summaryMetrics.Size += metrics.Size + summaryMetrics.ObjectNumber += metrics.ObjectNumber + summary.StorageClasses[storageClass] = summaryMetrics + } + log.Debugf("Bucket size and objects count: %v", bucket) + } + + var errs []error + var errMutex sync.Mutex for _, bucketName := range bucketNames { bucketName := strings.TrimSpace(bucketName) @@ -137,41 +179,31 @@ func fetchBucketData(s3BucketNames string, s3Client S3ClientInterface, s3Region go func(bucketName string) { defer wg.Done() - size, count, duration, err := calculateBucketMetrics(bucketName, s3Client) + storageClasses, duration, err := calculateBucketMetrics(bucketName, s3Client) if err != nil { - errorChan <- err + errMutex.Lock() + errs = append(errs, err) + errMutex.Unlock() return } - resultsChan <- Bucket{ - BucketName: bucketName, - BucketSize: size, - BucketObjectNumber: count, - ListDuration: duration, + bucket := Bucket{ + BucketName: bucketName, + StorageClasses: storageClasses, + ListDuration: duration, } + + processBucketResult(bucket) log.Debugf("Finish bucket %s processing", bucketName) }(bucketName) } wg.Wait() - close(resultsChan) - close(errorChan) - var errs []error - for err := range errorChan { - errs = append(errs, err) - } if len(errs) > 0 { log.Errorf("Encountered errors while processing buckets: %v", errs) } - for bucket := range resultsChan { - summary.S3Buckets = append(summary.S3Buckets, bucket) - summary.S3Size += bucket.BucketSize - summary.S3ObjectNumber += bucket.BucketObjectNumber - log.Debugf("Bucket size and objects count: %v", bucket) - } - if len(summary.S3Buckets) > 0 { summary.S3Status = true } @@ -181,9 +213,9 @@ func fetchBucketData(s3BucketNames string, s3Client S3ClientInterface, s3Region } // calculateBucketMetrics - computes the total size and object count for a bucket -func calculateBucketMetrics(bucketName string, s3Client S3ClientInterface) (float64, float64, time.Duration, error) { - var totalSize, objectCount float64 +func calculateBucketMetrics(bucketName string, s3Client S3ClientInterface) (map[string]StorageClassMetrics, time.Duration, error) { var continuationToken *string + storageClasses := make(map[string]StorageClassMetrics) start := time.Now() @@ -194,12 +226,19 @@ func calculateBucketMetrics(bucketName string, s3Client S3ClientInterface) (floa }) if err != nil { log.Errorf("Failed to list objects for bucket %s: %v", bucketName, err) - return 0, 0, 0, err + return nil, 0, err } for _, obj := range page.Contents { - totalSize += float64(*obj.Size) - objectCount++ + storageClass := string(obj.StorageClass) + if storageClass == "" { + storageClass = "STANDARD" + } + + metrics := storageClasses[storageClass] + metrics.Size += float64(*obj.Size) + metrics.ObjectNumber++ + storageClasses[storageClass] = metrics } if page.IsTruncated != nil && !*page.IsTruncated { @@ -209,5 +248,5 @@ func calculateBucketMetrics(bucketName string, s3Client S3ClientInterface) (floa } duration := time.Since(start) - return totalSize, objectCount, duration, nil + return storageClasses, duration, nil } diff --git a/controllers/s3talker_test.go b/controllers/s3talker_test.go index bc76fe9..ed92974 100644 --- a/controllers/s3talker_test.go +++ b/controllers/s3talker_test.go @@ -40,8 +40,8 @@ func TestS3UsageInfo_SingleBucket(t *testing.T) { mockClient.On("ListObjectsV2", mock.Anything, mock.Anything, mock.Anything).Return(&s3.ListObjectsV2Output{ Contents: []types.Object{ - {Size: aws.Int64(1024)}, - {Size: aws.Int64(2048)}, + {Size: aws.Int64(1024), StorageClass: "STANDARD"}, + {Size: aws.Int64(2048), StorageClass: "STANDARD"}, }, IsTruncated: aws.Bool(false), }, nil) @@ -50,8 +50,8 @@ func TestS3UsageInfo_SingleBucket(t *testing.T) { assert.NoError(t, err) assert.True(t, summary.S3Status) - assert.Equal(t, float64(3072), summary.S3Size) - assert.Equal(t, float64(2), summary.S3ObjectNumber) + assert.Equal(t, float64(3072), summary.StorageClasses["STANDARD"].Size) + assert.Equal(t, float64(2), summary.StorageClasses["STANDARD"].ObjectNumber) assert.Len(t, summary.S3Buckets, 1) } @@ -78,8 +78,8 @@ func TestS3UsageInfo_MultipleBuckets(t *testing.T) { assert.NoError(t, err) assert.True(t, summary.S3Status) - assert.Equal(t, float64(6144), summary.S3Size) - assert.Equal(t, float64(4), summary.S3ObjectNumber) + assert.Equal(t, float64(6144), summary.StorageClasses["STANDARD"].Size) + assert.Equal(t, float64(4), summary.StorageClasses["STANDARD"].ObjectNumber) assert.Len(t, summary.S3Buckets, 2) } @@ -114,8 +114,8 @@ func TestS3UsageInfo_EmptyBucketList(t *testing.T) { assert.NoError(t, err) assert.True(t, summary.S3Status) - assert.Equal(t, float64(9216), summary.S3Size) - assert.Equal(t, float64(6), summary.S3ObjectNumber) + assert.Equal(t, float64(9216), summary.StorageClasses["STANDARD"].Size) + assert.Equal(t, float64(6), summary.StorageClasses["STANDARD"].ObjectNumber) assert.Len(t, summary.S3Buckets, 3) } @@ -124,18 +124,20 @@ func TestCalculateBucketMetrics(t *testing.T) { mockClient.On("ListObjectsV2", mock.Anything, mock.Anything, mock.Anything).Return(&s3.ListObjectsV2Output{ Contents: []types.Object{ - {Size: aws.Int64(1024)}, - {Size: aws.Int64(2048)}, - {Size: aws.Int64(4096)}, + {Size: aws.Int64(1024), StorageClass: "STANDARD"}, + {Size: aws.Int64(2048), StorageClass: "STANDARD"}, + {Size: aws.Int64(4096), StorageClass: "GLACIER"}, }, IsTruncated: aws.Bool(false), }, nil) - size, count, duration, err := calculateBucketMetrics("bucket1", mockClient) + storageClasses, duration, err := calculateBucketMetrics("bucket1", mockClient) assert.NoError(t, err) - assert.Equal(t, float64(7168), size) - assert.Equal(t, float64(3), count) + assert.Equal(t, float64(3072), storageClasses["STANDARD"].Size) + assert.Equal(t, float64(2), storageClasses["STANDARD"].ObjectNumber) + assert.Equal(t, float64(4096), storageClasses["GLACIER"].Size) + assert.Equal(t, float64(1), storageClasses["GLACIER"].ObjectNumber) assert.Greater(t, duration, time.Duration(0)) } @@ -167,7 +169,7 @@ func TestS3UsageInfo_WithIAMRole(t *testing.T) { assert.NoError(t, err) assert.True(t, summary.S3Status) - assert.Equal(t, float64(100), summary.S3Size) - assert.Equal(t, float64(1), summary.S3ObjectNumber) + assert.Equal(t, float64(100), summary.StorageClasses["STANDARD"].Size) + assert.Equal(t, float64(1), summary.StorageClasses["STANDARD"].ObjectNumber) assert.Len(t, summary.S3Buckets, 1) } diff --git a/e2e/tests/test_s3_bucket_exporter.py b/e2e/tests/test_s3_bucket_exporter.py index 6df2e1d..8caa34b 100644 --- a/e2e/tests/test_s3_bucket_exporter.py +++ b/e2e/tests/test_s3_bucket_exporter.py @@ -85,25 +85,30 @@ def parse_metrics(self, metrics_text): try: if 's3_bucket_object_number' in line: bucket = line.split('bucketName="')[1].split('"')[0] + storage_class = line.split('storageClass="')[1].split('"')[0] count = float(line.split()[-1]) - parsed_metrics.setdefault(bucket, {})["object_count"] = count + parsed_metrics.setdefault(bucket, {}).setdefault("storage_classes", {}).setdefault(storage_class, {})["object_count"] = count elif 's3_bucket_size' in line: bucket = line.split('bucketName="')[1].split('"')[0] + storage_class = line.split('storageClass="')[1].split('"')[0] size = float(line.split()[-1]) - parsed_metrics.setdefault(bucket, {})["total_size"] = size + parsed_metrics.setdefault(bucket, {}).setdefault("storage_classes", {}).setdefault(storage_class, {})["total_size"] = size elif 's3_endpoint_up' in line: endpoint_up = float(line.split()[-1]) parsed_metrics["endpoint_up"] = endpoint_up elif 's3_total_object_number' in line: + storage_class = line.split('storageClass="')[1].split('"')[0] total_objects = float(line.split()[-1]) - parsed_metrics["total_object_number"] = total_objects + parsed_metrics.setdefault("total", {}).setdefault("storage_classes", {}).setdefault(storage_class, {})["object_count"] = total_objects elif 's3_total_size' in line: + storage_class = line.split('storageClass="')[1].split('"')[0] total_size = float(line.split()[-1]) - parsed_metrics["total_size"] = total_size + parsed_metrics.setdefault("total", {}).setdefault("storage_classes", {}).setdefault(storage_class, {})["total_size"] = total_size + except (IndexError, ValueError) as e: logger.warning(f"Error parsing metrics line: {line}. Error: {e}") return parsed_metrics @@ -113,15 +118,18 @@ def verify_bucket_metrics(self, bucket, metadata, bucket_metrics): if bucket not in bucket_metrics: raise AssertionError(f"Metrics for bucket '{bucket}' are missing") + storage_class = "STANDARD" # Assuming STANDARD storage class for test + metrics = bucket_metrics[bucket]["storage_classes"][storage_class] + # Verify object count - actual_count = bucket_metrics[bucket].get("object_count") + actual_count = metrics["object_count"] expected_count = len(metadata["files"]) assert actual_count == expected_count, ( f"Bucket '{bucket}' object count mismatch. Expected: {expected_count}, Got: {actual_count}" ) # Verify total size - actual_size = bucket_metrics[bucket].get("total_size") + actual_size = metrics["total_size"] expected_size = metadata["total_size"] assert abs(actual_size - expected_size) < 10, ( f"Bucket '{bucket}' size mismatch. Expected: {expected_size}, Got: {actual_size}" @@ -131,24 +139,27 @@ def verify_bucket_metrics(self, bucket, metadata, bucket_metrics): def verify_global_metrics(self, bucket_metadata, parsed_metrics): """Verify global metrics (total object count, total size, and endpoint status).""" + storage_class = "STANDARD" # Assuming STANDARD storage class for test + total_metrics = parsed_metrics["total"]["storage_classes"][storage_class] + total_objects_expected = sum(len(metadata["files"]) for metadata in bucket_metadata.values()) total_size_expected = sum(metadata["total_size"] for metadata in bucket_metadata.values()) # Verify total object count - assert parsed_metrics.get("total_object_number") == total_objects_expected, ( + assert total_metrics["object_count"] == total_objects_expected, ( f"Total object count mismatch. Expected: {total_objects_expected}, " - f"Got: {parsed_metrics.get('total_object_number')}" + f"Got: {total_metrics['object_count']}" ) # Verify total size - assert parsed_metrics.get("total_size") == total_size_expected, ( + assert total_metrics["total_size"] == total_size_expected, ( f"Total size mismatch. Expected: {total_size_expected}, " - f"Got: {parsed_metrics.get('total_size')}" + f"Got: {total_metrics['total_size']}" ) # Verify endpoint status - assert parsed_metrics.get("endpoint_up") == 1, ( - f"Endpoint status mismatch. Expected: 1, Got: {parsed_metrics.get('endpoint_up')}" + assert parsed_metrics["endpoint_up"] == 1, ( + f"Endpoint status mismatch. Expected: 1, Got: {parsed_metrics['endpoint_up']}" ) logger.info("Global metrics verified successfully") diff --git a/go.mod b/go.mod index ff147a9..6d1ea9b 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,10 @@ module github.com/tropnikovvl/s3-bucket-exporter go 1.23 require ( - github.com/aws/aws-sdk-go-v2 v1.34.0 - github.com/aws/aws-sdk-go-v2/config v1.29.2 - github.com/aws/aws-sdk-go-v2/credentials v1.17.55 - github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 + github.com/aws/aws-sdk-go-v2 v1.36.1 + github.com/aws/aws-sdk-go-v2/config v1.29.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.59 + github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/sirupsen/logrus v1.9.3 @@ -15,18 +15,18 @@ require ( require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect github.com/aws/smithy-go v1.22.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -39,7 +39,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/stretchr/objx v0.5.2 // indirect - golang.org/x/sys v0.29.0 // indirect - google.golang.org/protobuf v1.36.4 // indirect + golang.org/x/sys v0.30.0 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2df8745..862726c 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,37 @@ -github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU= -github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= +github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= +github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= -github.com/aws/aws-sdk-go-v2/config v1.29.2 h1:JuIxOEPcSKpMB0J+khMjznG9LIhIBdmqNiEcPclnwqc= -github.com/aws/aws-sdk-go-v2/config v1.29.2/go.mod h1:HktTHregOZwNSM/e7WTfVSu9RCX+3eOv+6ij27PtaYs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.55 h1:CDhKnDEaGkLA5ZszV/qw5uwN5M8rbv9Cl0JRN+PRsaM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.55/go.mod h1:kPD/vj+RB5MREDUky376+zdnjZpR+WgdBBvwrmnlmKE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 h1:kU7tmXNaJ07LsyN3BUgGqAmVmQtq0w6duVIHAKfp0/w= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25/go.mod h1:OiC8+OiqrURb1wrwmr/UbOVLFSWEGxjinj5C299VQdo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI= +github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= +github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 h1:kznaW4f81mNMlREkU9w3jUuJvU5g/KsqDV43ab7Rp6s= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.12/go.mod h1:bZy9r8e0/s0P7BSDHgMLXK2KvdyRRBIQ2blKlvLt0IU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 h1:mUwIpAvILeKFnRx4h1dEgGEFGuV8KJ3pEScZWVFYuZA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11/go.mod h1:JDJtD+b8HNVv71axz8+S5492KM8wTzHRFpMKQbPlYxw= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 h1:g9d+TOsu3ac7SgmY2dUf1qMgu/uJVTlQ4VCbH6hRxSw= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.10/go.mod h1:WZfNmntu92HO44MVZAubQaz3qCuIdeOdog2sADfU6hU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 h1:cCBJaT7EeEojpJ4s7wTDbhZlHVJOgNHN7iw6qVurGaw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 h1:ehvUZNVrGA1Usa6yYo8A8pUqrigRelWXSbcCqYpRLeI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0/go.mod h1:KuLNrwYJFaC2AVZ+CVVc12k9NyqwgWsoNNHjwqF6QNk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -75,10 +75,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= -google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/main.go b/main.go index 8e99e7b..176b84d 100644 --- a/main.go +++ b/main.go @@ -93,20 +93,25 @@ func (c S3Collector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(up, prometheus.GaugeValue, float64(s3Status), s3Endpoint, s3Region) log.Debugf("Cached S3 metrics %s: %+v", s3Endpoint, metrics) - descS := prometheus.NewDesc("s3_total_size", "S3 Total Bucket Size", []string{"s3Endpoint", "s3Region"}, nil) - descON := prometheus.NewDesc("s3_total_object_number", "S3 Total Object Number", []string{"s3Endpoint", "s3Region"}, nil) + descS := prometheus.NewDesc("s3_total_size", "S3 Total Bucket Size", []string{"s3Endpoint", "s3Region", "storageClass"}, nil) + descON := prometheus.NewDesc("s3_total_object_number", "S3 Total Object Number", []string{"s3Endpoint", "s3Region", "storageClass"}, nil) descDS := prometheus.NewDesc("s3_list_total_duration_seconds", "Total time spent listing objects across all buckets", []string{"s3Endpoint", "s3Region"}, nil) - ch <- prometheus.MustNewConstMetric(descS, prometheus.GaugeValue, float64(metrics.S3Size), s3Endpoint, s3Region) - ch <- prometheus.MustNewConstMetric(descON, prometheus.GaugeValue, float64(metrics.S3ObjectNumber), s3Endpoint, s3Region) + + for class, s3Metrics := range metrics.StorageClasses { + ch <- prometheus.MustNewConstMetric(descS, prometheus.GaugeValue, s3Metrics.Size, s3Endpoint, s3Region, class) + ch <- prometheus.MustNewConstMetric(descON, prometheus.GaugeValue, s3Metrics.ObjectNumber, s3Endpoint, s3Region, class) + } ch <- prometheus.MustNewConstMetric(descDS, prometheus.GaugeValue, float64(metrics.TotalListDuration.Seconds()), s3Endpoint, s3Region) + descBucketS := prometheus.NewDesc("s3_bucket_size", "S3 Bucket Size", []string{"s3Endpoint", "s3Region", "bucketName", "storageClass"}, nil) + descBucketON := prometheus.NewDesc("s3_bucket_object_number", "S3 Bucket Object Number", []string{"s3Endpoint", "s3Region", "bucketName", "storageClass"}, nil) + for _, bucket := range metrics.S3Buckets { - descBucketS := prometheus.NewDesc("s3_bucket_size", "S3 Bucket Size", []string{"s3Endpoint", "s3Region", "bucketName"}, nil) - descBucketON := prometheus.NewDesc("s3_bucket_object_number", "S3 Bucket Object Number", []string{"s3Endpoint", "s3Region", "bucketName"}, nil) + for class, s3Metrics := range bucket.StorageClasses { + ch <- prometheus.MustNewConstMetric(descBucketS, prometheus.GaugeValue, s3Metrics.Size, s3Endpoint, s3Region, bucket.BucketName, class) + ch <- prometheus.MustNewConstMetric(descBucketON, prometheus.GaugeValue, s3Metrics.ObjectNumber, s3Endpoint, s3Region, bucket.BucketName, class) + } descBucketDS := prometheus.NewDesc("s3_list_duration_seconds", "Time spent listing objects in bucket", []string{"s3Endpoint", "s3Region", "bucketName"}, nil) - - ch <- prometheus.MustNewConstMetric(descBucketS, prometheus.GaugeValue, float64(bucket.BucketSize), s3Endpoint, s3Region, bucket.BucketName) - ch <- prometheus.MustNewConstMetric(descBucketON, prometheus.GaugeValue, float64(bucket.BucketObjectNumber), s3Endpoint, s3Region, bucket.BucketName) ch <- prometheus.MustNewConstMetric(descBucketDS, prometheus.GaugeValue, float64(bucket.ListDuration.Seconds()), s3Endpoint, s3Region, bucket.BucketName) } } diff --git a/main_test.go b/main_test.go index 10a9e31..542949b 100644 --- a/main_test.go +++ b/main_test.go @@ -169,16 +169,24 @@ func TestS3Collector(t *testing.T) { metricsMutex.Lock() cachedMetrics = controllers.S3Summary{ - S3Status: true, - S3Size: 1000.0, - S3ObjectNumber: 50.0, + S3Status: true, + StorageClasses: map[string]controllers.StorageClassMetrics{ + "STANDARD": { + Size: 1000.0, + ObjectNumber: 50.0, + }, + }, TotalListDuration: 2 * time.Second, S3Buckets: []controllers.Bucket{ { - BucketName: "test-bucket", - BucketSize: 500.0, - BucketObjectNumber: 25.0, - ListDuration: 1 * time.Second, + BucketName: "test-bucket", + StorageClasses: map[string]controllers.StorageClassMetrics{ + "STANDARD": { + Size: 500.0, + ObjectNumber: 25.0, + }, + }, + ListDuration: 1 * time.Second, }, }, } @@ -198,10 +206,10 @@ func TestS3Collector(t *testing.T) { value float64 }{ {"s3_endpoint_up", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region}, 1.0}, - {"s3_total_size", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region}, 1000.0}, - {"s3_total_object_number", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region}, 50.0}, - {"s3_bucket_size", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region, "bucketName": "test-bucket"}, 500.0}, - {"s3_bucket_object_number", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region, "bucketName": "test-bucket"}, 25.0}, + {"s3_total_size", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region, "storageClass": "STANDARD"}, 1000.0}, + {"s3_total_object_number", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region, "storageClass": "STANDARD"}, 50.0}, + {"s3_bucket_size", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region, "bucketName": "test-bucket", "storageClass": "STANDARD"}, 500.0}, + {"s3_bucket_object_number", map[string]string{"s3Endpoint": s3Endpoint, "s3Region": s3Region, "bucketName": "test-bucket", "storageClass": "STANDARD"}, 25.0}, } expectedDuration := []struct { @@ -298,14 +306,16 @@ func TestUpdateMetrics(t *testing.T) { assert.NoError(t, cachedError, "Expected no error with mock client") assert.Equal(t, true, cachedMetrics.S3Status, "S3Status should be true") - assert.Equal(t, 1024.0, cachedMetrics.S3Size, "S3Size should match") - assert.Equal(t, 1.0, cachedMetrics.S3ObjectNumber, "S3ObjectNumber should match") + metrics := cachedMetrics.StorageClasses["STANDARD"] + assert.Equal(t, 1024.0, metrics.Size, "Total size should match") + assert.Equal(t, 1.0, metrics.ObjectNumber, "Total object number should match") require.Len(t, cachedMetrics.S3Buckets, 1, "Should have exactly one bucket") bucket := cachedMetrics.S3Buckets[0] assert.Equal(t, "test-bucket", bucket.BucketName, "BucketName should match") - assert.Equal(t, 1024.0, bucket.BucketSize, "BucketSize should match") - assert.Equal(t, 1.0, bucket.BucketObjectNumber, "BucketObjectNumber should match") + bucketMetrics := bucket.StorageClasses["STANDARD"] + assert.Equal(t, 1024.0, bucketMetrics.Size, "Bucket size should match") + assert.Equal(t, 1.0, bucketMetrics.ObjectNumber, "Bucket object number should match") assert.Greater(t, cachedMetrics.TotalListDuration, time.Duration(0), "TotalListDuration should be positive") assert.Greater(t, bucket.ListDuration, time.Duration(0), "Bucket ListDuration should be positive") }