From ebe240f01d2c437430d3d292cbde9450d0f27072 Mon Sep 17 00:00:00 2001 From: Ben Donnelly Date: Thu, 30 Nov 2023 14:46:09 +0000 Subject: [PATCH 1/3] chore(tracing): add spans for retainTenant - improve log message in retention --- pkg/deepdb/retention.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/deepdb/retention.go b/pkg/deepdb/retention.go index fd7742e..4d0c739 100644 --- a/pkg/deepdb/retention.go +++ b/pkg/deepdb/retention.go @@ -19,6 +19,7 @@ package deepdb import ( "context" + "github.com/opentracing/opentracing-go" "time" "github.com/go-kit/log/level" @@ -55,6 +56,10 @@ func (rw *readerWriter) doRetention(ctx context.Context) { } func (rw *readerWriter) retainTenant(ctx context.Context, tenantID string) { + span, ctx := opentracing.StartSpanFromContext(ctx, "rw.retainTenant") + span.SetTag("tenantID", tenantID) + defer span.Finish() + start := time.Now() defer func() { metricRetentionDuration.Observe(time.Since(start).Seconds()) }() @@ -101,7 +106,7 @@ func (rw *readerWriter) retainTenant(ctx context.Context, tenantID string) { case <-ctx.Done(): return default: - level.Info(rw.logger).Log("owns", rw.compactorSharder.Owns(b.BlockID.String()), "blockID", b.BlockID, "tenantID", tenantID) + level.Info(rw.logger).Log("owns", rw.compactorSharder.Owns(b.BlockID.String()), "blockID", b.BlockID, "tenantID", tenantID, "cutoff", cutoff, "compacted", b.CompactedTime, "iscut", b.CompactedTime.Before(cutoff)) if b.CompactedTime.Before(cutoff) && rw.compactorSharder.Owns(b.BlockID.String()) { level.Info(rw.logger).Log("msg", "deleting block", "blockID", b.BlockID, "tenantID", tenantID) err := rw.c.ClearBlock(b.BlockID, tenantID) From 419d76b4d280279ffda17f7bb0c4d68eb98f0128 Mon Sep 17 00:00:00 2001 From: Ben Donnelly Date: Thu, 30 Nov 2023 14:47:01 +0000 Subject: [PATCH 2/3] chore(tracing): add spans for retainTenant - improve log message in retention --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c174959..2de4b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # 1.0.3 (xx/xx/2023) +- **[CHANGE]**: add/improve monitoring for retention [#75](https://github.com/intergral/deep/pull/75) [@Umaaz](https://github.com/Umaaz) - **[CHANGE]**: add docs for run books [#74](https://github.com/intergral/deep/pull/74) [@Umaaz](https://github.com/Umaaz) - **[CHANGE]**: unify metric namespaces and subsystems [#73](https://github.com/intergral/deep/pull/73) [@Umaaz](https://github.com/Umaaz) - **[CHANGE]**: unify span tags for tenant [#70](https://github.com/intergral/deep/pull/70) [@Umaaz](https://github.com/Umaaz) From 2a7ef3d9705c216633b487c4c9ce940bb2df6443 Mon Sep 17 00:00:00 2001 From: Ben Donnelly Date: Thu, 30 Nov 2023 15:51:27 +0000 Subject: [PATCH 3/3] chore(format): fix formatting --- pkg/deepdb/retention.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/deepdb/retention.go b/pkg/deepdb/retention.go index 4d0c739..8a3bc46 100644 --- a/pkg/deepdb/retention.go +++ b/pkg/deepdb/retention.go @@ -19,9 +19,10 @@ package deepdb import ( "context" - "github.com/opentracing/opentracing-go" "time" + "github.com/opentracing/opentracing-go" + "github.com/go-kit/log/level" "github.com/intergral/deep/pkg/boundedwaitgroup"