Merge pull request #75 from intergral/retention_moni

chore(tracing): add spans for retainTenant

CHANGELOG.md

@@ -1,5 +1,6 @@
 <!-- 1.0.3 START -->
 # 1.0.3 (xx/xx/2023)
+- **[CHANGE]**: add/improve monitoring for retention [#75](https://github.com/intergral/deep/pull/75) [@Umaaz](https://github.com/Umaaz)
 - **[CHANGE]**: add docs for run books [#74](https://github.com/intergral/deep/pull/74) [@Umaaz](https://github.com/Umaaz)
 - **[CHANGE]**: unify metric namespaces and subsystems [#73](https://github.com/intergral/deep/pull/73) [@Umaaz](https://github.com/Umaaz)
 - **[CHANGE]**: unify span tags for tenant [#70](https://github.com/intergral/deep/pull/70) [@Umaaz](https://github.com/Umaaz)

pkg/deepdb/retention.go

@@ -21,6 +21,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/opentracing/opentracing-go"
+
 	"github.com/go-kit/log/level"
 
 	"github.com/intergral/deep/pkg/boundedwaitgroup"
@@ -55,6 +57,10 @@ func (rw *readerWriter) doRetention(ctx context.Context) {
 }
 
 func (rw *readerWriter) retainTenant(ctx context.Context, tenantID string) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "rw.retainTenant")
+	span.SetTag("tenantID", tenantID)
+	defer span.Finish()
+
 	start := time.Now()
 	defer func() { metricRetentionDuration.Observe(time.Since(start).Seconds()) }()
 
@@ -101,7 +107,7 @@ func (rw *readerWriter) retainTenant(ctx context.Context, tenantID string) {
 		case <-ctx.Done():
 			return
 		default:
-			level.Info(rw.logger).Log("owns", rw.compactorSharder.Owns(b.BlockID.String()), "blockID", b.BlockID, "tenantID", tenantID)
+			level.Info(rw.logger).Log("owns", rw.compactorSharder.Owns(b.BlockID.String()), "blockID", b.BlockID, "tenantID", tenantID, "cutoff", cutoff, "compacted", b.CompactedTime, "iscut", b.CompactedTime.Before(cutoff))
 			if b.CompactedTime.Before(cutoff) && rw.compactorSharder.Owns(b.BlockID.String()) {
 				level.Info(rw.logger).Log("msg", "deleting block", "blockID", b.BlockID, "tenantID", tenantID)
 				err := rw.c.ClearBlock(b.BlockID, tenantID)