diff --git a/images/agent/src/internal/cache/cache.go b/images/agent/src/internal/cache/cache.go index 7600bc89..2a3c9af0 100644 --- a/images/agent/src/internal/cache/cache.go +++ b/images/agent/src/internal/cache/cache.go @@ -148,6 +148,38 @@ func (c *Cache) FindVG(vgName string) *internal.VGData { return nil } +func (c *Cache) FindThinPoolMappers(thinLvData *LVData) (poolMapper, poolMetadataMapper string, err error) { + if thinLvData.Data.PoolName == "" { + err = fmt.Errorf("pool name is empty") + return + } + + c.m.RLock() + defer c.m.RUnlock() + + poolLv := c.lvs[c.configureLVKey(thinLvData.Data.VGName, thinLvData.Data.PoolName)] + if poolLv == nil { + err = fmt.Errorf("can't find pool %s", thinLvData.Data.PoolName) + return + } + + poolMapper = poolLv.Data.LVDmPath + + if poolLv.Data.MetadataLv == "" { + err = fmt.Errorf("metadata name is empty for pool %s", thinLvData.Data.PoolName) + return + } + + metaLv := c.lvs[c.configureLVKey(thinLvData.Data.VGName, poolLv.Data.MetadataLv)] + if metaLv == nil { + err = fmt.Errorf("can't find metadata %s", poolLv.Data.MetadataLv) + return + } + + poolMetadataMapper = metaLv.Data.LVDmPath + return +} + func (c *Cache) PrintTheCache(log logger.Logger) { log.Trace("*****************CACHE BEGIN*****************") log.Trace("[Devices BEGIN]") diff --git a/images/agent/src/internal/controller/llv/reconciler.go b/images/agent/src/internal/controller/llv/reconciler.go index 5981c248..b9bc0296 100644 --- a/images/agent/src/internal/controller/llv/reconciler.go +++ b/images/agent/src/internal/controller/llv/reconciler.go @@ -595,6 +595,30 @@ func (r *Reconciler) deleteLVIfNeeded(ctx context.Context, vgName string, llv *v r.log.Warning(fmt.Sprintf("[deleteLVIfNeeded] did not find LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) return false, nil } + var poolLv *cache.LVData + var usedRanges *utils.RangeCover + if lv.Data.PoolName != "" { + poolMapper, poolMetadataMapper, err := r.sdsCache.FindThinPoolMappers(lv) + if err != nil { + err = fmt.Errorf("finding mappers for thin pool %s", lv.Data.PoolName) + r.log.Error(err, fmt.Sprintf("[deleteLVIfNeeded] can't find pool for LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) + return true, err + } + + superblock, err := utils.ThinDump(ctx, r.log, poolMapper, poolMetadataMapper) + if err != nil { + err = fmt.Errorf("dumping thin pool map: %w", err) + r.log.Error(err, fmt.Sprintf("[deleteLVIfNeeded] can't find pool map for LV %s in VG %s", llv.Spec.ActualLVNameOnTheNode, vgName)) + return false, err + } + + ranges, err := utils.ThinVolumeUsedRanges(ctx, r.log, superblock, utils.LVMThinDeviceId(lv.Data.ThinId)) + if err != nil { + err = fmt.Errorf("finding used ranges for deviceId %d in thin pool %s", lv.Data.ThinId, lv.Data.PoolName) + return false, err + } + usedRanges = &ranges + } // this case prevents unexpected same-name LV deletions which does not actually belong to our LLV if !checkIfLVBelongsToLLV(llv, &lv.Data) { @@ -648,7 +672,7 @@ func (r *Reconciler) deleteLVIfNeeded(ctx context.Context, vgName string, llv *v } prevFailedMethod = &method r.log.Debug(fmt.Sprintf("[deleteLVIfNeeded] running cleanup for LV %s in VG %s with method %s", lvName, vgName, method)) - err = utils.VolumeCleanup(ctx, r.log, lv, utils.OsDeviceOpener(), vgName, lvName, method) + err = utils.VolumeCleanup(ctx, r.log, utils.OsDeviceOpener(), vgName, lvName, method, usedRanges) if err != nil { r.log.Error(err, fmt.Sprintf("[deleteLVIfNeeded] unable to clean up LV %s in VG %s with method %s", lvName, vgName, method)) return true, err diff --git a/images/agent/src/internal/type.go b/images/agent/src/internal/type.go index e99776d1..51670147 100644 --- a/images/agent/src/internal/type.go +++ b/images/agent/src/internal/type.go @@ -158,6 +158,8 @@ type LVData struct { ConvertLv string `json:"convert_lv"` LvTags string `json:"lv_tags"` ThinId uint64 `json:"thin_id"` + MetadataLv string `json:"metadata_lv"` + LVDmPath string `json:"lv_dm_path"` } func (lv LVData) GetUsedSize() (*resource.Quantity, error) { diff --git a/images/agent/src/internal/utils/commands.go b/images/agent/src/internal/utils/commands.go index 51564953..c4dce849 100644 --- a/images/agent/src/internal/utils/commands.go +++ b/images/agent/src/internal/utils/commands.go @@ -21,7 +21,6 @@ import ( "bytes" "context" "encoding/json" - "encoding/xml" "errors" "fmt" golog "log" @@ -106,7 +105,7 @@ func GetVG(vgName string) (vgData internal.VGData, command string, stdErr bytes. func GetAllLVs(ctx context.Context) (data []internal.LVData, command string, stdErr bytes.Buffer, err error) { var outs bytes.Buffer - args := []string{"lvs", "-o", "+vg_uuid,tags,thin_id", "--units", "B", "--nosuffix", "--reportformat", "json"} + args := []string{"lvs", "-o", "+vg_uuid,tags,thin_id,metadata_lv,lv_dm_path", "--units", "B", "--nosuffix", "--reportformat", "json"} extendedArgs := lvmStaticExtendedArgs(args) cmd := exec.CommandContext(ctx, internal.NSENTERCmd, extendedArgs...) cmd.Stdout = &outs @@ -752,22 +751,3 @@ func ThinDumpRaw(ctx context.Context, log logger.Logger, tpool, tmeta string) (o } return output.Bytes(), nil } - -func ThinDump(ctx context.Context, log logger.Logger, tpool, tmeta string) (superblock Superblock, err error) { - log.Trace(fmt.Sprintf("[ThinDump] calling for tpool %s tmeta %s", tpool, tmeta)) - - var rawOut []byte - rawOut, err = ThinDumpRaw(ctx, log, tpool, tmeta) - if err != nil { - return - } - - log.Debug("[ThinDump] unmarshaling") - if err = xml.Unmarshal(rawOut, &superblock); err != nil { - log.Error(err, "[ThinDump] unmarshaling error") - err = fmt.Errorf("parsing metadata: %w", err) - return - } - - return superblock, nil -} diff --git a/images/agent/src/internal/utils/range_cover.go b/images/agent/src/internal/utils/range_cover.go index 136d56df..886402ba 100644 --- a/images/agent/src/internal/utils/range_cover.go +++ b/images/agent/src/internal/utils/range_cover.go @@ -44,7 +44,7 @@ func (rc RangeCover) Merged() (RangeCover, error) { } last := Range{Start: 0, Count: 0} - reduced := rc[:0] + reduced := make(RangeCover, 0, len(rc)) for _, d := range rc { if last.Count == 0 { last = d diff --git a/images/agent/src/internal/utils/thin_dump.go b/images/agent/src/internal/utils/thin_dump.go index c0d4cf43..08aa8c79 100644 --- a/images/agent/src/internal/utils/thin_dump.go +++ b/images/agent/src/internal/utils/thin_dump.go @@ -17,13 +17,17 @@ limitations under the License. package utils import ( + "agent/internal/logger" + "context" "encoding/xml" + "errors" + "fmt" ) type ( - LVMTime int64 - LVMTransaction int64 - LVMThinDeviceId int64 + LVMTime = int64 + LVMTransaction = int64 + LVMThinDeviceId = int64 ) type Superblock struct { @@ -63,3 +67,49 @@ type SingleMapping struct { DataBlock int64 `xml:"data_block,attr"` Time LVMTime `xml:"time,attr"` } + +func ThinDump(ctx context.Context, log logger.Logger, tpool, tmeta string) (superblock Superblock, err error) { + log.Trace(fmt.Sprintf("[ThinDump] calling for tpool %s tmeta %s", tpool, tmeta)) + + var rawOut []byte + rawOut, err = ThinDumpRaw(ctx, log, tpool, tmeta) + if err != nil { + return + } + + log.Debug("[ThinDump] unmarshaling") + if err = xml.Unmarshal(rawOut, &superblock); err != nil { + log.Error(err, "[ThinDump] unmarshaling error") + err = fmt.Errorf("parsing metadata: %w", err) + return + } + + return superblock, nil +} + +func ThinVolumeUsedRanges(ctx context.Context, log logger.Logger, superblock Superblock, deviceId LVMThinDeviceId) (ranges RangeCover, err error) { + log.Trace(fmt.Sprintf("[ThinVolumeUsedRanges] calling for deviceId %d", deviceId)) + for _, device := range superblock.Devices { + if device.DevId != deviceId { + continue + } + + ranges = make(RangeCover, 0, len(device.RangeMappings)+len(device.SingleMappings)) + + for _, mapping := range device.RangeMappings { + ranges = append(ranges, Range{Start: mapping.DataBegin, Count: mapping.Length}) + } + + for _, mapping := range device.SingleMappings { + ranges = append(ranges, Range{Start: mapping.DataBlock, Count: 1}) + } + + ranges, err := ranges.Merged() + if err != nil { + err = fmt.Errorf("finding used ranges: %w", err) + } + + return ranges.Multiplied(superblock.DataBlockSize), nil + } + return ranges, errors.New("device not found") +} diff --git a/images/agent/src/internal/utils/volume_cleanup_ee.go b/images/agent/src/internal/utils/volume_cleanup_ee.go index 17fe93b4..e4e1cb32 100644 --- a/images/agent/src/internal/utils/volume_cleanup_ee.go +++ b/images/agent/src/internal/utils/volume_cleanup_ee.go @@ -18,11 +18,10 @@ import ( "github.com/deckhouse/sds-node-configurator/lib/go/common/pkg/feature" "golang.org/x/sys/unix" - "agent/internal/cache" "agent/internal/logger" ) -func VolumeCleanup(ctx context.Context, log logger.Logger, lvData *cache.LVData, deviceOpener BlockDeviceOpener, vgName string, lvName, volumeCleanup string) error { +func VolumeCleanup(ctx context.Context, log logger.Logger, deviceOpener BlockDeviceOpener, vgName string, lvName, volumeCleanup string, usedRanges *RangeCover) error { log.Trace(fmt.Sprintf("[VolumeCleanup] cleaning up volume %s in volume group %s using %s", lvName, vgName, volumeCleanup)) if !feature.VolumeCleanupEnabled() { return fmt.Errorf("volume cleanup is not supported in your edition") @@ -35,9 +34,9 @@ func VolumeCleanup(ctx context.Context, log logger.Logger, lvData *cache.LVData, switch volumeCleanup { case "RandomFillSinglePass": - err = volumeCleanupOverwrite(ctx, log, deviceOpener, devicePath, randomSource, 1) + err = volumeCleanupOverwrite(ctx, log, deviceOpener, devicePath, randomSource, 1, usedRanges) case "RandomFillThreePass": - err = volumeCleanupOverwrite(ctx, log, deviceOpener, devicePath, randomSource, 3) + err = volumeCleanupOverwrite(ctx, log, deviceOpener, devicePath, randomSource, 3, usedRanges) case "Discard": err = volumeCleanupDiscard(ctx, log, deviceOpener, devicePath) default: @@ -52,7 +51,7 @@ func VolumeCleanup(ctx context.Context, log logger.Logger, lvData *cache.LVData, return nil } -func volumeCleanupOverwrite(_ context.Context, log logger.Logger, deviceOpener BlockDeviceOpener, devicePath, inputPath string, passes int) (err error) { +func volumeCleanupOverwrite(_ context.Context, log logger.Logger, deviceOpener BlockDeviceOpener, devicePath, inputPath string, passes int, usedRanges *RangeCover) (err error) { log.Trace(fmt.Sprintf("[volumeCleanupOverwrite] overwriting %s by %s in %d passes", devicePath, inputPath, passes)) closeFile := func(file BlockDevice) { log.Trace(fmt.Sprintf("[volumeCleanupOverwrite] closing %s", file.Name())) @@ -77,30 +76,37 @@ func volumeCleanupOverwrite(_ context.Context, log logger.Logger, deviceOpener B } defer closeFile(output) - bytesToWrite, err := output.Size() - if err != nil { - log.Error(err, "[volumeCleanupOverwrite] Finding volume size") - return fmt.Errorf("can't find the size of device %s: %w", devicePath, err) + if usedRanges == nil { + size, err := output.Size() + if err != nil { + log.Error(err, "[volumeCleanupOverwrite] Finding volume size") + return fmt.Errorf("can't find the size of device %s: %w", devicePath, err) + } + + usedRanges = &RangeCover{Range{Start: 0, Count: size}} } bufferSize := 1024 * 1024 * 4 buffer := make([]byte, bufferSize) for pass := 0; pass < passes; pass++ { - log.Debug(fmt.Sprintf("[volumeCleanupOverwrite] Overwriting %d bytes. Pass %d", bytesToWrite, pass)) - start := time.Now() - written, err := io.CopyBuffer( - io.NewOffsetWriter(output, 0), - io.LimitReader(input, bytesToWrite), - buffer) - log.Info(fmt.Sprintf("[volumeCleanupOverwrite] Overwriting is done in %s", time.Since(start).String())) - if err != nil { - log.Error(err, fmt.Sprintf("[volumeCleanupOverwrite] copying from %s to %s", inputPath, devicePath)) - return fmt.Errorf("copying from %s to %s: %w", inputPath, devicePath, err) - } - - if written != bytesToWrite { - log.Error(err, fmt.Sprintf("[volumeCleanupOverwrite] only %d bytes written, expected %d", written, bytesToWrite)) - return fmt.Errorf("only %d bytes written, expected %d", written, bytesToWrite) + for _, usedRange := range *usedRanges { + bytesToWrite := usedRange.Count + log.Debug(fmt.Sprintf("[volumeCleanupOverwrite] Overwriting %d bytes with offset %d. Pass %d", bytesToWrite, usedRange.Start, pass)) + start := time.Now() + written, err := io.CopyBuffer( + io.NewOffsetWriter(output, usedRange.Start), + io.LimitReader(input, bytesToWrite), + buffer) + log.Info(fmt.Sprintf("[volumeCleanupOverwrite] Overwriting is done in %s", time.Since(start).String())) + if err != nil { + log.Error(err, fmt.Sprintf("[volumeCleanupOverwrite] copying from %s to %s", inputPath, devicePath)) + return fmt.Errorf("copying from %s to %s: %w", inputPath, devicePath, err) + } + + if written != bytesToWrite { + log.Error(err, fmt.Sprintf("[volumeCleanupOverwrite] only %d bytes written, expected %d", written, bytesToWrite)) + return fmt.Errorf("only %d bytes written, expected %d", written, bytesToWrite) + } } } diff --git a/images/agent/src/internal/utils/volume_cleanup_ee_test.go b/images/agent/src/internal/utils/volume_cleanup_ee_test.go index 9d9b5682..68eb32b0 100644 --- a/images/agent/src/internal/utils/volume_cleanup_ee_test.go +++ b/images/agent/src/internal/utils/volume_cleanup_ee_test.go @@ -20,7 +20,6 @@ import ( "go.uber.org/mock/gomock" "golang.org/x/sys/unix" - "agent/internal/cache" "agent/internal/logger" . "agent/internal/mock_utils" . "agent/internal/utils" @@ -32,7 +31,7 @@ var _ = Describe("Cleaning up volume", func() { var opener *MockBlockDeviceOpener var device *MockBlockDevice var err error - var lv *cache.LVData + var rangeCover *RangeCover vgName := "vg" lvName := "lv" var method string @@ -46,7 +45,7 @@ var _ = Describe("Cleaning up volume", func() { }) doCall := func() { - err = VolumeCleanup(context.Background(), log, lv, opener, vgName, lvName, method) + err = VolumeCleanup(context.Background(), log, opener, vgName, lvName, method, rangeCover) if !feature.VolumeCleanupEnabled() { Expect(err).To(MatchError("volume cleanup is not supported in your edition")) }