Skip to content

Commit 2dbbfb8

Browse files
benmcclellandbenmcclelland
authored
Merge pull request #1090 from versity/feat/versioning-checksums
feat: Implements checksums for ListObjectVersions and CopyObject acti…
2 parents 37a1412 + fcafb57 commit 2dbbfb8

File tree

4 files changed

+149
-28
lines changed

4 files changed

+149
-28
lines changed

‎backend/posix/posix.go

+48-14
Original file line numberDiff line numberDiff line change
@@ -915,14 +915,22 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
915915
Key: &path,
916916
})
917917
} else {
918+
// Retreive checksum
919+
checksum, err := p.retrieveChecksums(nil, bucket, path)
920+
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
921+
return nil, fmt.Errorf("get checksum: %w", err)
922+
}
923+
918924
objects = append(objects, types.ObjectVersion{
919-
ETag: &etag,
920-
Key: &path,
921-
LastModified: backend.GetTimePtr(fi.ModTime()),
922-
Size: &size,
923-
VersionId: &versionId,
924-
IsLatest: getBoolPtr(true),
925-
StorageClass: types.ObjectVersionStorageClassStandard,
925+
ETag: &etag,
926+
Key: &path,
927+
LastModified: backend.GetTimePtr(fi.ModTime()),
928+
Size: &size,
929+
VersionId: &versionId,
930+
IsLatest: getBoolPtr(true),
931+
StorageClass: types.ObjectVersionStorageClassStandard,
932+
ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm},
933+
ChecksumType: checksum.Type,
926934
})
927935
}
928936

@@ -998,6 +1006,12 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
9981006
// so this will just set etag to "" if its not already set
9991007
etag := string(etagBytes)
10001008
size := nf.Size()
1009+
// Retreive checksum
1010+
checksum, err := p.retrieveChecksums(nil, versionPath, nullVersionId)
1011+
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
1012+
return nil, fmt.Errorf("get checksum: %w", err)
1013+
}
1014+
10011015
nullVersionIdObj = &types.ObjectVersion{
10021016
ETag: &etag,
10031017
Key: &path,
@@ -1006,6 +1020,10 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
10061020
VersionId: backend.GetPtrFromString("null"),
10071021
IsLatest: getBoolPtr(false),
10081022
StorageClass: types.ObjectVersionStorageClassStandard,
1023+
ChecksumAlgorithm: []types.ChecksumAlgorithm{
1024+
checksum.Algorithm,
1025+
},
1026+
ChecksumType: checksum.Type,
10091027
}
10101028
}
10111029
}
@@ -1110,14 +1128,21 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
11101128
IsLatest: getBoolPtr(false),
11111129
})
11121130
} else {
1131+
// Retreive checksum
1132+
checksum, err := p.retrieveChecksums(nil, versionPath, versionId)
1133+
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
1134+
return nil, fmt.Errorf("get checksum: %w", err)
1135+
}
11131136
objects = append(objects, types.ObjectVersion{
1114-
ETag: &etag,
1115-
Key: &path,
1116-
LastModified: backend.GetTimePtr(f.ModTime()),
1117-
Size: &size,
1118-
VersionId: &versionId,
1119-
IsLatest: getBoolPtr(false),
1120-
StorageClass: types.ObjectVersionStorageClassStandard,
1137+
ETag: &etag,
1138+
Key: &path,
1139+
LastModified: backend.GetTimePtr(f.ModTime()),
1140+
Size: &size,
1141+
VersionId: &versionId,
1142+
IsLatest: getBoolPtr(false),
1143+
StorageClass: types.ObjectVersionStorageClassStandard,
1144+
ChecksumAlgorithm: []types.ChecksumAlgorithm{checksum.Algorithm},
1145+
ChecksumType: checksum.Type,
11211146
})
11221147
}
11231148

@@ -3855,6 +3880,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
38553880
var sha1 *string
38563881
var sha256 *string
38573882
var crc64nvme *string
3883+
var chType types.ChecksumType
38583884

38593885
dstObjdPath := filepath.Join(dstBucket, dstObject)
38603886
if dstObjdPath == objPath {
@@ -3882,6 +3908,8 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
38823908
return nil, fmt.Errorf("get obj checksums: %w", err)
38833909
}
38843910

3911+
chType = checksums.Type
3912+
38853913
if input.ChecksumAlgorithm != "" {
38863914
// If a different checksum algorith is specified
38873915
// first caclculate and store the checksum
@@ -3923,6 +3951,10 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
39233951
crc64nvme = &sum
39243952
}
39253953

3954+
// If a new checksum is calculated, the checksum type
3955+
// should be FULL_OBJECT
3956+
chType = types.ChecksumTypeFullObject
3957+
39263958
err = p.storeChecksums(f, dstBucket, dstObject, checksums)
39273959
if err != nil {
39283960
return nil, fmt.Errorf("store checksum: %w", err)
@@ -3970,6 +4002,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
39704002
sha1 = res.ChecksumSHA1
39714003
sha256 = res.ChecksumSHA256
39724004
crc64nvme = res.ChecksumCRC64NVME
4005+
chType = res.ChecksumType
39734006
}
39744007

39754008
fi, err = os.Stat(dstObjdPath)
@@ -3986,6 +4019,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3.
39864019
ChecksumSHA1: sha1,
39874020
ChecksumSHA256: sha256,
39884021
ChecksumCRC64NVME: crc64nvme,
4022+
ChecksumType: chType,
39894023
},
39904024
VersionId: version,
39914025
CopySourceVersionId: &srcVersionId,

‎tests/integration/group-tests.go

+2
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,7 @@ func TestVersioning(s *S3Conf) {
703703
ListObjectVersions_with_delete_markers(s)
704704
ListObjectVersions_containing_null_versionId_obj(s)
705705
ListObjectVersions_single_null_versionId_object(s)
706+
ListObjectVersions_checksum(s)
706707
// Multipart upload
707708
Versioning_Multipart_Upload_success(s)
708709
Versioning_Multipart_Upload_overwrite_an_object(s)
@@ -1162,6 +1163,7 @@ func GetIntTests() IntTests {
11621163
"ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers,
11631164
"ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj,
11641165
"ListObjectVersions_single_null_versionId_object": ListObjectVersions_single_null_versionId_object,
1166+
"ListObjectVersions_checksum": ListObjectVersions_checksum,
11651167
"Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success,
11661168
"Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object,
11671169
"Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId,

‎tests/integration/tests.go

+42-10
Original file line numberDiff line numberDiff line change
@@ -5281,7 +5281,7 @@ func ListObjectVersions_VD_success(s *S3Conf) error {
52815281
return err
52825282
}
52835283

5284-
if !compareVersions(res.Versions, versions) {
5284+
if !compareVersions(versions, res.Versions) {
52855285
return fmt.Errorf("expected object versions output to be %v, instead got %v", versions, res.Versions)
52865286
}
52875287
return nil
@@ -14041,6 +14041,7 @@ func Versioning_PutObject_overwrite_null_versionId_obj(s *S3Conf) error {
1404114041
Size: &lgth,
1404214042
VersionId: &nullVersionId,
1404314043
StorageClass: types.ObjectVersionStorageClassStandard,
14044+
ChecksumType: out.res.ChecksumType,
1404414045
},
1404514046
}, versions...)
1404614047

@@ -14134,6 +14135,7 @@ func Versioning_CopyObject_success(s *S3Conf) error {
1413414135
Size: &srcObjLen,
1413514136
VersionId: out.VersionId,
1413614137
StorageClass: types.ObjectVersionStorageClassStandard,
14138+
ChecksumType: out.CopyObjectResult.ChecksumType,
1413714139
},
1413814140
}, dstObjVersions...)
1413914141

@@ -15055,7 +15057,7 @@ func Versioning_DeleteObject_suspended(s *S3Conf) error {
1505515057
},
1505615058
}
1505715059

15058-
if !compareVersions(res.Versions, versions) {
15060+
if !compareVersions(versions, res.Versions) {
1505915061
return fmt.Errorf("expected the versions to be %v, instead got %v", versions, res.Versions)
1506015062
}
1506115063
if !compareDelMarkers(res.DeleteMarkers, delMarkers) {
@@ -15298,7 +15300,7 @@ func ListObjectVersions_list_single_object_versions(s *S3Conf) error {
1529815300
return err
1529915301
}
1530015302

15301-
if !compareVersions(out.Versions, versions) {
15303+
if !compareVersions(versions, out.Versions) {
1530215304
return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions)
1530315305
}
1530415306

@@ -15335,7 +15337,7 @@ func ListObjectVersions_list_multiple_object_versions(s *S3Conf) error {
1533515337
return err
1533615338
}
1533715339

15338-
if !compareVersions(out.Versions, versions) {
15340+
if !compareVersions(versions, out.Versions) {
1533915341
return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, out.Versions)
1534015342
}
1534115343

@@ -15390,7 +15392,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error {
1539015392
return fmt.Errorf("expected the NextVersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.NextVersionIdMarker)
1539115393
}
1539215394

15393-
if !compareVersions(out.Versions, versions[:maxKeys]) {
15395+
if !compareVersions(versions[:maxKeys], out.Versions) {
1539415396
return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[:maxKeys], out.Versions)
1539515397
}
1539615398

@@ -15418,7 +15420,7 @@ func ListObjectVersions_multiple_object_versions_truncated(s *S3Conf) error {
1541815420
return fmt.Errorf("expected the VersionIdMarker to be %v, instead got %v", *versions[maxKeys].VersionId, *out.VersionIdMarker)
1541915421
}
1542015422

15421-
if !compareVersions(out.Versions, versions[maxKeys:]) {
15423+
if !compareVersions(versions[maxKeys:], out.Versions) {
1542215424
return fmt.Errorf("expected the resulting object versions to be %v, instead got %v", versions[maxKeys:], out.Versions)
1542315425
}
1542415426

@@ -15463,7 +15465,7 @@ func ListObjectVersions_with_delete_markers(s *S3Conf) error {
1546315465
return err
1546415466
}
1546515467

15466-
if !compareVersions(res.Versions, versions) {
15468+
if !compareVersions(versions, res.Versions) {
1546715469
return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, res.Versions)
1546815470
}
1546915471
if !compareDelMarkers(res.DeleteMarkers, delMarkers) {
@@ -15535,7 +15537,7 @@ func ListObjectVersions_containing_null_versionId_obj(s *S3Conf) error {
1553515537
return err
1553615538
}
1553715539

15538-
if !compareVersions(res.Versions, versions) {
15540+
if !compareVersions(versions, res.Versions) {
1553915541
return fmt.Errorf("expected the listed object versions to be %v, instead got %v", versions, res.Versions)
1554015542
}
1554115543

@@ -15600,14 +15602,44 @@ func ListObjectVersions_single_null_versionId_object(s *S3Conf) error {
1560015602
if !compareDelMarkers(resp.DeleteMarkers, delMarkers) {
1560115603
return fmt.Errorf("expected the delete markers list to be %v, instaed got %v", delMarkers, resp.DeleteMarkers)
1560215604
}
15603-
if !compareVersions(resp.Versions, versions) {
15605+
if !compareVersions(versions, resp.Versions) {
1560415606
return fmt.Errorf("expected the object versions list to be %v, instead got %v", versions, resp.Versions)
1560515607
}
1560615608

1560715609
return nil
1560815610
})
1560915611
}
1561015612

15613+
func ListObjectVersions_checksum(s *S3Conf) error {
15614+
testName := "ListObjectVersions_checksum"
15615+
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
15616+
versions := []types.ObjectVersion{}
15617+
for i, algo := range types.ChecksumAlgorithmCrc32.Values() {
15618+
vers, err := createObjVersions(s3client, bucket, fmt.Sprintf("obj-%v", i), 1, withChecksumAlgo(algo))
15619+
if err != nil {
15620+
return err
15621+
}
15622+
15623+
versions = append(versions, vers...)
15624+
}
15625+
15626+
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
15627+
res, err := s3client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
15628+
Bucket: &bucket,
15629+
})
15630+
cancel()
15631+
if err != nil {
15632+
return err
15633+
}
15634+
15635+
if !compareVersions(versions, res.Versions) {
15636+
return fmt.Errorf("expected the versions to be %+v, instead got %+v", versions, res.Versions)
15637+
}
15638+
15639+
return nil
15640+
}, withVersioning(types.BucketVersioningStatusEnabled))
15641+
}
15642+
1561115643
func Versioning_Multipart_Upload_success(s *S3Conf) error {
1561215644
testName := "Versioning_Multipart_Upload_success"
1561315645
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
@@ -15761,7 +15793,7 @@ func Versioning_Multipart_Upload_overwrite_an_object(s *S3Conf) error {
1576115793
},
1576215794
}, objVersions...)
1576315795

15764-
if !compareVersions(resp.Versions, versions) {
15796+
if !compareVersions(versions, resp.Versions) {
1576515797
return fmt.Errorf("expected the resulting versions to be %v, instead got %v", versions, resp.Versions)
1576615798
}
1576715799

‎tests/integration/utils.go

+57-4
Original file line numberDiff line numberDiff line change
@@ -1112,7 +1112,22 @@ func pfxStrings(pfxs []types.CommonPrefix) []string {
11121112
return pfxStrs
11131113
}
11141114

1115-
func createObjVersions(client *s3.Client, bucket, object string, count int) ([]types.ObjectVersion, error) {
1115+
type versCfg struct {
1116+
checksumAlgorithm types.ChecksumAlgorithm
1117+
}
1118+
1119+
type versOpt func(*versCfg)
1120+
1121+
func withChecksumAlgo(algo types.ChecksumAlgorithm) versOpt {
1122+
return func(vc *versCfg) { vc.checksumAlgorithm = algo }
1123+
}
1124+
1125+
func createObjVersions(client *s3.Client, bucket, object string, count int, opts ...versOpt) ([]types.ObjectVersion, error) {
1126+
cfg := new(versCfg)
1127+
for _, o := range opts {
1128+
o(cfg)
1129+
}
1130+
11161131
versions := []types.ObjectVersion{}
11171132
for i := 0; i < count; i++ {
11181133
rNumber, err := rand.Int(rand.Reader, big.NewInt(100000))
@@ -1130,15 +1145,40 @@ func createObjVersions(client *s3.Client, bucket, object string, count int) ([]t
11301145
}
11311146

11321147
isLatest := i == count-1
1133-
1134-
versions = append(versions, types.ObjectVersion{
1148+
version := types.ObjectVersion{
11351149
ETag: r.res.ETag,
11361150
IsLatest: &isLatest,
11371151
Key: &object,
11381152
Size: &dataLength,
11391153
VersionId: r.res.VersionId,
11401154
StorageClass: types.ObjectVersionStorageClassStandard,
1141-
})
1155+
ChecksumType: r.res.ChecksumType,
1156+
}
1157+
1158+
switch {
1159+
case r.res.ChecksumCRC32 != nil:
1160+
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
1161+
types.ChecksumAlgorithmCrc32,
1162+
}
1163+
case r.res.ChecksumCRC32C != nil:
1164+
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
1165+
types.ChecksumAlgorithmCrc32c,
1166+
}
1167+
case r.res.ChecksumCRC64NVME != nil:
1168+
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
1169+
types.ChecksumAlgorithmCrc64nvme,
1170+
}
1171+
case r.res.ChecksumSHA1 != nil:
1172+
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
1173+
types.ChecksumAlgorithmSha1,
1174+
}
1175+
case r.res.ChecksumSHA256 != nil:
1176+
version.ChecksumAlgorithm = []types.ChecksumAlgorithm{
1177+
types.ChecksumAlgorithmSha256,
1178+
}
1179+
}
1180+
1181+
versions = append(versions, version)
11421182
}
11431183

11441184
versions = reverseSlice(versions)
@@ -1198,6 +1238,19 @@ func compareVersions(v1, v2 []types.ObjectVersion) bool {
11981238
if version.StorageClass != v2[i].StorageClass {
11991239
return false
12001240
}
1241+
if version.ChecksumType != "" {
1242+
if version.ChecksumType != v2[i].ChecksumType {
1243+
return false
1244+
}
1245+
}
1246+
if len(version.ChecksumAlgorithm) != 0 {
1247+
if len(v2[i].ChecksumAlgorithm) == 0 {
1248+
return false
1249+
}
1250+
if version.ChecksumAlgorithm[0] != v2[i].ChecksumAlgorithm[0] {
1251+
return false
1252+
}
1253+
}
12011254
}
12021255

12031256
return true

0 commit comments

Comments
 (0)