From 05881f103ebd4a5dc847d3a5dda6f137910fc4a9 Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Fri, 17 Jan 2025 13:34:54 -0500 Subject: [PATCH] fix: enable disabled s3 e2e tests & update tests after default checksums (#1505) --- services/s3/e2eTest/src/PaginatorTest.kt | 4 --- services/s3/e2eTest/src/S3ChecksumTest.kt | 7 ++-- services/s3/e2eTest/src/S3ExpressTest.kt | 13 ++----- services/s3/e2eTest/src/S3IntegrationTest.kt | 3 -- services/s3/e2eTest/src/S3TestUtils.kt | 36 +------------------- 5 files changed, 6 insertions(+), 57 deletions(-) diff --git a/services/s3/e2eTest/src/PaginatorTest.kt b/services/s3/e2eTest/src/PaginatorTest.kt index f35ea41fff9..865f0633b6f 100644 --- a/services/s3/e2eTest/src/PaginatorTest.kt +++ b/services/s3/e2eTest/src/PaginatorTest.kt @@ -19,7 +19,6 @@ import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Test import org.junit.jupiter.api.TestInstance -import kotlin.test.Ignore import kotlin.test.assertContentEquals import kotlin.time.Duration.Companion.seconds @@ -41,10 +40,7 @@ class PaginatorTest { S3TestUtils.deleteBucketAndAllContents(client, testBucket) } - // FIXME: Enable test - // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 // ListParts has a strange pagination termination condition via [IsTerminated]. Verify it actually works correctly. - @Ignore @Test fun testListPartsPagination() = runBlocking { val chunk = "!".repeat(5 * 1024 * 1024).encodeToByteArray() // Parts must be at least 5MB diff --git a/services/s3/e2eTest/src/S3ChecksumTest.kt b/services/s3/e2eTest/src/S3ChecksumTest.kt index 48f6894cc64..cf0df9ba826 100644 --- a/services/s3/e2eTest/src/S3ChecksumTest.kt +++ b/services/s3/e2eTest/src/S3ChecksumTest.kt @@ -3,7 +3,7 @@ package aws.sdk.kotlin.e2etest import aws.sdk.kotlin.e2etest.S3TestUtils.deleteBucketContents import aws.sdk.kotlin.e2etest.S3TestUtils.deleteMultiPartUploads import aws.sdk.kotlin.e2etest.S3TestUtils.getAccountId -import aws.sdk.kotlin.e2etest.S3TestUtils.getBucketByName +import aws.sdk.kotlin.e2etest.S3TestUtils.getTestBucket import aws.sdk.kotlin.e2etest.S3TestUtils.responseCodeFromPut import aws.sdk.kotlin.services.s3.* import aws.sdk.kotlin.services.s3.model.* @@ -24,14 +24,13 @@ import kotlin.time.Duration.Companion.seconds @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3ChecksumTest { private val client = S3Client { region = "us-west-2" } - private val testBucket = "s3-test-bucket-ci-motorcade" + private lateinit var testBucket: String private fun testKey(): String = "test-object" + UUID.randomUUID() @BeforeAll private fun setUp(): Unit = runBlocking { val accountId = getAccountId() - // FIXME: Use randomly generated bucket instead of hardcoded one - getBucketByName(client, testBucket, "us-west-2", accountId) + testBucket = getTestBucket(client, "us-west-2", accountId) } @AfterAll diff --git a/services/s3/e2eTest/src/S3ExpressTest.kt b/services/s3/e2eTest/src/S3ExpressTest.kt index f831583b38c..690d44e42ae 100644 --- a/services/s3/e2eTest/src/S3ExpressTest.kt +++ b/services/s3/e2eTest/src/S3ExpressTest.kt @@ -136,7 +136,7 @@ class S3ExpressTest { } @Test - fun testUploadPartContainsNoDefaultChecksum() = runTest { + fun testUploadPartContainsCRC32Checksum() = runTest { val testBucket = testBuckets.first() val testObject = "I-will-be-uploaded-in-parts-!" @@ -153,7 +153,7 @@ class S3ExpressTest { var eTagPartTwo: String? client.withConfig { - interceptors += NoChecksumValidatingInterceptor() + interceptors += CRC32ChecksumValidatingInterceptor() }.use { validatingClient -> eTagPartOne = validatingClient.uploadPart { bucket = testBucket @@ -210,13 +210,4 @@ class S3ExpressTest { } } } - - private class NoChecksumValidatingInterceptor : HttpInterceptor { - override fun readBeforeTransmit(context: ProtocolRequestInterceptorContext) { - val headers = context.protocolRequest.headers - if (headers.contains(S3_EXPRESS_SESSION_TOKEN_HEADER)) { - assertFalse(headers.names().any { it.startsWith("x-amz-checksum-") }) - } - } - } } diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index 7b9c22fa456..b0e72f80097 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -188,9 +188,6 @@ class S3BucketOpsIntegrationTest { } } - // FIXME: Enable test - // Seeing: S3Exception: Checksum Type mismatch occurred, expected checksum Type: null, actual checksum Type: crc32 - @Ignore @Test fun testMultipartUpload(): Unit = runBlocking { s3WithAllEngines { s3 -> diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index 21b8f311c47..d3e46bcb5c5 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -34,9 +34,7 @@ object S3TestUtils { const val DEFAULT_REGION = "us-west-2" // The E2E test account only has permission to operate on buckets with the prefix "s3-test-bucket-" - // Non-checksum E2E tests will use and delete hardcoded bucket required for checksum tests if TEST_BUCKET_PREFIX="s3-test-bucket-" via `deleteBucketAndAllContents` - // TODO: Change back to "s3-test-bucket-" - private const val TEST_BUCKET_PREFIX = "s3-test-bucket-temp-" + private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" private const val S3_MAX_BUCKET_NAME_LENGTH = 63 // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html private const val S3_EXPRESS_DIRECTORY_BUCKET_SUFFIX = "--x-s3" @@ -100,38 +98,6 @@ object S3TestUtils { testBucket } - suspend fun getBucketByName( - client: S3Client, - targetBucket: String, - region: String? = null, - accountId: String? = null, - ): Unit = withTimeout(60.seconds) { - try { - val targetBucketRegion = client - .headBucket { - this.bucket = targetBucket - expectedBucketOwner = accountId - }.bucketRegion - - if (targetBucketRegion != region) { - throw RuntimeException( - "The requested bucket ($targetBucket) already exists in another region than the one requested ($region)", - ) - } - } catch (e: Throwable) { - println("Creating S3 bucket: $targetBucket") - - client.createBucket { - bucket = targetBucket - createBucketConfiguration { - locationConstraint = BucketLocationConstraint.fromValue(region ?: client.config.region!!) - } - } - - client.waitUntilBucketExists { bucket = targetBucket } - } - } - suspend fun getTestDirectoryBucket(client: S3Client, suffix: String) = withTimeout(60.seconds) { var testBucket = client.listBuckets() .buckets