diff --git a/.gitignore b/.gitignore index d9773299..ae8a3b4e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ **/*.project **/*.classpath **/*.factorypath -**/dependency-reduced-pom.xml \ No newline at end of file +**/dependency-reduced-pom.xml +/integration-tests/* \ No newline at end of file diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java index 949e6563..372aa97f 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/AWSSchemaRegistryClient.java @@ -42,11 +42,15 @@ import software.amazon.awssdk.services.glue.model.DataFormat; import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionRequest; import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaResponse; import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; import software.amazon.awssdk.services.glue.model.GetTagsRequest; import software.amazon.awssdk.services.glue.model.GetTagsResponse; import software.amazon.awssdk.services.glue.model.GlueRequest; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsRequest; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsResponse; import software.amazon.awssdk.services.glue.model.MetadataKeyValuePair; import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataRequest; import software.amazon.awssdk.services.glue.model.PutSchemaVersionMetadataResponse; @@ -56,9 +60,12 @@ import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionResponse; import software.amazon.awssdk.services.glue.model.RegistryId; import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaVersionListItem; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.StringJoiner; import java.util.UUID; @@ -180,7 +187,7 @@ public GetSchemaVersionResponse getSchemaVersionResponse(@NonNull String schemaV return schemaVersionResponse; } - private GetSchemaVersionRequest getSchemaVersionRequest(String schemaVersionId) { + public GetSchemaVersionRequest getSchemaVersionRequest(String schemaVersionId) { GetSchemaVersionRequest getSchemaVersionRequest = GetSchemaVersionRequest.builder() .schemaVersionId(schemaVersionId).build(); return getSchemaVersionRequest; @@ -193,6 +200,43 @@ private void validateSchemaVersionResponse(GetSchemaVersionResponse schemaVersio } } + /** + * Get Schema by passing the schema id. + * @param schemaId Schema Id + * @return schema returns the schema corresponding to the + * schema id passed and null in case service is not able to found the + * schema corresponding to schema id. + * @throws AWSSchemaRegistryException on any error while fetching the schema + */ + public GetSchemaResponse getSchemaResponse(@NonNull SchemaId schemaId) + throws AWSSchemaRegistryException { + GetSchemaResponse schemaResponse = null; + + try { + schemaResponse = client.getSchema(getSchemaRequest(schemaId)); + validateSchemaResponse(schemaResponse, schemaId); + } catch (Exception e) { + String errorMessage = String.format("Failed to get schema Id = %s", schemaId); + throw new AWSSchemaRegistryException(errorMessage, e); + } + + return schemaResponse; + } + + private GetSchemaRequest getSchemaRequest(SchemaId schemaId) { + GetSchemaRequest getSchemaRequest = GetSchemaRequest.builder() + .schemaId(schemaId) + .build(); + return getSchemaRequest; + } + + private void validateSchemaResponse(GetSchemaResponse schemaResponse, SchemaId schemaId) { + if (schemaResponse == null) { + String message = String.format("Schema is not present for the schema id = %s", schemaId); + throw new AWSSchemaRegistryException(message); + } + } + private UUID returnSchemaVersionIdIfAvailable(GetSchemaByDefinitionResponse response) { if (response.schemaVersionId() != null && response.statusAsString().equals(AWSSchemaRegistryConstants.SchemaVersionStatus.AVAILABLE.toString())) { @@ -266,6 +310,53 @@ public UUID createSchema(String schemaName, return schemaVersionId; } + /** + * List all versions of the schema + * @param schemaName Schema Name + * @return List of schema versions + * @throws AWSSchemaRegistryException on any error during the registration and fetching of schema version + */ + public List getSchemaVersions(String schemaName) { + ListSchemaVersionsRequest listSchemaVersionsRequest = getListSchemaVersionsRequest(schemaName); + List schemaVersionList = new ArrayList<>(); + boolean done = false; + try { + while (!done) { + //Get list of schema versions from source registry + ListSchemaVersionsResponse listSchemaVersionsResponse = client.listSchemaVersions(listSchemaVersionsRequest); + schemaVersionList = listSchemaVersionsResponse.schemas(); + + //Keep paginating till the end + if (listSchemaVersionsResponse.nextToken() == null) { + done = true; + } + + //Create the request object to get next set of results using the nextToken + listSchemaVersionsRequest = getListSchemaVersionsRequest(schemaName, listSchemaVersionsResponse.nextToken()); + } + } catch (Exception e) { + String errorMessage = String.format("Failed to get schema version for schema = %s", schemaName); + throw new AWSSchemaRegistryException(errorMessage, e); + } + + return schemaVersionList; + } + + private ListSchemaVersionsRequest getListSchemaVersionsRequest(String schemaName, String nextToken) { + return ListSchemaVersionsRequest + .builder() + .nextToken(nextToken) + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + } + + private ListSchemaVersionsRequest getListSchemaVersionsRequest(String schemaName) { + return ListSchemaVersionsRequest + .builder() + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + } + /** * Register the schema and return schema version Id once it is available. * @param schemaDefinition Schema Definition diff --git a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java index ce7d870d..a7713b97 100644 --- a/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java +++ b/common/src/main/java/com/amazonaws/services/schemaregistry/common/configs/GlueSchemaRegistryConfiguration.java @@ -323,7 +323,7 @@ private void validateAndSetJacksonDeserializationFeatures(Map configs } } - private boolean isPresent(Map configs, + public boolean isPresent(Map configs, String key) { if (!GlueSchemaRegistryUtils.getInstance() .checkIfPresentInMap(configs, key)) { diff --git a/cross-region-replication-converter/.gitignore b/cross-region-replication-converter/.gitignore new file mode 100644 index 00000000..8a498597 --- /dev/null +++ b/cross-region-replication-converter/.gitignore @@ -0,0 +1,40 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +resources/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/cross-region-replication-converter/pom.xml b/cross-region-replication-converter/pom.xml new file mode 100644 index 00000000..4ea2e987 --- /dev/null +++ b/cross-region-replication-converter/pom.xml @@ -0,0 +1,221 @@ + + + 4.0.0 + + ${parent.groupId} + schema-registry-cross-region-kafkaconnect-converter + ${parent.version} + AWS Cross Region Glue Schema Registry Kafka Connect Schema Replication Converter + The AWS Glue Schema Registry Kafka Connect Converter enables Java developers to easily replicate + schemas across different AWS Glue Schema Registries + + https://aws.amazon.com/glue + jar + + + software.amazon.glue + schema-registry-parent + 1.1.20 + ../pom.xml + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + ossrh + https://aws.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://aws.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + scm:git:https://github.com/aws/aws-glue-schema-registry.git + scm:git:git@github.com:aws/aws-glue-schema-registry.git + https://github.com/awslabs/aws-glue-schema-registry.git + + + + + ${parent.groupId} + schema-registry-serde + ${parent.version} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + maven-plugin + + + + org.apache.kafka + connect-api + + + org.projectlombok + lombok + + + org.mockito + mockito-core + test + + + org.mockito + mockito-junit-jupiter + test + + + org.junit.vintage + junit-vintage-engine + 5.7.0 + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.jupiter + junit-jupiter-params + test + + + org.junit.platform + junit-platform-commons + test + + + org.junit.jupiter + junit-jupiter-api + test + + + junit + junit + test + + + org.powermock + powermock-reflect + 2.0.7 + test + + + uk.co.jemos.podam + podam + 7.2.5.RELEASE + test + + + software.amazon.glue + schema-registry-kafkaconnect-converter + 1.1.20 + compile + + + + + + + maven-shade-plugin + 3.2.1 + + + + + package + + shade + + + + + + org.apache.avro + avro-maven-plugin + ${avro.version} + + + generate-test-sources + test-sources + + schema + + + + generate-sources + sources + + schema + + + + + true + String + + + + + + + publishing + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + true + + sonatype-nexus-staging + https://aws.oss.sonatype.org + false + + + + + + + \ No newline at end of file diff --git a/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverter.java b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverter.java new file mode 100644 index 00000000..79b0bb5d --- /dev/null +++ b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverter.java @@ -0,0 +1,367 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +import com.amazonaws.services.schemaregistry.common.AWSSchemaRegistryClient; +import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializerImpl; +import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; +import com.amazonaws.services.schemaregistry.exception.GlueSchemaRegistryIncompatibleDataException; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializerImpl; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import lombok.Data; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.storage.Converter; +import org.jetbrains.annotations.NotNull; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.services.glue.model.AlreadyExistsException; +import software.amazon.awssdk.services.glue.model.Compatibility; +import software.amazon.awssdk.services.glue.model.GetSchemaResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; +import software.amazon.awssdk.services.glue.model.MetadataInfo; +import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataResponse; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaVersionListItem; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + + +@Data +@Slf4j +public class AWSGlueCrossRegionSchemaReplicationConverter implements Converter { + + private AwsCredentialsProvider credentialsProvider; + private GlueSchemaRegistryDeserializerImpl deserializer; + private GlueSchemaRegistrySerializerImpl serializer; + private boolean isKey; + private Map sourceConfigs; + private Map targetConfigs; + private SchemaReplicationGlueSchemaRegistryConfiguration targetGlueSchemaRegistryConfiguration; + private SchemaReplicationGlueSchemaRegistryConfiguration sourceGlueSchemaRegistryConfiguration; + + @NonNull + private AWSSchemaRegistryClient targetSchemaRegistryClient; + @NonNull + private AWSSchemaRegistryClient sourceSchemaRegistryClient; + + @NonNull + @VisibleForTesting + protected LoadingCache schemaDefinitionToVersionCache; + + + /** + * Constructor used by Kafka Connect user. + */ + public AWSGlueCrossRegionSchemaReplicationConverter(){} + + /** + * Constructor accepting AWSCredentialsProvider. + * + * @param credentialsProvider AWSCredentialsProvider instance. + */ + public AWSGlueCrossRegionSchemaReplicationConverter( + AwsCredentialsProvider credentialsProvider, + GlueSchemaRegistryDeserializerImpl deserializerImpl, + GlueSchemaRegistrySerializerImpl serializerImpl) { + + this.credentialsProvider = credentialsProvider; + this.deserializer = deserializerImpl; + this.serializer = serializerImpl; + } + + /** + * Constructor accepting AWSSchemaRegistryClient. + * + * @param sourceSchemaRegistryClient AWSSchemaRegistryClient instance. + * @param targetSchemaRegistryClient AWSSchemaRegistryClient instance. + */ + public AWSGlueCrossRegionSchemaReplicationConverter( + AWSSchemaRegistryClient sourceSchemaRegistryClient, + AWSSchemaRegistryClient targetSchemaRegistryClient, + GlueSchemaRegistryDeserializerImpl deserializerImpl, + GlueSchemaRegistrySerializerImpl serializerImpl) { + + this.sourceSchemaRegistryClient = sourceSchemaRegistryClient; + this.targetSchemaRegistryClient = targetSchemaRegistryClient; + this.deserializer = deserializerImpl; + this.serializer = serializerImpl; + } + + /** + * Configure the Schema Replication Converter. + * @param configs configuration elements for the converter + * @param isKey true if key, false otherwise + */ + @Override + public void configure(Map configs, boolean isKey) { + this.isKey = isKey; + // TODO: Support credentialProvider passed on by the user + // https://github.com/awslabs/aws-glue-schema-registry/issues/293 + if (credentialsProvider == null) { + credentialsProvider = DefaultCredentialsProvider.builder().build(); + } + + // Put the source and target regions into configurations respectively + sourceConfigs = new HashMap<>(configs); + targetConfigs = new HashMap<>(configs); + + validateRequiredConfigsIfPresent(configs); + + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION) != null) { + sourceConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT) != null) { + sourceConfigs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME) != null) { + sourceConfigs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME)); + } + + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION) != null) { + targetConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME) != null) { + targetConfigs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME)); + } + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT) != null) { + targetConfigs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT)); + } + + targetConfigs.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, true); + + targetGlueSchemaRegistryConfiguration = new SchemaReplicationGlueSchemaRegistryConfiguration(targetConfigs); + sourceGlueSchemaRegistryConfiguration = new SchemaReplicationGlueSchemaRegistryConfiguration(sourceConfigs); + + this.schemaDefinitionToVersionCache = CacheBuilder.newBuilder() + .maximumSize(targetGlueSchemaRegistryConfiguration.getCacheSize()) + .refreshAfterWrite(targetGlueSchemaRegistryConfiguration.getTimeToLiveMillis(), TimeUnit.MILLISECONDS) + .build(new SchemaDefinitionToVersionCache()); + + if (targetSchemaRegistryClient == null) { + targetSchemaRegistryClient = new AWSSchemaRegistryClient(credentialsProvider, targetGlueSchemaRegistryConfiguration); + } + if (sourceSchemaRegistryClient == null) { + sourceSchemaRegistryClient = new AWSSchemaRegistryClient(credentialsProvider, sourceGlueSchemaRegistryConfiguration); + } + + if (serializer == null) { + serializer = new GlueSchemaRegistrySerializerImpl(credentialsProvider, targetGlueSchemaRegistryConfiguration); + } + if (deserializer == null) { + deserializer = new GlueSchemaRegistryDeserializerImpl(credentialsProvider, sourceGlueSchemaRegistryConfiguration); + } + } + + @Override + public byte[] fromConnectData(String topic, org.apache.kafka.connect.data.Schema schema, Object value) { + if (value == null) return null; + byte[] bytes = (byte[]) value; + + try { + byte[] deserializedBytes = deserializer.getData(bytes); + Schema deserializedSchema = deserializer.getSchema(bytes); + createSchemaAndRegisterAllSchemaVersions(deserializedSchema); + return serializer.encode(topic, deserializedSchema, deserializedBytes); + } catch (GlueSchemaRegistryIncompatibleDataException ex) { + //This exception is raised when the header bytes don't have schema id, version byte or compression byte + //This determines the data doesn't have schema information in it, so the actual message is returned. + return bytes; + } catch (SerializationException | AWSSchemaRegistryException e) { + throw new DataException("Converting Kafka Connect data to byte[] failed due to serialization/deserialization error: ", e); + } catch (ExecutionException e) { + //TODO: Proper messaging and error handling + throw new DataException("Converting Kafka Connect data to byte[] failed due to serialization/deserialization error: ", e); + } + } + + private void validateRequiredConfigsIfPresent(Map configs) { + if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION) == null) { + throw new DataException("Source Region is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION) == null && configs.get(AWSSchemaRegistryConstants.AWS_REGION) == null) { + throw new DataException("Target Region is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME) == null) { + throw new DataException("Source Registry is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME) == null && configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME) == null) { + throw new DataException("Target Registry is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT) == null) { + throw new DataException("Source Endpoint is not provided."); + } else if (configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT) == null && configs.get(AWSSchemaRegistryConstants.AWS_ENDPOINT) == null) { + throw new DataException("Target Endpoint is not provided."); + } + } + + /** + * This method is not intended to be used for the CrossRegionReplicationConverter given it is integrated with a source connector + * + */ + @Override + public SchemaAndValue toConnectData(String topic, byte[] value) { + throw new UnsupportedOperationException("This method is not supported"); + } + + @VisibleForTesting + UUID createSchemaAndRegisterAllSchemaVersions( + @NonNull Schema schema) throws AWSSchemaRegistryException, ExecutionException { + + UUID schemaVersionId; + + try { + return schemaDefinitionToVersionCache.get(schema); + } catch (Exception ex) { + Map schemaWithVersionId = new HashMap<>(); + String schemaName = schema.getSchemaName(); + String schemaNameFromArn = ""; + String schemaDefinition = ""; + String dataFormat = schema.getDataFormat(); + Map metadataInfo = new HashMap<>(); + GetSchemaVersionResponse schemaVersionResponse = null; + + //Get compatibility mode for each schema + Compatibility compatibility = getCompatibilityMode(schema); + + targetGlueSchemaRegistryConfiguration.setCompatibilitySetting(compatibility); + targetSchemaRegistryClient = new AWSSchemaRegistryClient(credentialsProvider, targetGlueSchemaRegistryConfiguration); + + try { + + //Get list of all schema versions + List schemaVersionList = getSchemaVersionsOrderedByVersionNumber(schemaName, targetGlueSchemaRegistryConfiguration.getReplicateSchemaVersionCount()); + + for (int idx = 0; idx < schemaVersionList.size(); idx++) { + //Get details of each schema versions + schemaVersionResponse = + sourceSchemaRegistryClient.getSchemaVersionResponse(schemaVersionList.get(idx).schemaVersionId()); + + schemaNameFromArn = getSchemaNameFromArn(schemaVersionList.get(idx).schemaArn()); + schemaDefinition = schemaVersionResponse.schemaDefinition(); + + //Get the metadata information for each version + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = sourceSchemaRegistryClient.querySchemaVersionMetadata(UUID.fromString(schemaVersionResponse.schemaVersionId())); + metadataInfo = getMetadataInfo(querySchemaVersionMetadataResponse.metadataInfoMap()); + //Create the schema with the first schema version + if (idx == 0) { + //Create the schema + schemaVersionId = createSchema(schemaNameFromArn, schemaDefinition, dataFormat, metadataInfo, schemaVersionResponse); + } else { + //Register subsequent schema versions + schemaVersionId = targetSchemaRegistryClient.registerSchemaVersion(schemaVersionResponse.schemaDefinition(), + schemaNameFromArn, dataFormat, metadataInfo); + } + + cacheAllSchemaVersions(schemaVersionId, schemaWithVersionId, schemaNameFromArn, schemaVersionResponse); + } + } catch (AlreadyExistsException e) { + log.warn("Schema is already created, this could be caused by multiple producers/MM2 racing to auto-create schema."); + schemaVersionId = targetSchemaRegistryClient.registerSchemaVersion(schemaDefinition, schemaName, dataFormat, metadataInfo); + cacheAllSchemaVersions(schemaVersionId, schemaWithVersionId, schemaNameFromArn, schemaVersionResponse); + targetSchemaRegistryClient.putSchemaVersionMetadata(schemaVersionId, metadataInfo); + } catch (Exception e) { + String errorMessage = String.format("Exception occurred while fetching or registering schema name = %s ", schemaName); + //TODO: Will this exception be ever thrown? + throw new AWSSchemaRegistryException(errorMessage, e); + } + } + + schemaVersionId = schemaDefinitionToVersionCache.get(schema); + return schemaVersionId; + } + + private void cacheAllSchemaVersions(UUID schemaVersionId, Map schemaWithVersionId, String schemaNameFromArn, GetSchemaVersionResponse getSchemaVersionResponse) { + Schema schemaVersionSchema = new Schema(getSchemaVersionResponse.schemaDefinition(), getSchemaVersionResponse.dataFormat().toString(), schemaNameFromArn); + + //Create a map of schema and schemaVersionId + schemaWithVersionId.put(schemaVersionSchema, schemaVersionId); + //Cache all the schema versions for a Glue Schema Registry schema + schemaWithVersionId.entrySet() + .stream() + .forEach(item -> { + schemaDefinitionToVersionCache.put(item.getKey(), item.getValue()); + }); + } + + private UUID createSchema(String schemaNameFromArn, String schemaDefinition, String dataFormat, Map metadataInfo, GetSchemaVersionResponse getSchemaVersionResponse) { + UUID schemaVersionId; + log.info("Auto Creating schema with schemaName: {} and schemaDefinition : {}", + schemaNameFromArn, getSchemaVersionResponse.schemaDefinition()); + + schemaVersionId = targetSchemaRegistryClient.createSchema( + schemaNameFromArn, + dataFormat, + schemaDefinition, new HashMap<>()); //TODO: Get metadata of Schema + + //Add version metadata to the schema version + targetSchemaRegistryClient.putSchemaVersionMetadata(schemaVersionId, metadataInfo); + return schemaVersionId; + } + + public List getSchemaVersionsOrderedByVersionNumber(String schemaName, Integer replicateSchemaVersionCount) { + //Copy the schemaVersionList to a new list as the existing list is not modifiable. + List schemaVersionList = sourceSchemaRegistryClient.getSchemaVersions(schemaName); + List modifiableSchemaVersionList = new ArrayList<>(schemaVersionList); + + //Sort the schemaVersionList based on versionNumber in ascending order. + //This is important as the item in the list are in random order + //and we need to maintain the ordering of versions + Collections.sort(modifiableSchemaVersionList, Comparator.comparing(SchemaVersionListItem::versionNumber)); + + //Get the list of schema versions equal to the replicateSchemaVersionCount + //If the list is smaller than replicateSchemaVersionCount, return the whole list. + modifiableSchemaVersionList = modifiableSchemaVersionList.subList(0, + Math.min(replicateSchemaVersionCount, modifiableSchemaVersionList.size())); + + return modifiableSchemaVersionList; + } + + private Compatibility getCompatibilityMode(@NotNull Schema schema) { + GetSchemaResponse schemaResponse = sourceSchemaRegistryClient.getSchemaResponse(SchemaId.builder() + .schemaName(schema.getSchemaName()) + .registryName(sourceGlueSchemaRegistryConfiguration.getSourceRegistryName()) + .build()); + + Compatibility compatibility = schemaResponse.compatibility(); + return compatibility; + } + + private String getSchemaNameFromArn(String schemaArn) { + String[] tokens = schemaArn.split(Pattern.quote("/")); + return tokens[tokens.length - 1]; + } + + private Map getMetadataInfo(Map metadataInfoMap) { + Map metadata = new HashMap<>(); + Iterator> iterator = metadataInfoMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + metadata.put(entry.getKey(), entry.getValue().metadataValue()); + } + + return metadata; + } + + @RequiredArgsConstructor + private class SchemaDefinitionToVersionCache extends CacheLoader { + @Override + public UUID load(Schema schema) { + return targetSchemaRegistryClient.getSchemaVersionIdByDefinition( + schema.getSchemaDefinition(), schema.getSchemaName(), schema.getDataFormat()); + } + } +} diff --git a/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationGlueSchemaRegistryConfiguration.java b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationGlueSchemaRegistryConfiguration.java new file mode 100644 index 00000000..d11242eb --- /dev/null +++ b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationGlueSchemaRegistryConfiguration.java @@ -0,0 +1,84 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; + +@Slf4j +@Getter +public class SchemaReplicationGlueSchemaRegistryConfiguration extends GlueSchemaRegistryConfiguration { + private String sourceEndPoint; + private String sourceRegion; + private String targetEndPoint; + private String targetRegion; + private String sourceRegistryName; + private String targetRegistryName; + private int replicateSchemaVersionCount; + + public SchemaReplicationGlueSchemaRegistryConfiguration(Map configs) { + super(configs); + buildSchemaReplicationSchemaRegistryConfigs(configs); + } + + private void buildSchemaReplicationSchemaRegistryConfigs(Map configs) { + validateAndSetAWSSourceRegion(configs); + validateAndSetAWSTargetRegion(configs); + validateAndSetAWSSourceEndpoint(configs); + validateAndSetAWSTargetEndpoint(configs); + validateAndSetSourceRegistryName(configs); + validateAndSetTargetRegistryName(configs); + validateAndSetReplicateSchemaVersionCount(configs); + } + + private void validateAndSetAWSSourceRegion(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION)) { + this.sourceRegion = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION)); + } + } + + private void validateAndSetAWSTargetRegion(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION)) { + this.targetRegion = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION)); + } else { + this.targetRegion = this.getRegion(); + } + } + + private void validateAndSetSourceRegistryName(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME)) { + this.sourceRegistryName = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME)); + } + } + + private void validateAndSetTargetRegistryName(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME)) { + this.targetRegistryName = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME)); + } else { + this.targetRegistryName = this.getRegistryName(); + } + } + + private void validateAndSetAWSSourceEndpoint(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT)) { + this.sourceEndPoint = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT)); + } + } + + private void validateAndSetAWSTargetEndpoint(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT)) { + this.targetEndPoint = String.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT)); + } else { + this.targetEndPoint = this.getEndPoint(); + } + } + + private void validateAndSetReplicateSchemaVersionCount(Map configs) { + if (isPresent(configs, SchemaReplicationSchemaRegistryConstants.REPLICATE_SCHEMA_VERSION_COUNT)) { + this.replicateSchemaVersionCount = Integer.valueOf(configs.get(SchemaReplicationSchemaRegistryConstants.REPLICATE_SCHEMA_VERSION_COUNT).toString()); + } else { + this.replicateSchemaVersionCount = SchemaReplicationSchemaRegistryConstants.DEFAULT_REPLICATE_SCHEMA_VERSION_COUNT; + } + } +} diff --git a/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationSchemaRegistryConstants.java b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationSchemaRegistryConstants.java new file mode 100644 index 00000000..3f3d3089 --- /dev/null +++ b/cross-region-replication-converter/src/main/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/SchemaReplicationSchemaRegistryConstants.java @@ -0,0 +1,36 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +public class SchemaReplicationSchemaRegistryConstants { + /** + * AWS source endpoint to use while initializing the client for service. + */ + public static final String AWS_SOURCE_ENDPOINT = "source.endpoint"; + /** + * AWS source region to use while initializing the client for service. + */ + public static final String AWS_SOURCE_REGION = "source.region"; + /** + * AWS target endpoint to use while initializing the client for service. + */ + public static final String AWS_TARGET_ENDPOINT = "target.endpoint"; + /** + * AWS target region to use while initializing the client for service. + */ + public static final String AWS_TARGET_REGION = "target.region"; + /** + * Number of schema versions to replicate from source to target + */ + public static final String REPLICATE_SCHEMA_VERSION_COUNT = "replicateSchemaVersionCount"; + /** + * Default number of schema versions to replicate from source to target + */ + public static final Integer DEFAULT_REPLICATE_SCHEMA_VERSION_COUNT = 10; + /** + * Source Registry Name. + */ + public static final String SOURCE_REGISTRY_NAME = "source.registry.name"; + /** + * Target Registry Name. + */ + public static final String TARGET_REGISTRY_NAME = "target.registry.name"; +} diff --git a/cross-region-replication-converter/src/test/avro/AvroMessage.avsc b/cross-region-replication-converter/src/test/avro/AvroMessage.avsc new file mode 100644 index 00000000..4a784cd6 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/AvroMessage.avsc @@ -0,0 +1,944 @@ +{ + "type" : "record", + "name" : "AvroMessage", + "namespace" : "io.test.avro.core", + "fields" : [ { + "name" : "payload", + "type" : [ "null", { + "type" : "record", + "name" : "Event", + "namespace" : "io.test.trade.v1.order", + "fields" : [ { + "name" : "state", + "type" : { + "type" : "record", + "name" : "State", + "fields" : [ { + "name" : "orderId", + "type" : { + "type" : "record", + "name" : "Id", + "namespace" : "io.test.trade.v1", + "doc" : "Id of an order or position.", + "fields" : [ { + "name" : "source", + "type" : { + "type" : "enum", + "name" : "Source", + "symbols" : [ "ORDER_SERVER", "CLIENT", "UNIVERSE", "L2", "L2_CHAIN", "EXCHANGE", "UNIVERSE_ATTR", "UNDEFINED" ] + } + }, { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "The ID to reference this order." + }, { + "name" : "accountId", + "type" : { + "type" : "record", + "name" : "Id", + "namespace" : "io.test.trade.v1.common.account", + "fields" : [ { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "The account that the order is associated to." + }, { + "name" : "allocation", + "type" : { + "type" : "record", + "name" : "Allocation", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "direction", + "type" : { + "type" : "enum", + "name" : "Direction", + "symbols" : [ "BUY", "SELL" ] + } + }, { + "name" : "size", + "type" : { + "type" : "record", + "name" : "Size", + "fields" : [ { + "name" : "value", + "type" : "double" + } ] + } + }, { + "name" : "displaySize", + "type" : "Size", + "doc" : "Size used for presentation and external reporting purposes. Note: Margining, Profit/Loss calculation, exposure, etc., should multiply this size with lotSize for calculations." + }, { + "name" : "displaySizeUnit", + "type" : { + "type" : "enum", + "name" : "DisplaySizeUnit", + "symbols" : [ "SHARES", "CONTRACTS", "AMOUNT_PER_POINTS" ] + }, + "doc" : "Define how the displaySize is expressed." + }, { + "name" : "lotSize", + "type" : "double", + "doc" : "Defined on the instrument. Clients on spread-bet accounts use lot size of 1, while CFD and StockBroking clients use lot size configured on the instrument. Dealers can book orders on any client account with a lot size of 1. Hedge accounts have different rules - they are generally booked in lots with lotSize 1, except equities (and equity options)" + }, { + "name" : "currency", + "type" : { + "type" : "record", + "name" : "ISOCurrency", + "fields" : [ { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "Currency of the order/position" + } ] + }, + "doc" : "Details of the allocation of this order, size/amount, currency and direction." + }, { + "name" : "instrument", + "type" : { + "type" : "record", + "name" : "Instrument", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "bookingCodeType", + "type" : { + "type" : "enum", + "name" : "BookingCodeType", + "symbols" : [ "EPIC", "ISIN_AND_CURRENCY" ] + }, + "doc" : "Indicates if the booking was made using an ISIN or EPIC. If EPIC, unique instrument identifier is the EPIC. If ISIN_AND_CURRENCY, the unique identifier is ISIN and CURRENCY. In this last case EPIC is also set for internal usage." + }, { + "name" : "epic", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "This field is populated for all booking operations and it represents the id of the instrument on which the booking was made. Note: This field will probably be made optional when contract-ids are introduced" + }, { + "name" : "isin", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "ISIN is populated for stock-broking deals", + "default" : null + }, { + "name" : "level", + "type" : { + "type" : "record", + "name" : "Level", + "fields" : [ { + "name" : "value", + "type" : "double" + } ] + }, + "doc" : "The level at which the position is booked. This level is used for pnl, margining, auto-hedge, exposure calculation, and other such purposes. This level should always be displayLevel multiplied by scaling factor, however, there are a few UV based flows where that constraint doesn't hold." + }, { + "name" : "displayLevel", + "type" : "Level", + "doc" : "The level displayed in the front-end. Note: PnL calculation, auto-hedge and other such operations should multiply by scaling factor." + }, { + "name" : "scalingFactor", + "type" : "double", + "doc" : "The scaling factor used for booking this position." + }, { + "name" : "instrumentType", + "type" : [ "null", { + "type" : "enum", + "name" : "InstrumentType", + "symbols" : [ "SHARES", "CURRENCIES", "INDICES", "BINARY", "FAST_BINARY", "COMMODITIES", "RATES", "OPTIONS_SHARES", "OPTIONS_CURRENCIES", "OPTIONS_INDICES", "OPTIONS_COMMODITIES", "OPTIONS_RATES", "BUNGEE_SHARES", "BUNGEE_CURRENCIES", "BUNGEE_INDICES", "BUNGEE_COMMODITIES", "BUNGEE_RATES", "CAPPED_BUNGEE", "TEST_MARKETS", "SPORTS", "SECTORS" ] + } ], + "doc" : "Type of the instrument on which the booking is made. This field is made optional to accommodate positions missing instrument type for some reason (very old positions, UV based legacy flows, etc)", + "default" : null + }, { + "name" : "marketName", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "scalingFactorOnInstrumentIfDifferent", + "type" : [ "null", "double" ], + "doc" : "If the scaling factor on the instrument is different from the scaling factor used to book this position then this field carries this new scaling factor. This field is used by a trade anomaly report (maintained by XCON)", + "default" : null + } ] + }, + "doc" : "Details on booked level and instrument information" + }, { + "name" : "timestamps", + "type" : { + "type" : "record", + "name" : "Timestamps", + "namespace" : "io.test.trade.v1.common", + "doc" : "See http://test.io/wiki/Position+History+Tactical+Fixes", + "fields" : [ { + "name" : "created", + "type" : { + "type" : "record", + "name" : "UTCTimestamp", + "fields" : [ { + "name" : "value", + "type" : "long" + } ] + }, + "doc" : "Timestamp of the trade's creation time" + }, { + "name" : "lastModified", + "type" : [ "null", "UTCTimestamp" ], + "doc" : "Timestamp of the trade's modification time. For RESTATE, this field indicates the timestamp of the restate", + "default" : null + }, { + "name" : "lastEdited", + "type" : [ "null", "UTCTimestamp" ], + "doc" : "Applicable only for RESTATES and specifies the timestamp at which this trade was last edited", + "default" : null + }, { + "name" : "margin", + "type" : [ "null", "UTCTimestamp" ], + "doc" : "Timestamp of the trade's margin time.", + "default" : null + } ] + }, + "doc" : "Timestamps of when the order was created, modified or last edited." + }, { + "name" : "isForceOpen", + "type" : "boolean", + "doc" : "Upon full fill, should this order close positions existing in opposite direction?", + "default" : false + }, { + "name" : "attachedStop", + "type" : [ "null", { + "type" : "record", + "name" : "Stop", + "namespace" : "io.test.trade.v1.common.contingent", + "fields" : [ { + "name" : "value", + "type" : "double", + "doc" : "Stop value can be expressed either as a Level or Distance. Use this field in conjunction with valueType" + }, { + "name" : "valueType", + "type" : { + "type" : "enum", + "name" : "StopValueType", + "symbols" : [ "DISTANCE", "LEVEL" ] + }, + "doc" : "Represents the unit in which the stop value is expressed" + }, { + "name" : "isGuaranteed", + "type" : "boolean", + "default" : false + }, { + "name" : "lrPremium", + "type" : [ "null", "double" ], + "doc" : "This field represents a multiplier to be applied to the trade's size to derive a limited risk fee (LR Fee). The LR fee is a monetary amount and is expressed in the currency of the order.", + "default" : null + }, { + "name" : "trailingStop", + "type" : [ "null", { + "type" : "record", + "name" : "TrailingStop", + "fields" : [ { + "name" : "distance", + "type" : "double", + "default" : 0.0 + }, { + "name" : "increment", + "type" : "double", + "default" : 0.0 + } ] + } ], + "default" : null + }, { + "name" : "orderIds", + "type" : [ "null", { + "type" : "array", + "items" : "io.test.trade.v1.Id" + } ], + "doc" : "Ids identifying this stop.", + "default" : null + } ] + } ], + "doc" : "An attached Stop is a 'stop-loss' order; an instruction to close a position when a certain level is breached, to minimize loss.", + "default" : null + }, { + "name" : "attachedLimit", + "type" : [ "null", { + "type" : "record", + "name" : "Limit", + "namespace" : "io.test.trade.v1.common.contingent", + "fields" : [ { + "name" : "value", + "type" : "double", + "doc" : "Limit value can be expressed either as a Level or Distance. Use this field in conjunction with valueType" + }, { + "name" : "valueType", + "type" : { + "type" : "enum", + "name" : "LimitValueType", + "symbols" : [ "DISTANCE", "LEVEL" ] + }, + "doc" : "Represents the unit in which the limit value is expressed" + }, { + "name" : "orderIds", + "type" : [ "null", { + "type" : "array", + "items" : "io.test.trade.v1.Id" + } ], + "doc" : "Ids identifying this limit.", + "default" : null + } ] + } ], + "doc" : "An attached limit is a 'profit-limit' order; an instruction to close a position when a certain level is breached, to guarantee profit.", + "default" : null + }, { + "name" : "legacyInfo", + "type" : { + "type" : "record", + "name" : "LegacyInfo", + "namespace" : "io.test.trade.v1.common", + "doc" : "Legacy information for retro compatibility purpose. Should not be used in any new service.", + "fields" : [ { + "name" : "uvCurrency", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "deprecated field", + "default" : "" + }, { + "name" : "marketCommodity", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "deprecated field", + "default" : "" + }, { + "name" : "prompt", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "Deprecated field. Represents the period at which the instrument expires. This field is also referred to as 'period' in some legacy messages", + "default" : null + }, { + "name" : "submitOrderType", + "type" : [ "null", { + "type" : "enum", + "name" : "SubmitOrderType", + "symbols" : [ "UNCONTROLLED_OPEN", "UNCONTROLLED_CLOSE", "CONTROLLED_OPEN", "CONTROLLED_CLOSE", "UNCONTROLLED_OPEN_WITH_STOP", "UNCONTROLLED_CLOSE_WITH_STOP", "MANUAL_POSITION_DELETE", "OPEN_WITH_EXPIRY_STOP" ] + } ], + "doc" : "deprecated field", + "default" : null + }, { + "name" : "requestType", + "type" : [ "null", { + "type" : "enum", + "name" : "RequestType", + "symbols" : [ "AMEND_ORDER", "FINANCE_ORDER", "CFD_ORDER", "PHYSICAL", "UNATTACHED_LIMIT_ORDER", "UNATTACHED_STOP_ORDER", "UNATTACHED_ORDER_DELETE", "UNATTACHED_ORDER_FILL", "UNATTACHED_BUFFER_LIMITS", "UNATTACHED_BUFFER_LIMITS_DELETE", "MARKET_ORDER" ] + } ], + "doc" : "deprecated field", + "default" : null + }, { + "name" : "exchangeRateEpic", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "Deprecated field. This field represents the fx rate epic mapped to the trade's currency.", + "default" : null + } ] + }, + "doc" : "Legacy attributes, a hang over from UV, eg market commod." + }, { + "name" : "channel", + "type" : { + "type" : "record", + "name" : "Channel", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "value", + "type" : { + "type" : "string", + "avro.java.string" : "String" + } + } ] + }, + "doc" : "The channel though which the order was placed , eg. WEB, L2. This will not change if the order is amended. For this, see channel in change info." + }, { + "name" : "expiry", + "type" : { + "type" : "record", + "name" : "Expiry", + "namespace" : "io.test.trade.v1.order.common", + "fields" : [ { + "name" : "timeInForce", + "type" : { + "type" : "enum", + "name" : "TimeInForce", + "namespace" : "io.test.trade.v1.common", + "symbols" : [ "DAY", "GOOD_TILL_CANCEL", "AT_THE_OPENING", "IMMEDIATE_OR_CANCEL", "FILL_OR_KILL", "GOOD_TILL_CROSSING", "GOOD_TILL_DATE", "AT_THE_CLOSE", "DAY_ALL_SESSIONS" ] + } + }, { + "name" : "goodTillDateTimestamp", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + } ] + }, + "doc" : "The date/time of when this order expires." + }, { + "name" : "accountAttributes", + "type" : { + "type" : "record", + "name" : "Attributes", + "namespace" : "io.test.trade.v1.common.account", + "fields" : [ { + "name" : "accountProduct", + "type" : [ "null", { + "type" : "enum", + "name" : "Product", + "symbols" : [ "SPREAD_BET", "CFD", "PHYSICAL" ] + } ], + "default" : null + }, { + "name" : "locale", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "doc" : "Represents the locale of the account, such as en_gb. It is highly likely that this field will be removed in the future." + }, { + "name" : "powerOfAttorneyName", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "The POA name specified on the order that booked this position.", + "default" : null + }, { + "name" : "convertOnCloseCurrency", + "type" : [ "null", "io.test.trade.v1.common.ISOCurrency" ], + "doc" : "Retrieved from the convert on close information stored on this position. Note: Only populated if convert-on-close is applicable for this position.", + "default" : null + }, { + "name" : "currency", + "type" : [ "null", "io.test.trade.v1.common.ISOCurrency" ], + "doc" : "Account's currency in ISO format", + "default" : null + } ] + }, + "doc" : "Account attributes such as convert on close details and Power Of Attorney name." + }, { + "name" : "dmaOrder", + "type" : [ "null", { + "type" : "record", + "name" : "Order", + "namespace" : "io.test.trade.v1.common.dma", + "fields" : [ { + "name" : "pseudoPositionId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "doc" : "Represents ID of position created by a partially filled DMA order.", + "default" : null + }, { + "name" : "orderType", + "type" : { + "type" : "enum", + "name" : "OrderType", + "symbols" : [ "MARKET", "LIMIT", "STOP", "STOP_LIMIT", "MARKET_ON_CLOSE", "WITH_OR_WITHOUT", "LIMIT_OR_BETTER", "LIMIT_WITH_OR_WITHOUT", "ON_BASIS", "ON_CLOSE", "LIMIT_ON_CLOSE", "FOREX_MARKET", "PREVIOUSLY_QUOTED", "PREVIOUSLY_INDICATED", "FOREX_LIMIT", "PEGGED", "TRADE_REPORT", "FAST_BINARY", "UNKNOWN" ] + } + }, { + "name" : "timeInForce", + "type" : "io.test.trade.v1.common.TimeInForce" + }, { + "name" : "originalSize", + "type" : "io.test.trade.v1.common.Size", + "doc" : "The original size on a DMA working order. This is in display terms and does not include lotSize" + }, { + "name" : "fills", + "type" : [ "null", { + "type" : "record", + "name" : "Fills", + "fields" : [ { + "name" : "aggregatedFill", + "type" : [ "null", { + "type" : "array", + "items" : { + "type" : "record", + "name" : "AggregatedFill", + "doc" : "Aggregated information of fills received per hedge account", + "fields" : [ { + "name" : "hedgeAccountId", + "type" : "io.test.trade.v1.common.account.Id" + }, { + "name" : "averageLevel", + "type" : "io.test.trade.v1.common.Level", + "doc" : "A volume-weighted-average level of all fills originating from this hedge account" + }, { + "name" : "totalSize", + "type" : "io.test.trade.v1.common.Size", + "doc" : "Total size of all fills received from this hedge account" + }, { + "name" : "averageExchangeFee", + "type" : "double", + "doc" : "The fee is expressed in account's currency." + } ] + } + } ], + "doc" : "A collection of DMA fills aggregated per hedge account", + "default" : null + }, { + "name" : "updateType", + "type" : { + "type" : "enum", + "name" : "FillsUpdateType", + "symbols" : [ "ADD", "COPY", "DELETE_ALL" ] + }, + "default" : "COPY" + }, { + "name" : "nextWorkingOrderId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "isDMAInteractable", + "type" : "boolean", + "default" : true + }, { + "name" : "executionPricePreference", + "type" : [ "null", { + "type" : "enum", + "name" : "ExecType", + "symbols" : [ "ASK", "BID" ] + } ], + "doc" : "Called executionInstruction in current schema. This field represents DMA FX Stop Order execution price preference, could be either empty, ASK(0) or BID(9) and indicates whether one's order gets executed closer to the Bid or Ask side compared to the specified order direction.", + "default" : null + }, { + "name" : "uvOrderId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "default" : null + }, { + "name" : "isPseudoPosition", + "type" : "boolean", + "doc" : "Is this position a partial fill for a DMA order?", + "default" : false + }, { + "name" : "nextPseudoPositionId", + "type" : [ "null", "io.test.trade.v1.Id" ], + "doc" : "In a DMA amend scenario, the id of a pseudo-position changes and this field indicates the new pseudo position id.", + "default" : null + } ] + } ], + "doc" : "DMA order attributes such as order type", + "default" : null + }, { + "name" : "additionalIds", + "type" : [ "null", { + "type" : "array", + "items" : "io.test.trade.v1.Id" + } ], + "doc" : "Additional ids used to reference this order.", + "default" : null + }, { + "name" : "stockBrokingAttributes", + "type" : [ "null", { + "type" : "record", + "name" : "Attributes", + "namespace" : "io.test.trade.v1.order.stockbroking", + "fields" : [ { + "name" : "settlementDate", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + }, { + "name" : "tradeDate", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + }, { + "name" : "charges", + "type" : [ "null", { + "type" : "record", + "name" : "Charges", + "fields" : [ { + "name" : "commission", + "type" : { + "type" : "record", + "name" : "Money", + "namespace" : "io.test.trade.v1.common", + "fields" : [ { + "name" : "currency", + "type" : "ISOCurrency" + }, { + "name" : "amount", + "type" : "double" + } ] + } + }, { + "name" : "physicalCharges", + "type" : [ "null", { + "type" : "array", + "items" : { + "type" : "record", + "name" : "Charge", + "fields" : [ { + "name" : "code", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "default" : "" + }, { + "name" : "name", + "type" : { + "type" : "string", + "avro.java.string" : "String" + }, + "default" : "" + }, { + "name" : "amount", + "type" : "io.test.trade.v1.common.Money" + }, { + "name" : "rate", + "type" : "double" + }, { + "name" : "threshold", + "type" : "double" + } ] + } + } ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "reservedCash", + "type" : [ "null", "io.test.trade.v1.common.Money" ], + "default" : null + } ] + } ], + "doc" : "Stock Broking specific attributes such as settlement date and trade date.", + "default" : null + }, { + "name" : "commissionInstructions", + "type" : [ "null", { + "type" : "record", + "name" : "Instructions", + "namespace" : "io.test.trade.v1.common.commission", + "fields" : [ { + "name" : "bypasses", + "type" : [ "null", { + "type" : "record", + "name" : "Bypasses", + "fields" : [ { + "name" : "legacyCRPremium", + "type" : "boolean", + "doc" : "For guaranteed stops, should bypass reserving LR Premium fee", + "default" : false + }, { + "name" : "commission", + "type" : "boolean", + "doc" : "Should commission be bypassed", + "default" : false + }, { + "name" : "charges", + "type" : "boolean", + "doc" : "Should charges be bypassed", + "default" : false + }, { + "name" : "consideration", + "type" : "boolean", + "doc" : "Should consideration based fee be bypassed", + "default" : false + } ] + } ], + "default" : null + }, { + "name" : "overrideType", + "type" : [ "null", { + "type" : "enum", + "name" : "OverrideType", + "doc" : "AMOUNT: This type used when dealer wants to fixed Commission charge in Client's base currency. When the Amount value is Zero, no commission will be charged.\n. WEB_RATES: This type is used when dealer wants the client's web rates to be used. Otherwise an input from IG Dealer will cause the phone rates to be used.\nPERCENT: This type is used when dealer wants to supply the percentage rate to be used for commission calculation", + "symbols" : [ "AMOUNT", "PERCENT", "WEB_RATES" ] + } ], + "default" : null + }, { + "name" : "rate", + "type" : [ "null", "double" ], + "default" : null + }, { + "name" : "comment", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + } ] + } ], + "doc" : "instructions of which changes to bypass or override.", + "default" : null + }, { + "name" : "additionalLeg", + "type" : [ "null", { + "type" : "record", + "name" : "AdditionalLeg", + "namespace" : "io.test.trade.v1.order.common", + "fields" : [ { + "name" : "instrument", + "type" : "io.test.trade.v1.common.Instrument" + }, { + "name" : "marketCommodity", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "direction", + "type" : "io.test.trade.v1.common.Direction" + }, { + "name" : "averagePrice", + "type" : [ "null", "io.test.trade.v1.common.Level" ], + "default" : null + } ] + } ], + "doc" : "DMA orders on hedge accounts can optionally have an additional leg (instrument) to book the same order.", + "default" : null + }, { + "name" : "profileData", + "type" : [ "null", { + "type" : "record", + "name" : "ProfileData", + "fields" : [ { + "name" : "parentAccountId", + "type" : "io.test.trade.v1.common.account.Id", + "doc" : "For Profile orders this is the reference to the parent account." + }, { + "name" : "parentOrderId", + "type" : "io.test.trade.v1.Id", + "doc" : "For profile orders this will be the parent order id." + } ] + } ], + "doc" : "Profile orders where the order is processed on the parent account and booked against the child accounts.", + "default" : null + }, { + "name" : "lockState", + "type" : [ "null", { + "type" : "record", + "name" : "State", + "namespace" : "io.test.trade.v1.common.lock", + "fields" : [ { + "name" : "idOfLockingDMAOrder", + "type" : "io.test.trade.v1.Id", + "doc" : "Contains id of a DMA order that has locked this position, presumably for explicitly closing this position" + }, { + "name" : "holder", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "source", + "type" : [ "null", { + "type" : "enum", + "name" : "Source", + "symbols" : [ "COM", "DMA", "STOP_MONITOR" ] + } ], + "default" : null + }, { + "name" : "stopMonitorState", + "type" : [ "null", { + "type" : "enum", + "name" : "StopMonitorState", + "symbols" : [ "COM", "DMA", "STOP_MONITOR" ] + } ], + "default" : null + } ] + } ], + "doc" : "Indicates if the order is locked and they type of lock.", + "default" : null + }, { + "name" : "narrative", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "doc" : "Free text for reference. Not used in processing.", + "default" : null + } ] + } + }, { + "name" : "changeInfo", + "type" : { + "type" : "record", + "name" : "Info", + "namespace" : "io.test.trade.v1.order.change", + "fields" : [ { + "name" : "action", + "type" : { + "type" : "enum", + "name" : "Action", + "namespace" : "io.test.trade.v1.common.change", + "symbols" : [ "UPDATE", "DELETE", "NEW", "RESTATE" ] + } + }, { + "name" : "channel", + "type" : [ "null", "io.test.trade.v1.common.Channel" ], + "default" : null + }, { + "name" : "attachedStop", + "type" : [ "null", { + "type" : "record", + "name" : "Stop", + "namespace" : "io.test.trade.v1.common.change.attached", + "fields" : [ { + "name" : "action", + "type" : { + "type" : "enum", + "name" : "Action", + "symbols" : [ "NEW", "DELETED", "UPDATED" ] + } + }, { + "name" : "distance", + "type" : "double" + }, { + "name" : "trailingStop", + "type" : [ "null", { + "type" : "record", + "name" : "TrailingStop", + "fields" : [ { + "name" : "action", + "type" : "Action" + }, { + "name" : "distance", + "type" : "double" + }, { + "name" : "increment", + "type" : "double" + } ] + } ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "attachedLimit", + "type" : [ "null", { + "type" : "record", + "name" : "Limit", + "namespace" : "io.test.trade.v1.common.change.attached", + "fields" : [ { + "name" : "action", + "type" : "Action" + }, { + "name" : "distance", + "type" : "double" + } ] + } ], + "default" : null + }, { + "name" : "transactOrderReference", + "type" : [ "null", "io.test.trade.v1.Id" ], + "default" : null + }, { + "name" : "transactTimestamp", + "type" : [ "null", "io.test.trade.v1.common.UTCTimestamp" ], + "default" : null + } ] + } + }, { + "name" : "transaction", + "type" : [ "null", { + "type" : "record", + "name" : "Info", + "namespace" : "io.test.trade.v1.common.transaction", + "fields" : [ { + "name" : "id", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "group", + "type" : [ "null", { + "type" : "record", + "name" : "Group", + "fields" : [ { + "name" : "size", + "type" : "int" + }, { + "name" : "messageIndex", + "type" : "int" + } ] + } ], + "default" : null + } ] + } ], + "default" : null + } ] + } ], + "default" : null + }, { + "name" : "properties", + "type" : [ "null", { + "type" : "map", + "values" : { + "type" : "string", + "avro.java.string" : "String" + }, + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "uuid", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "clientId", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "partitionKey", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "correlationId", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + }, { + "name" : "clusterId", + "type" : [ "null", { + "type" : "string", + "avro.java.string" : "String" + } ], + "default" : null + } ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/DocTestRecord.avsc b/cross-region-replication-converter/src/test/avro/DocTestRecord.avsc new file mode 100644 index 00000000..b83348ef --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/DocTestRecord.avsc @@ -0,0 +1,20 @@ +{ + "type" : "record", + "name" : "DocTestRecord", + "namespace" : "io.test.avro.doc", + "doc" : "Some record document.", + "fields" : [ { + "name" : "obj", + "type" : { + "type" : "record", + "name" : "DocTestRecord1", + "doc" : "Some nested record document.", + "fields" : [ { + "name" : "data", + "type" : "string", + "doc" : "Some nested record field document." + } ] + }, + "doc" : "Some field document." + } ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/Enum.avsc b/cross-region-replication-converter/src/test/avro/Enum.avsc new file mode 100644 index 00000000..4d24ad60 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/Enum.avsc @@ -0,0 +1,16 @@ +{ + "namespace": "foo.bar", + "type": "record", + "name": "EnumTest", + "fields": [ + {"name": "testkey", "type": "string"}, + { + "name": "kind", + "type": { + "name": "Kind", + "type": "enum", + "symbols" : ["ONE", "TWO", "THREE"] + } + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/EnumUnion.avsc b/cross-region-replication-converter/src/test/avro/EnumUnion.avsc new file mode 100644 index 00000000..7e90cd0a --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/EnumUnion.avsc @@ -0,0 +1,22 @@ +{ + "type": "record", + "name": "EnumUnion", + "namespace": "com.connect.avro", + "fields": [ + { + "name": "userType", + "type": [ + "null", + { + "type": "enum", + "name": "UserType", + "symbols": [ + "ANONYMOUS", + "REGISTERED" + ] + } + ], + "default": null + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/MultiTypeUnionMessage.avsc b/cross-region-replication-converter/src/test/avro/MultiTypeUnionMessage.avsc new file mode 100644 index 00000000..35aa90b7 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/MultiTypeUnionMessage.avsc @@ -0,0 +1,45 @@ +{ + "type": "record", + "name": "MultiTypeUnionMessage", + "namespace": "io.test.avro.union", + "fields": [ + { + "name": "CompositeRecord", + "type": [ + "null", + { + "type": "record", + "name": "FirstOption", + "fields": [ + { + "name": "x", + "type": "string" + }, + { + "name": "y", + "type": "long" + } + ] + }, + { + "type": "record", + "name": "SecondOption", + "fields": [ + { + "name": "a", + "type": "string" + }, + { + "name": "b", + "type": "long" + } + ] + }, + { + "type": "array", + "items": "string" + } + ] + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDefault.avsc b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDefault.avsc new file mode 100644 index 00000000..12b80326 --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDefault.avsc @@ -0,0 +1,45 @@ +{ + "name": "RepeatedTypeWithDefault", + "namespace": "com.rr.avro.test", + "type": "record", + "fields": [ + { + "name": "stringField", + "type": "string", + "default": "field's default" + }, + { + "name": "anotherStringField", + "type": "string" + }, + { + "name": "enumField", + "default": "ONE", + "type": { + "name": "Kind", + "type": "enum", + "symbols" : ["ONE", "TWO", "THREE"] + } + }, + { + "name": "anotherEnumField", + "type": "Kind", + "default": "TWO" + }, + { + "name": "enumFieldWithDiffDefault", + "default": "B", + "type": { + "name": "someKind", + "type": "enum", + "symbols": ["A", "B", "C"], + "default": "A" + } + }, + { + "name": "floatField", + "type": "float", + "default": 9.18 + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDocFull.avsc b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDocFull.avsc new file mode 100644 index 00000000..5cf02e9b --- /dev/null +++ b/cross-region-replication-converter/src/test/avro/RepeatedTypeWithDocFull.avsc @@ -0,0 +1,96 @@ +{ + "name": "RepeatedTypeWithDoc", + "namespace": "com.rr.avro.test", + "type": "record", + "doc": "record's doc", + "fields": [ + { + "name": "stringField", + "type": "string", + "doc": "field's doc" + }, + { + "name": "anotherStringField", + "type": "string" + }, + { + "name": "recordField", + "doc": "record field's doc", + "type": { + "name": "NestedRecord", + "type": "record", + "doc": "nested record's doc", + "fields": [ + { + "name": "nestedRecordField", + "doc": "nested record field's doc", + "type": { + "name": "FixedType", + "type": "fixed", + "size": 4 + } + }, + { + "name": "anotherNestedRecordField", + "type": "FixedType" + } + ] + } + }, + { + "name": "anotherRecordField", + "type": "NestedRecord", + "doc": "another record field's doc" + }, + { + "name": "recordFieldWithoutDoc", + "type": "NestedRecord" + }, + { + "name": "doclessRecordField", + "type": { + "name": "DoclessNestedRecord", + "type": "record", + "fields": [ + { + "name": "aField", + "type": "string" + } + ] + } + }, + { + "name": "doclessRecordFieldWithDoc", + "type": "DoclessNestedRecord", + "doc": "docless record field's doc" + }, + { + "name": "enumField", + "doc": "enum field's doc", + "type": { + "name": "Kind", + "type": "enum", + "doc": "enum's doc", + "symbols" : ["ONE", "TWO", "THREE"] + } + }, + { + "name": "anotherEnumField", + "type": "Kind", + "doc": "another enum field's doc" + }, + { + "name": "doclessEnumField", + "type": "Kind" + }, + { + "name": "diffEnumField", + "type": { + "name": "anotherKind", + "type": "enum", + "doc": "diffEnum's doc", + "symbols": ["A", "B", "C"] + } + } + ] +} \ No newline at end of file diff --git a/cross-region-replication-converter/src/test/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverterTest.java b/cross-region-replication-converter/src/test/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverterTest.java new file mode 100644 index 00000000..cc97294a --- /dev/null +++ b/cross-region-replication-converter/src/test/java/com/amazonaws/services/crossregion/schemaregistry/kafkaconnect/AWSGlueCrossRegionSchemaReplicationConverterTest.java @@ -0,0 +1,986 @@ +package com.amazonaws.services.crossregion.schemaregistry.kafkaconnect; + +import com.amazonaws.services.schemaregistry.common.AWSSchemaRegistryClient; +import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializerImpl; +import com.amazonaws.services.schemaregistry.exception.AWSSchemaRegistryException; +import com.amazonaws.services.schemaregistry.exception.GlueSchemaRegistryIncompatibleDataException; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializerImpl; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.google.common.cache.LoadingCache; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.errors.DataException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.Compatibility; +import software.amazon.awssdk.services.glue.model.CreateSchemaRequest; +import software.amazon.awssdk.services.glue.model.CreateSchemaResponse; +import software.amazon.awssdk.services.glue.model.DataFormat; +import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaByDefinitionResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaResponse; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsRequest; +import software.amazon.awssdk.services.glue.model.ListSchemaVersionsResponse; +import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataRequest; +import software.amazon.awssdk.services.glue.model.QuerySchemaVersionMetadataResponse; +import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionRequest; +import software.amazon.awssdk.services.glue.model.RegisterSchemaVersionResponse; +import software.amazon.awssdk.services.glue.model.RegistryId; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaStatus; +import software.amazon.awssdk.services.glue.model.SchemaVersionListItem; + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for testing RegisterSchema class. + */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) + +public class AWSGlueCrossRegionSchemaReplicationConverterTest { + @Mock + private AwsCredentialsProvider credProvider; + @Mock + private GlueSchemaRegistryDeserializerImpl deserializer; + @Mock + private GlueSchemaRegistrySerializerImpl serializer; + private GlueClient mockGlueClient; + private AWSSchemaRegistryClient sourceSchemaRegistryClient; + private AWSSchemaRegistryClient targetSchemaRegistryClient; + private final static byte[] ENCODED_DATA = new byte[] { 8, 9, 12, 83, 82 }; + private final static byte[] USER_DATA = new byte[] { 12, 83, 82 }; + private static final String testTopic = "User-Topic"; + private AWSGlueCrossRegionSchemaReplicationConverter converter; + private static final UUID SCHEMA_ID_FOR_TESTING = UUID.fromString("f8b4a7f0-9c96-4e4a-a687-fb5de9ef0c63"); + private static final UUID SCHEMA_ID_FOR_TESTING2 = UUID.fromString("310153e9-9a54-4b12-a513-a23fc543ed2f"); + private AWSSchemaRegistryClient awsSchemaRegistryClient; + private String userSchemaDefinition; + private String userSchemaDefinition2; + + byte[] genericBytes = new byte[] {3, 0, -73, -76, -89, -16, -100, -106, 78, 74, -90, -121, -5, + 93, -23, -17, 12, 99, 10, 115, 97, 110, 115, 97, -58, 1, 6, 114, 101, 100}; + byte[] avroBytes = new byte[] {3, 0, 84, 24, 47, -109, 37, 124, 74, 77, -100, + -98, -12, 118, 41, 32, 57, -66, 30, 101, 110, 116, 101, 114, 116, 97, 105, 110, 109, 101, 110, + 116, 95, 50, 0, 0, 0, 0, 0, 0, 20, 64}; + byte[] jsonBytes = new byte[] {3, 0, -73, -76, -89, -16, -100, -106, 78, 74, -90, -121, -5, 93, -23, -17, 12, 99, 123, 34, + 102, 105, 114, 115, 116, 78, 97, 109, 101, 34, 58, 34, 74, 111, 104, 110, 34, 44, 34, 108, 97, + 115, 116, 78, 97, 109, 101, 34, 58, 34, 68, 111, 101, 34, 44, 34, 97, 103, 101, 34, 58, 50, 49, + 125}; + byte[] protobufBytes = "foo".getBytes(StandardCharsets.UTF_8); + + @BeforeEach + void setUp() { + mockGlueClient = mock(GlueClient.class); + awsSchemaRegistryClient = new AWSSchemaRegistryClient(mockGlueClient); + sourceSchemaRegistryClient = new AWSSchemaRegistryClient(mockGlueClient); + targetSchemaRegistryClient = new AWSSchemaRegistryClient(mockGlueClient); + userSchemaDefinition = "{Some-avro-schema}"; + userSchemaDefinition2 = "{Some-avro-schema-v2}"; + } + + /** + * Test for Converter config method. + */ + @Test + public void testConverter_configure() throws NoSuchFieldException, IllegalAccessException { + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + assertNotNull(converter); + assertNotNull(converter.getCredentialsProvider()); + assertNotNull(converter.getSerializer()); + assertNotNull(converter.getDeserializer()); + assertNotNull(converter.isKey()); + } + + /** + * Test for Converter when source region config is not provided. + */ + @Test + public void testConverter_sourceRegionNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoSourceRegionProperties(), false)); + assertEquals("Source Region is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source registry config is not provided. + */ + @Test + public void testConverter_sourceRegistryNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoSourceRegistryProperties(), false)); + assertEquals("Source Registry is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source endpoint config is not provided. + */ + @Test + public void testConverter_sourceEndpointNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoSourceEndpointProperties(), false)); + assertEquals("Source Endpoint is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when target region config is not provided. + */ + @Test + public void testConverter_targetRegionNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoTargetRegionProperties(), false)); + assertEquals("Target Region is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source registry config is not provided. + */ + @Test + public void testConverter_targetRegistryNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoTargetRegistryProperties(), false)); + assertEquals("Target Registry is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when source endpoint config is not provided. + */ + @Test + public void testConverter_targetEndpointNotProvided_throwsException(){ + converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + Exception exception = assertThrows(DataException.class, () -> converter.configure(getNoTargetEndpointProperties(), false)); + assertEquals("Target Endpoint is not provided.", exception.getMessage()); + } + + /** + * Test for Converter when no target specific config is provided + */ + @Test + public void testConverter_noTargetDetails_Succeeds() throws NoSuchFieldException, IllegalAccessException { + Map configs = getPropertiesNoTargetDetails(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + assertNotNull(converter.getSerializer()); + } + + /** + * Test Converter when it returns null given the input value is null. + */ + @Test + public void testConverter_fromConnectData_returnsByte0() throws NoSuchFieldException, IllegalAccessException { + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + assertNull(converter.fromConnectData(testTopic, expected.schema(), null)); + } + + /** + * Test Converter when serializer throws exception with Avro schema. + */ + @Test + public void testConverter_fromConnectData_serializer_avroSchema_throwsException() throws NoSuchFieldException, IllegalAccessException { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.AVRO.name(), "schemaFoo"); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(SCHEMA_REGISTRY_SCHEMA, SCHEMA_ID_FOR_TESTING); + + doReturn(USER_DATA) + .when(deserializer).getData(genericBytes); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + when(serializer.encode(testTopic, SCHEMA_REGISTRY_SCHEMA, USER_DATA)).thenThrow(new AWSSchemaRegistryException()); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when the deserializer throws exception with Avro schema. + */ + @Test + public void testConverter_fromConnectData_deserializer_avroSchema_throwsException() throws NoSuchFieldException, IllegalAccessException { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.AVRO.name(), "schemaFoo"); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(SCHEMA_REGISTRY_SCHEMA, SCHEMA_ID_FOR_TESTING); + + when((deserializer).getData(genericBytes)).thenThrow(new AWSSchemaRegistryException()); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(null, SCHEMA_REGISTRY_SCHEMA, USER_DATA); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when Avro schema is replicated. + */ + @Test + public void testConverter_fromConnectData_avroSchema_succeeds() throws NoSuchFieldException, IllegalAccessException { + String schemaDefinition = "{\"namespace\":\"com.amazonaws.services.schemaregistry.serializers.avro\",\"type\":\"record\",\"name\":\"payment\",\"fields\":[{\"name\":\"id\",\"type\":\"string\"},{\"name\":\"id_6\",\"type\":\"double\"}]}"; + Schema testSchema = new Schema(schemaDefinition, DataFormat.AVRO.name(), testTopic); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(testSchema, SCHEMA_ID_FOR_TESTING); + + doReturn(genericBytes). + when(deserializer).getData(avroBytes); + doReturn(testSchema). + when(deserializer).getSchema(avroBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(testTopic, testSchema, genericBytes); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), avroBytes), ENCODED_DATA); + } + + /** + * Test Converter when serializer throws exception with JSON schema. + */ + @Test + public void testConverter_fromConnectData_serializer_jsonSchema_throwsException() throws NoSuchFieldException, IllegalAccessException { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.JSON.name(), "schemaFoo"); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(SCHEMA_REGISTRY_SCHEMA, SCHEMA_ID_FOR_TESTING); + + doReturn(USER_DATA) + .when(deserializer).getData(genericBytes); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + when(serializer.encode(testTopic, SCHEMA_REGISTRY_SCHEMA, USER_DATA)).thenThrow(new AWSSchemaRegistryException()); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when the deserializer throws exception with JSON schema. + */ + @Test + public void testConverter_fromConnectData_deserializer_jsonSchema_throwsException() throws NoSuchFieldException, IllegalAccessException { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.JSON.name(), "schemaFoo"); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(SCHEMA_REGISTRY_SCHEMA, SCHEMA_ID_FOR_TESTING); + + when((deserializer).getData(genericBytes)).thenThrow(new AWSSchemaRegistryException()); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode("schemaFoo", SCHEMA_REGISTRY_SCHEMA, USER_DATA); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when JSON schema is replicated. + */ + @Test + public void testConverter_fromConnectData_jsonSchema_succeeds() throws NoSuchFieldException, IllegalAccessException { + String testSchemaDefinition = "{\"$id\":\"https://example.com/geographical-location.schema.json\"," + + "\"$schema\":\"http://json-schema.org/draft-07/schema#\",\"title\":\"Longitude " + + "and Latitude Values\",\"description\":\"A geographical coordinate.\"," + + "\"required\":[\"latitude\",\"longitude\"],\"type\":\"object\"," + + "\"properties\":{\"latitude\":{\"type\":\"number\",\"minimum\":-90," + + "\"maximum\":90},\"longitude\":{\"type\":\"number\",\"minimum\":-180," + + "\"maximum\":180}},\"additionalProperties\":false}"; + Schema testSchema = new Schema(testSchemaDefinition, DataFormat.JSON.name(), testTopic); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(testSchema, SCHEMA_ID_FOR_TESTING); + + doReturn(genericBytes). + when(deserializer).getData(jsonBytes); + doReturn(testSchema). + when(deserializer).getSchema(jsonBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(testTopic, testSchema, genericBytes); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), jsonBytes), ENCODED_DATA); + } + + /** + * Test Converter when message without schema is replicated. + */ + @Test + public void testConverter_fromConnectData_noSchema_succeeds() throws NoSuchFieldException, IllegalAccessException { + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + when(deserializer.getData(genericBytes)).thenThrow(new GlueSchemaRegistryIncompatibleDataException("No schema in message")); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), genericBytes), genericBytes); + } + + /** + * Test Converter when serializer throws exception with protobuf schema. + */ + @Test + public void testConverter_fromConnectData_serializer_protobufSchema_throwsException() throws NoSuchFieldException, IllegalAccessException { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.PROTOBUF.name(), "schemaFoo"); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(SCHEMA_REGISTRY_SCHEMA, SCHEMA_ID_FOR_TESTING); + + doReturn(USER_DATA) + .when(deserializer).getData(genericBytes); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + when(serializer.encode(testTopic, SCHEMA_REGISTRY_SCHEMA, USER_DATA)).thenThrow(new AWSSchemaRegistryException()); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when the deserializer throws exception with protobuf schema. + */ + @Test + public void testConverter_fromConnectData_deserializer_protobufSchema_throwsException() throws NoSuchFieldException, IllegalAccessException { + Schema SCHEMA_REGISTRY_SCHEMA = new Schema("{}", DataFormat.PROTOBUF.name(), "schemaFoo"); + Struct expected = createStructRecord(); + + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(SCHEMA_REGISTRY_SCHEMA, SCHEMA_ID_FOR_TESTING); + + when((deserializer).getData(genericBytes)).thenThrow(new AWSSchemaRegistryException()); + doReturn(SCHEMA_REGISTRY_SCHEMA) + .when(deserializer).getSchema(genericBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode("schemaFoo", SCHEMA_REGISTRY_SCHEMA, USER_DATA); + assertThrows(DataException.class, () -> converter.fromConnectData(testTopic, expected.schema(), genericBytes)); + } + + /** + * Test Converter when Protobuf schema is replicated. + */ + @Test + public void getSchema_protobuf_succeeds() throws NoSuchFieldException, IllegalAccessException, ExecutionException { + Map configs = getTestConfigs(); + Schema testSchema = new Schema("foo", DataFormat.PROTOBUF.name(), testTopic); + Struct expected = createStructRecord(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + converter.schemaDefinitionToVersionCache.put(testSchema, SCHEMA_ID_FOR_TESTING); + + doReturn(genericBytes). + when(deserializer).getData(protobufBytes); + doReturn(testSchema). + when(deserializer).getSchema(protobufBytes); + doReturn(ENCODED_DATA) + .when(serializer).encode(testTopic, testSchema, genericBytes); + assertEquals(converter.fromConnectData(testTopic, expected.schema(), protobufBytes), ENCODED_DATA); + } + + /** + * Test toConnectData when IllegalAccessException is thrown. + */ + @Test + public void toConnectData_throwsException() throws NoSuchFieldException, IllegalAccessException { + Map configs = getTestConfigs(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + assertThrows(UnsupportedOperationException.class, () -> converter.toConnectData(testTopic, genericBytes)); + } + + + + @Test + public void testCreateSchemaAndRegisterAllSchemaVersions_nullSchemaDefinition_throwsException() { + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + Schema schema = new Schema(null, DataFormat.AVRO.name(), "test-schema-name"); + + Assertions.assertThrows(NullPointerException.class, () -> converter.createSchemaAndRegisterAllSchemaVersions(schema)); + } + + @Test + public void testCreateSchemaAndRegisterAllSchemaVersions_nullSchemaName_throwsException() { + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + Schema schema = new Schema(userSchemaDefinition, DataFormat.AVRO.name(), null); + + Assertions.assertThrows(NullPointerException.class, () -> converter.createSchemaAndRegisterAllSchemaVersions(schema)); + } + + @Test + public void testCreateSchemaAndRegisterAllSchemaVersions_nullSchemaDataFormat_throwsException() { + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + Schema schema = new Schema(userSchemaDefinition,null, "test-schema-name"); + + Assertions.assertThrows(NullPointerException.class, () -> converter.createSchemaAndRegisterAllSchemaVersions(schema)); + } + + @Test + public void testCreateSchemaAndRegisterAllSchemaVersions_WhenVersionIsPresent_ReturnsIt() throws Exception { + Map configs = getTestConfigs(); + + String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); + String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); + String dataFormatName = DataFormat.AVRO.name(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + mockGetSchemaByDefinition(schemaName, registryName); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + Schema schema = new Schema(userSchemaDefinition, dataFormatName, schemaName); + UUID schemaVersionId = converter.createSchemaAndRegisterAllSchemaVersions(schema); + + assertEquals(SCHEMA_ID_FOR_TESTING, schemaVersionId); + } + + @Test + public void testCreateSchemaAndRegisterAllSchemaVersions_schemaVersionNotPresent_autoRegistersSchemaVersion() throws Exception { + Map configs = getTestConfigs(); + + String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); + String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); + String dataFormatName = DataFormat.AVRO.name(); + + Long schemaVersionNumber = 1L; + Long schemaVersionNumber2 = 2L; + SchemaId requestSchemaId = SchemaId.builder().schemaName(schemaName).registryName(registryName).build(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + mockGetSchema(schemaName, registryName); + mockCreateSchema(schemaName, dataFormatName, glueSchemaRegistryConfiguration); + mockRegisterSchemaVersion(schemaVersionNumber, requestSchemaId); + mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); + mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); + mockQuerySchemaVersionMetadata(); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + Schema schema = new Schema(userSchemaDefinition, dataFormatName, schemaName); + UUID schemaVersionId = converter.createSchemaAndRegisterAllSchemaVersions(schema); + + assertEquals(SCHEMA_ID_FOR_TESTING, schemaVersionId); + } + +// @Test +// public void testCreateSchemaAndRegisterAllSchemaVersions_OnUnknownException_ThrowsException() throws Exception { +// Map configs = getConfigsWithAutoRegistrationSetting(true); +// +// String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); +// String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); +// String dataFormatName = DataFormat.AVRO.name(); +// +// GlueSchemaRegistryConfiguration awsSchemaRegistrySerDeConfigs = new GlueSchemaRegistryConfiguration(configs); +// awsSchemaRegistryClient = +// configureAWSSchemaRegistryClientWithSerdeConfig(awsSchemaRegistryClient, awsSchemaRegistrySerDeConfigs); +// +// mockGetSchemaByDefinition_ThrowException(schemaName, registryName); +// +// schemaByDefinitionFetcher = new SchemaByDefinitionFetcher(awsSchemaRegistryClient, awsSchemaRegistrySerDeConfigs); +// +// Exception exception = assertThrows(AWSSchemaRegistryException.class, +// () -> schemaByDefinitionFetcher +// .getORRegisterSchemaVersionIdV2(userSchemaDefinition, schemaName, dataFormatName, Compatibility.FORWARD, getMetadata())); +// assertTrue( +// exception.getMessage().contains("Exception occurred while fetching or registering schema definition")); +// } + + @Test + public void testCreateSchemaAndRegisterAllSchemaVersions_schemaNotPresent_autoCreatesSchemaAndRegisterSchemaVersions_retrieveFromCache() throws Exception { + Map configs = getTestConfigs(); + + String schemaName = configs.get(AWSSchemaRegistryConstants.SCHEMA_NAME); + String registryName = configs.get(AWSSchemaRegistryConstants.REGISTRY_NAME); + String dataFormatName = DataFormat.AVRO.name(); + Long schemaVersionNumber = 1L; + Long schemaVersionNumber2 = 2L; + SchemaId requestSchemaId = SchemaId.builder().schemaName(schemaName).registryName(registryName).build(); + + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration = new GlueSchemaRegistryConfiguration(configs); + sourceSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(sourceSchemaRegistryClient, glueSchemaRegistryConfiguration); + targetSchemaRegistryClient = configureAWSSchemaRegistryClientWithSerdeConfig(targetSchemaRegistryClient, glueSchemaRegistryConfiguration); + + GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = getSchemaByDefinitionRequest(schemaName, registryName); + mockGetSchema(schemaName, registryName); + mockCreateSchema(schemaName, dataFormatName, glueSchemaRegistryConfiguration); + mockRegisterSchemaVersion(schemaVersionNumber, requestSchemaId); + mockGetSchemaVersions(schemaVersionNumber, schemaVersionNumber2); + mockListSchemaVersions(schemaName, registryName, schemaVersionNumber, schemaVersionNumber2); + mockQuerySchemaVersionMetadata(); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(sourceSchemaRegistryClient, targetSchemaRegistryClient, deserializer, serializer); + converter.configure(configs, false); + + LoadingCache cache = converter.schemaDefinitionToVersionCache; + + Schema expectedSchema = new Schema(userSchemaDefinition, dataFormatName, schemaName); + Schema expectedSchema2 = new Schema(userSchemaDefinition2, dataFormatName, schemaName); + + //Ensure cache is empty to start with. + assertEquals(0, cache.size()); + + //First call will create schema and register other schema versions + Schema schema = new Schema(userSchemaDefinition, dataFormatName, schemaName); + converter.createSchemaAndRegisterAllSchemaVersions(schema); + + //Ensure cache is populated + assertEquals(2, cache.size()); + + //Ensure corresponding UUID matches with schema + assertEquals(SCHEMA_ID_FOR_TESTING, cache.get(expectedSchema)); + assertEquals(SCHEMA_ID_FOR_TESTING2, cache.get(expectedSchema2)); + + //Second call will be served from cache + converter.createSchemaAndRegisterAllSchemaVersions(schema); + + //Third call will be served from cache + schema = new Schema(userSchemaDefinition2, dataFormatName, schemaName); + converter.createSchemaAndRegisterAllSchemaVersions(schema); + + //Ensure cache is populated + assertEquals(2, cache.size()); + + //Ensure only 1 call happened. + verify(mockGlueClient, times(1)).getSchemaByDefinition(getSchemaByDefinitionRequest); + } + + private void mockGetSchemaByDefinition_ThrowException(String schemaName, String registryName) { + GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient + .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); + + AWSSchemaRegistryException awsSchemaRegistryException = + new AWSSchemaRegistryException(new RuntimeException("Unknown")); + + when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)).thenThrow(awsSchemaRegistryException); + } + + private GetSchemaByDefinitionRequest getSchemaByDefinitionRequest(String schemaName, String registryName) { + return awsSchemaRegistryClient + .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); + } + + private void mockQuerySchemaVersionMetadata() { + QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest = QuerySchemaVersionMetadataRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .build(); + + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse = QuerySchemaVersionMetadataResponse + .builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .metadataInfoMap(new HashMap<>()) + .build(); + + QuerySchemaVersionMetadataRequest querySchemaVersionMetadataRequest2 = QuerySchemaVersionMetadataRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .build(); + + QuerySchemaVersionMetadataResponse querySchemaVersionMetadataResponse2 = QuerySchemaVersionMetadataResponse + .builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .metadataInfoMap(new HashMap<>()) + .build(); + + when(mockGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest)).thenReturn(querySchemaVersionMetadataResponse); + when(mockGlueClient.querySchemaVersionMetadata(querySchemaVersionMetadataRequest2)).thenReturn(querySchemaVersionMetadataResponse2); + } + + private void mockGetSchemaVersions(Long schemaVersionNumber, Long schemaVersionNumber2) { + GetSchemaVersionRequest getSchemaVersionRequest = GetSchemaVersionRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()).build(); + + GetSchemaVersionResponse getSchemaVersionResponse = GetSchemaVersionResponse.builder() + .schemaDefinition(userSchemaDefinition) + .versionNumber(schemaVersionNumber) + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .dataFormat(DataFormat.AVRO) + .status(String.valueOf(AWSSchemaRegistryConstants.SchemaVersionStatus.AVAILABLE)) + .build(); + + when(mockGlueClient.getSchemaVersion(getSchemaVersionRequest)).thenReturn(getSchemaVersionResponse); + + GetSchemaVersionRequest getSchemaVersionRequest2 = GetSchemaVersionRequest.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()).build(); + + GetSchemaVersionResponse getSchemaVersionResponse2 = GetSchemaVersionResponse.builder() + .schemaDefinition(userSchemaDefinition2) + .versionNumber(schemaVersionNumber2) + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .dataFormat(DataFormat.AVRO) + .status(String.valueOf(AWSSchemaRegistryConstants.SchemaVersionStatus.AVAILABLE)) + .build(); + + when(mockGlueClient.getSchemaVersion(getSchemaVersionRequest2)).thenReturn(getSchemaVersionResponse2); + } + + private void mockGetSchema(String schemaName, String registryName) { + GetSchemaRequest getSchemaRequest = GetSchemaRequest.builder() + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) + .build(); + + GetSchemaResponse getSchemaResponse = GetSchemaResponse.builder() + .compatibility(Compatibility.FORWARD) + .schemaName(schemaName) + .dataFormat(DataFormat.AVRO) + .registryName(registryName) + .schemaStatus(SchemaStatus.AVAILABLE) + .build(); + + when(mockGlueClient.getSchema(getSchemaRequest)).thenReturn(getSchemaResponse); + } + + private void mockCreateSchema(String schemaName, String dataFormatName, GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration) { + CreateSchemaResponse createSchemaResponse = CreateSchemaResponse.builder() + .schemaName(schemaName) + .dataFormat(dataFormatName) + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .build(); + CreateSchemaRequest createSchemaRequest = CreateSchemaRequest.builder() + .dataFormat(DataFormat.AVRO) + .description(glueSchemaRegistryConfiguration.getDescription()) + .schemaName(schemaName) + .schemaDefinition(userSchemaDefinition) + .compatibility(Compatibility.FORWARD) + .tags(glueSchemaRegistryConfiguration.getTags()) + .registryId(RegistryId.builder().registryName(glueSchemaRegistryConfiguration.getRegistryName()).build()) + .build(); + + when(mockGlueClient.createSchema(createSchemaRequest)).thenReturn(createSchemaResponse); + } + + private void mockListSchemaVersions(String schemaName, String registryName, Long schemaVersionNumber, Long schemaVersionNumber2) { + ListSchemaVersionsResponse listSchemaVersionsResponse = ListSchemaVersionsResponse.builder() + .schemas(SchemaVersionListItem. + builder(). + schemaArn("test/"+ schemaName). + schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()). + versionNumber(schemaVersionNumber). + status("CREATED"). + build(), + SchemaVersionListItem. + builder(). + schemaArn("test/"+ schemaName). + schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()). + versionNumber(schemaVersionNumber2). + status("CREATED"). + build() + ) + .nextToken(null) + .build(); + ListSchemaVersionsRequest listSchemaVersionsRequest = ListSchemaVersionsRequest.builder() + .schemaId(SchemaId.builder().schemaName(schemaName).registryName(registryName).build()) + .build(); + + when(mockGlueClient.listSchemaVersions(listSchemaVersionsRequest)).thenReturn(listSchemaVersionsResponse); + } + + private void mockRegisterSchemaVersion(Long schemaVersionNumber, SchemaId requestSchemaId) { + RegisterSchemaVersionRequest registerSchemaVersionRequest = RegisterSchemaVersionRequest.builder() + .schemaDefinition(userSchemaDefinition2) + .schemaId(requestSchemaId) + .build(); + RegisterSchemaVersionResponse registerSchemaVersionResponse = RegisterSchemaVersionResponse.builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING2.toString()) + .versionNumber(schemaVersionNumber) + .build(); + when(mockGlueClient.registerSchemaVersion(registerSchemaVersionRequest)) + .thenReturn(registerSchemaVersionResponse); + } + + private void mockGetSchemaByDefinition(String schemaName, String registryName) { + GetSchemaByDefinitionRequest getSchemaByDefinitionRequest = awsSchemaRegistryClient + .buildGetSchemaByDefinitionRequest(userSchemaDefinition, schemaName, registryName); + + GetSchemaByDefinitionResponse getSchemaByDefinitionResponse = + GetSchemaByDefinitionResponse + .builder() + .schemaVersionId(SCHEMA_ID_FOR_TESTING.toString()) + .status(String.valueOf(AWSSchemaRegistryConstants.SchemaVersionStatus.AVAILABLE)) + .build(); + + when(mockGlueClient.getSchemaByDefinition(getSchemaByDefinitionRequest)) + .thenReturn(getSchemaByDefinitionResponse); + } + + private Map getMetadata() { + Map metadata = new HashMap<>(); + metadata.put("event-source-1", "topic1"); + metadata.put("event-source-2", "topic2"); + metadata.put("event-source-3", "topic3"); + metadata.put("event-source-4", "topic4"); + metadata.put("event-source-5", "topic5"); + return metadata; + } + + /** + * To create a map of configurations without source region. + * + * @return a map of configurations + */ + private Map getNoSourceRegionProperties() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + + return props; + } + + /** + * To create a map of configurations without source registry. + * + * @return a map of configurations + */ + private Map getNoSourceRegistryProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + + return props; + } + + /** + * To create a map of configurations without target registry. + * + * @return a map of configurations + */ + private Map getNoTargetRegistryProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + + return props; + } + + /** + * To create a map of configurations without source endpoint. + * + * @return a map of configurations + */ + private Map getNoSourceEndpointProperties() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create a map of configurations without source endpoint. + * + * @return a map of configurations + */ + private Map getNoTargetEndpointProperties() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create a map of configurations without target region. + * + * @return a map of configurations + */ + private Map getNoTargetRegionProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + + return props; + } + + /** + * To create a map of configurations without target region, target endpoint and target registry name + * but is replaced by the provided region, endpoint and registry name config. + * + * @return a map of configurations + */ + private Map getPropertiesNoTargetDetails() { + Map props = new HashMap<>(); + + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-west-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://test"); + + return props; + } + + /** + * To create a map of configurations. + * + * @return a map of configurations + */ + private Map getTestConfigs() { + Map localConfigs = new HashMap<>(); + localConfigs.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://test"); + localConfigs.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://test"); + localConfigs.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + localConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-west-2"); + localConfigs.put(AWSSchemaRegistryConstants.COMPATIBILITY_SETTING, Compatibility.FORWARD.toString()); + localConfigs.put(AWSSchemaRegistryConstants.SCHEMA_NAME, "User-Topic"); + localConfigs.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "User-Topic"); + localConfigs.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "User-Topic"); + return localConfigs; + } + + /** + * To create Connect Struct record. + * + * @return Connect Struct + */ + private Struct createStructRecord() { + org.apache.kafka.connect.data.Schema schema = SchemaBuilder.struct() + .build(); + return new Struct(schema); + } + + private AWSSchemaRegistryClient configureAWSSchemaRegistryClientWithSerdeConfig( + AWSSchemaRegistryClient awsSchemaRegistryClient, + GlueSchemaRegistryConfiguration glueSchemaRegistryConfiguration) + throws NoSuchFieldException, IllegalAccessException { + Field serdeConfigField = AWSSchemaRegistryClient.class.getDeclaredField("glueSchemaRegistryConfiguration"); + serdeConfigField.setAccessible(true); + serdeConfigField.set(awsSchemaRegistryClient, glueSchemaRegistryConfiguration); + + return awsSchemaRegistryClient; + } +} diff --git a/integration-tests/docker-compose.yml b/integration-tests/docker-compose.yml index a7d133aa..bb5718b3 100644 --- a/integration-tests/docker-compose.yml +++ b/integration-tests/docker-compose.yml @@ -1,10 +1,13 @@ -version: '2' - services: - zookeeper: + zookeeper-kafka: image: 'public.ecr.aws/bitnami/zookeeper:latest' - ports: - - '2181:2182' + container_name: zk-kafka + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + zookeeper-kafka-target: + image: 'public.ecr.aws/bitnami/zookeeper:latest' + container_name: zk-kafka-target environment: - ALLOW_ANONYMOUS_LOGIN=yes @@ -12,15 +15,48 @@ services: image: 'public.ecr.aws/bitnami/kafka:2.8' ports: - '9092:9092' - links: - - zookeeper - container_name: local_kafka + container_name: kafka environment: - KAFKA_BROKER_ID=1 - - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-kafka:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT + - KAFKA_CFG_LISTENERS=CLIENT://:29092,EXTERNAL://:9092 + - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka:29092,EXTERNAL://localhost:9092 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + + kafka-target: + image: 'public.ecr.aws/bitnami/kafka:2.8' + ports: + - '9093:9093' + container_name: kafka-target + environment: + - KAFKA_BROKER_ID=2 + - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-kafka-target:2181 + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT + - KAFKA_CFG_LISTENERS=CLIENT://:29092,EXTERNAL://:9093 + - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka-target:29092,EXTERNAL://localhost:9093 + - KAFKA_CFG_INTER_BROKER_LISTENER_NAME=CLIENT + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + + mirrormaker: + build: ./mirrormaker + container_name: mirrormaker + environment: - ALLOW_PLAINTEXT_LISTENER=yes + - SOURCE=kafka:29092 + - DESTINATION=kafka-target:29092 + - TOPICS=^SchemaReplicationTests.* + - ACLS_ENABLED=false + - AWS_PROFILE=default + volumes: + - glue-schema-registry-plugins:/opt/plugins + - "${HOME}/.aws/:/.aws:ro" + depends_on: + - kafka + - kafka-target localstack: container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" @@ -36,3 +72,12 @@ services: volumes: - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - "/var/run/docker.sock:/var/run/docker.sock" + +volumes: + glue-schema-registry-plugins: + driver: local + driver_opts: + type: none + device: ../ + o: bind + diff --git a/integration-tests/mirrormaker/Dockerfile b/integration-tests/mirrormaker/Dockerfile new file mode 100644 index 00000000..ad44e212 --- /dev/null +++ b/integration-tests/mirrormaker/Dockerfile @@ -0,0 +1,39 @@ +FROM public.ecr.aws/bitnami/kafka:2.8 +USER root +RUN install_packages gettext + +RUN mkdir -p /opt/plugins +RUN chown 1234 /opt/plugins + +RUN mkdir -p ~/.aws +RUN chmod 1234 ~/.aws + +# Install the AWS CLI +RUN \ + apt-get update -y && \ + apt-get install -y wget vim python3 unzip python-is-python3 python3-venv && \ + wget "s3.amazonaws.com/aws-cli/awscli-bundle.zip" -O "awscli-bundle.zip" && \ + unzip awscli-bundle.zip && \ + ./awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws && \ + rm awscli-bundle.zip && \ + rm -rf awscli-bundle + +ADD ./mm2-configs/connect-standalone.properties /opt/mm2/connect-standalone.properties +ADD ./mm2-configs/mirror-checkpoint-connector.properties /opt/mm2/mirror-checkpoint-connector.properties +ADD ./mm2-configs/mirror-heartbeat-connector.properties /opt/mm2/mirror-heartbeat-connector.properties +ADD ./mm2-configs/mirror-source-connector.properties /opt/mm2/mirror-source-connector.properties + +ADD ./run.sh /opt/mm2/run.sh +RUN chmod +x /opt/mm2/run.sh + +RUN mkdir -p /var/run/mm2 +RUN chown 1234 /var/run/mm2 + +ENV TOPICS .* +ENV SOURCE "localhost:9092" +ENV DESTINATION "localhost:9093" +ENV ACLS_ENABLED "false" +ENV AWS_PROFILE "default" + +USER 1234 +CMD /opt/mm2/run.sh diff --git a/integration-tests/mirrormaker/mm2-configs/connect-standalone.properties b/integration-tests/mirrormaker/mm2-configs/connect-standalone.properties new file mode 100644 index 00000000..97e2fed5 --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/connect-standalone.properties @@ -0,0 +1,12 @@ +bootstrap.servers=${DESTINATION} + +key.converter=org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter + +key.converter.schemas.enable=true +value.converter.schemas.enable=true + +offset.storage.file.filename=/tmp/connect.offsets +offset.flush.interval.ms=10000 + +plugin.path=/opt/plugins/ diff --git a/integration-tests/mirrormaker/mm2-configs/mirror-checkpoint-connector.properties b/integration-tests/mirrormaker/mm2-configs/mirror-checkpoint-connector.properties new file mode 100644 index 00000000..10792369 --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/mirror-checkpoint-connector.properties @@ -0,0 +1,13 @@ +name=mm2-cpc +connector.class=org.apache.kafka.connect.mirror.MirrorCheckpointConnector +clusters=src,dst +source.cluster.alias=src +target.cluster.alias=dst +source.cluster.bootstrap.servers=${SOURCE} +target.cluster.bootstrap.servers=${DESTINATION} +tasks.max=1 +key.converter=org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter +replication.factor=1 +checkpoints.topic.replication.factor=1 +emit.checkpoints.interval.seconds=20 \ No newline at end of file diff --git a/integration-tests/mirrormaker/mm2-configs/mirror-heartbeat-connector.properties b/integration-tests/mirrormaker/mm2-configs/mirror-heartbeat-connector.properties new file mode 100644 index 00000000..176bd923 --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/mirror-heartbeat-connector.properties @@ -0,0 +1,13 @@ +name=mm2-hbc +connector.class=org.apache.kafka.connect.mirror.MirrorHeartbeatConnector +clusters=src,dst +source.cluster.alias=src +target.cluster.alias=dst +source.cluster.bootstrap.servers=${SOURCE} +target.cluster.bootstrap.servers=${DESTINATION} +tasks.max=1 +key.converter= org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter +replication.factor=1 +heartbeats.topic.replication.factor=1 +emit.heartbeats.interval.seconds=20 \ No newline at end of file diff --git a/integration-tests/mirrormaker/mm2-configs/mirror-source-connector.properties b/integration-tests/mirrormaker/mm2-configs/mirror-source-connector.properties new file mode 100644 index 00000000..44a304ff --- /dev/null +++ b/integration-tests/mirrormaker/mm2-configs/mirror-source-connector.properties @@ -0,0 +1,36 @@ +name=mm2-msc +connector.class=org.apache.kafka.connect.mirror.MirrorSourceConnector +clusters=src,dst +source.cluster.alias=src +target.cluster.alias=dst +source.cluster.bootstrap.servers=${SOURCE} +target.cluster.bootstrap.servers=${DESTINATION} +topics=${TOPICS} +tasks.max=3 +key.converter=com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.AWSGlueCrossRegionSchemaReplicationConverter +value.converter=com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.AWSGlueCrossRegionSchemaReplicationConverter +replication.factor=1 +offset-syncs.topic.replication.factor=1 +sync.topic.acls.enabled=${ACLS_ENABLED} +refresh.topics.interval.seconds=20 +refresh.groups.interval.seconds=20 +consumer.group.id=mm2-msc-cons +producer.enable.idempotence=true +key.converter.schemas.enable=false +value.converter.schemas.enable=true +key.converter.source.endpoint=https://glue.us-east-1.amazonaws.com +key.converter.source.region=us-east-1 +key.converter.target.endpoint=https://glue.us-east-2.amazonaws.com +key.converter.target.region=us-east-2 +key.converter.source.registry.name=default-registry +key.converter.target.registry.name=default-registry +key.converter.replicateSchemaVersionCount=30 +value.converter.source.endpoint=https://glue.us-east-1.amazonaws.com +value.converter.source.region=us-east-1 +value.converter.target.endpoint=https://glue.us-east-2.amazonaws.com +value.converter.target.region=us-east-2 +value.converter.source.registry.name=default-registry +value.converter.target.registry.name=default-registry +value.converter.replicateSchemaVersionCount=30 +errors.log.enable=true +errors.log.include.messages=true \ No newline at end of file diff --git a/integration-tests/mirrormaker/run.sh b/integration-tests/mirrormaker/run.sh new file mode 100644 index 00000000..8d6c516f --- /dev/null +++ b/integration-tests/mirrormaker/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +envsubst < /opt/mm2/connect-standalone.properties > /var/run/mm2/connect-standalone.properties +envsubst < /opt/mm2/mirror-checkpoint-connector.properties > /var/run/mm2/mirror-checkpoint-connector.properties +envsubst < /opt/mm2/mirror-heartbeat-connector.properties > /var/run/mm2/mirror-heartbeat-connector.properties +envsubst < /opt/mm2/mirror-source-connector.properties > /var/run/mm2/mirror-source-connector.properties + +/opt/bitnami/kafka/bin/connect-standalone.sh \ + /var/run/mm2/connect-standalone.properties \ + /var/run/mm2/mirror-heartbeat-connector.properties \ + /var/run/mm2/mirror-checkpoint-connector.properties \ + /var/run/mm2/mirror-source-connector.properties \ No newline at end of file diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 05ea2450..57f8410e 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -66,6 +66,11 @@ schema-registry-serde ${parent.version} + + software.amazon.glue + schema-registry-cross-region-kafkaconnect-converter + ${parent.version} + software.amazon.glue schema-registry-kafkastreams-serde diff --git a/integration-tests/run-local-tests.sh b/integration-tests/run-local-tests.sh old mode 100644 new mode 100755 index 66c797cf..a4ba9218 --- a/integration-tests/run-local-tests.sh +++ b/integration-tests/run-local-tests.sh @@ -115,9 +115,10 @@ cleanUpConnectFiles() { cleanUpDockerResources || true # Start Kafka using docker command asynchronously -docker-compose up --no-attach localstack & -sleep 10 -## Run mvn tests for Kafka and Kinesis Platforms +docker compose up --no-attach localstack & +## Pause to allow docker build to complete +sleep 60 +## Run mvn tests for Kafka, Kinesis Platforms and Schema Replication cd .. && mvn --file integration-tests/pom.xml verify -Psurefire -X && cd integration-tests cleanUpDockerResources @@ -131,7 +132,7 @@ downloadMongoDBConnector copyGSRConverters runConnectTests() { - docker-compose up --no-attach localstack & + docker compose up --no-attach localstack & setUpMongoDBLocal startKafkaConnectTasks ${1} echo "Waiting for Sink task to pick up data.." diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java index aff3527b..548363ba 100644 --- a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/properties/GlueSchemaRegistryConnectionProperties.java @@ -19,4 +19,8 @@ public interface GlueSchemaRegistryConnectionProperties { // Glue Service Endpoint String REGION = Regions.getCurrentRegion() == null ? "us-east-2" : Regions.getCurrentRegion().getName().toLowerCase(); String ENDPOINT = String.format("https://glue.%s.amazonaws.com", REGION); + String SRC_REGION = Regions.getCurrentRegion() == null ? "us-east-1" : Regions.getCurrentRegion().getName().toLowerCase(); + String SRC_ENDPOINT = String.format("https://glue.%s.amazonaws.com", SRC_REGION); + String DEST_REGION = "us-east-2"; + String DEST_ENDPOINT = String.format("https://glue.%s.amazonaws.com", DEST_REGION); } diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/AWSGlueCrossRegionSchemaReplicationIntegrationTest.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/AWSGlueCrossRegionSchemaReplicationIntegrationTest.java new file mode 100644 index 00000000..af92fb99 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/AWSGlueCrossRegionSchemaReplicationIntegrationTest.java @@ -0,0 +1,337 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.AWSGlueCrossRegionSchemaReplicationConverter; +import com.amazonaws.services.crossregion.schemaregistry.kafkaconnect.SchemaReplicationSchemaRegistryConstants; +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryKafkaDeserializer; +import com.amazonaws.services.schemaregistry.integrationtests.generators.*; +import com.amazonaws.services.schemaregistry.integrationtests.properties.GlueSchemaRegistryConnectionProperties; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import com.amazonaws.services.schemaregistry.utils.AvroRecordType; +import com.amazonaws.services.schemaregistry.utils.ProtobufMessageType; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.Compatibility; +import software.amazon.awssdk.services.glue.model.DataFormat; +import software.amazon.awssdk.services.glue.model.DeleteSchemaRequest; +import software.amazon.awssdk.services.glue.model.SchemaId; + +import java.net.URI; +import java.net.URISyntaxException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * The test class for schema replication related tests for Glue Schema Registry + */ +@Slf4j +public class AWSGlueCrossRegionSchemaReplicationIntegrationTest { + private static final String SRC_CLUSTER_ALIAS = "src"; + private static final String TOPIC_NAME_PREFIX = "SchemaReplicationTests"; + private static final String TOPIC_NAME_PREFIX_CONVERTER = "SchemaRegistryTests"; + private static final String SCHEMA_REGISTRY_SRC_ENDPOINT_OVERRIDE = GlueSchemaRegistryConnectionProperties.SRC_ENDPOINT; + private static final String SCHEMA_REGISTRY_DEST_ENDPOINT_OVERRIDE = GlueSchemaRegistryConnectionProperties.DEST_ENDPOINT; + private static final String SRC_REGION = GlueSchemaRegistryConnectionProperties.SRC_REGION; + private static final String DEST_REGION = GlueSchemaRegistryConnectionProperties.DEST_REGION; + private static final String RECORD_TYPE = "GENERIC_RECORD"; + private static final List COMPATIBILITIES = Compatibility.knownValues() + .stream() + .filter(c -> c.toString().equals("NONE") + || c.toString().equals("BACKWARD")) + .collect(Collectors.toList()); + private static LocalKafkaClusterHelper srcKafkaClusterHelper = new LocalKafkaClusterHelper(); + private static LocalKafkaClusterHelper destKafkaClusterHelper = new LocalKafkaClusterHelper(); + private static AwsCredentialsProvider awsCredentialsProvider = DefaultCredentialsProvider.builder() + .build(); + private static List schemasToCleanUp = new ArrayList<>(); + private final TestDataGeneratorFactory testDataGeneratorFactory = new TestDataGeneratorFactory(); + + private static Stream testArgumentsProvider() { + Stream.Builder argumentBuilder = Stream.builder(); + for (DataFormat dataFormat : DataFormat.knownValues()) { + //TODO: Remove if logic + //if (dataFormat == DataFormat.PROTOBUF) { + for (Compatibility compatibility : COMPATIBILITIES) { + //if (compatibility == Compatibility.BACKWARD) { + for (AWSSchemaRegistryConstants.COMPRESSION compression : + AWSSchemaRegistryConstants.COMPRESSION.values()) { + argumentBuilder.add(Arguments.of(dataFormat, RECORD_TYPE, compatibility, compression)); + } + //} + } + //} + } + return argumentBuilder.build(); + } + + private static Pair createAndGetKafkaHelper(String topicNamePrefix) throws Exception { + final String topic = String.format("%s-%s-%s", topicNamePrefix, Instant.now() + .atOffset(ZoneOffset.UTC) + .format(DateTimeFormatter.ofPattern("yy-MM-dd-HH-mm")), RandomStringUtils.randomAlphanumeric(4)); + + final String srcBootstrapString = srcKafkaClusterHelper.getSrcClusterBootstrapString(); + final KafkaHelper kafkaHelper = new KafkaHelper(srcBootstrapString, srcKafkaClusterHelper.getOrCreateCluster()); + kafkaHelper.createTopic(topic, srcKafkaClusterHelper.getNumberOfPartitions(), srcKafkaClusterHelper.getReplicationFactor()); + return Pair.of(topic, kafkaHelper); + } + + @Test + public void testProduceConsumeWithoutSchemaRegistry() throws Exception { + log.info("Starting the test for producing and consuming messages via Kafka ..."); + + final Pair srcKafkaHelperPair = createAndGetKafkaHelper(TOPIC_NAME_PREFIX); + String topic = srcKafkaHelperPair.getKey(); + KafkaHelper srcKafkaHelper = srcKafkaHelperPair.getValue(); + KafkaHelper destKafkaHelper = new KafkaHelper(destKafkaClusterHelper.getDestClusterBootstrapString(), destKafkaClusterHelper.getOrCreateCluster()); + + final int recordsProduced = 20; + srcKafkaHelper.doProduce(topic, recordsProduced); + + //Delay added to allow MM2 copy the data to destination cluster + //before consuming the records from the destination cluster + Thread.sleep(10000); + + ConsumerProperties consumerProperties = ConsumerProperties.builder() + .topicName(String.format("%s.%s",SRC_CLUSTER_ALIAS, topic)) + .build(); + + int recordsConsumed = destKafkaHelper.doConsume(consumerProperties); + log.info("Producing {} records, and consuming {} records", recordsProduced, recordsConsumed); + + assertEquals(recordsConsumed, recordsProduced); + log.info("Finish the test for producing/consuming messages via Kafka."); + } + + @ParameterizedTest + @MethodSource("testArgumentsProvider") + public void testProduceConsumeWithSchemaRegistryForAllThreeDataFormatsWithMM2(final DataFormat dataFormat, + final AvroRecordType avroRecordType, + final Compatibility compatibility) throws Exception { + log.info("Starting the test for producing and consuming {} messages via Kafka ...", dataFormat.name()); + final Pair srcKafkaHelperPair = createAndGetKafkaHelper(TOPIC_NAME_PREFIX); + String topic = srcKafkaHelperPair.getKey(); + KafkaHelper srcKafkaHelper = srcKafkaHelperPair.getValue(); + KafkaHelper destKafkaHelper = new KafkaHelper(destKafkaClusterHelper.getDestClusterBootstrapString(), destKafkaClusterHelper.getOrCreateCluster()); + + TestDataGenerator testDataGenerator = testDataGeneratorFactory.getInstance( + TestDataGeneratorType.valueOf(dataFormat, avroRecordType, compatibility)); + List records = testDataGenerator.createRecords(); + + String schemaName = String.format("%s-%s", topic, dataFormat.name()); + schemasToCleanUp.add(schemaName); + + ProducerProperties producerProperties = ProducerProperties.builder() + .topicName(topic) + .schemaName(schemaName) + .dataFormat(dataFormat.name()) + .compatibilityType(compatibility.name()) + .autoRegistrationEnabled("true") + .build(); + + List> producerRecords = + srcKafkaHelper.doProduceRecords(producerProperties, records); + + //Delay added to allow MM2 copy the data to destination cluster + //before consuming the records from the destination cluster + Thread.sleep(30000); + + ConsumerProperties.ConsumerPropertiesBuilder consumerPropertiesBuilder = ConsumerProperties.builder() + .topicName(String.format("%s.%s",SRC_CLUSTER_ALIAS, topic)); + + consumerPropertiesBuilder.protobufMessageType(ProtobufMessageType.DYNAMIC_MESSAGE.getName()); + consumerPropertiesBuilder.avroRecordType(avroRecordType.getName()); // Only required for the case of AVRO + + List> consumerRecords = destKafkaHelper.doConsumeRecords(consumerPropertiesBuilder.build()); + + assertRecordsEquality(producerRecords, consumerRecords); + log.info("Finished test for producing/consuming {} messages via Kafka.", dataFormat.name()); + } + + @ParameterizedTest + @MethodSource("testArgumentsProvider") + public void testProduceConsumeWithSchemaRegistryForAllThreeDataFormatsWithConverter(final DataFormat dataFormat, + final AvroRecordType avroRecordType, + final Compatibility compatibility) throws Exception { + log.info("Starting the test for producing and consuming {} messages via Kafka ...", dataFormat.name()); + final Pair srcKafkaHelperPair = createAndGetKafkaHelper(TOPIC_NAME_PREFIX_CONVERTER); + String topic = srcKafkaHelperPair.getKey(); + KafkaHelper srcKafkaHelper = srcKafkaHelperPair.getValue(); + + TestDataGenerator testDataGenerator = testDataGeneratorFactory.getInstance( + TestDataGeneratorType.valueOf(dataFormat, avroRecordType, compatibility)); + List records = testDataGenerator.createRecords(); + + String schemaName = String.format("%s-%s", topic, dataFormat.name()); + schemasToCleanUp.add(schemaName); + + ProducerProperties producerProperties = ProducerProperties.builder() + .topicName(topic) + .schemaName(schemaName) + .dataFormat(dataFormat.name()) + .compatibilityType(compatibility.name()) + .autoRegistrationEnabled("true") + .build(); + + List> producerRecords = + srcKafkaHelper.doProduceRecords(producerProperties, records); + + ConsumerProperties.ConsumerPropertiesBuilder consumerPropertiesBuilder = ConsumerProperties.builder() + .topicName(topic); + + consumerPropertiesBuilder.protobufMessageType(ProtobufMessageType.DYNAMIC_MESSAGE.getName()); + consumerPropertiesBuilder.avroRecordType(avroRecordType.getName()); // Only required for the case of AVRO + + List> consumerRecords = srcKafkaHelper.doConsumeRecordsWithByteArrayDeserializer(consumerPropertiesBuilder.build()); + + AWSGlueCrossRegionSchemaReplicationConverter converter = new AWSGlueCrossRegionSchemaReplicationConverter(); + converter.configure(getTestProperties(), false); + + GlueSchemaRegistryKafkaDeserializer deserializer = new GlueSchemaRegistryKafkaDeserializer( + DefaultCredentialsProvider.builder().build(), + getTestProperties()); + + List consumerRecordsDeserialized = new ArrayList<>(); + + for (ConsumerRecord record: consumerRecords) { + byte[] serializedData = converter.fromConnectData(topic, null, record.value()); + Object deserializedData = deserializer.deserialize(topic, serializedData); + consumerRecordsDeserialized.add(deserializedData); + } + + assertRecordsEqualityV2(producerRecords, consumerRecordsDeserialized); + log.info("Finished test for producing/consuming {} messages via Kafka.", dataFormat.name()); + } + + private Map getTestProperties() { + Map props = new HashMap<>(); + + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_REGION, "us-east-1"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_REGION, "us-east-2"); + props.put(AWSSchemaRegistryConstants.AWS_REGION, "us-east-2"); + props.put(SchemaReplicationSchemaRegistryConstants.SOURCE_REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.TARGET_REGISTRY_NAME, "default-registry"); + props.put(AWSSchemaRegistryConstants.REGISTRY_NAME, "default-registry"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_SOURCE_ENDPOINT, "https://glue.us-east-1.amazonaws.com"); + props.put(SchemaReplicationSchemaRegistryConstants.AWS_TARGET_ENDPOINT, "https://glue.us-east-2.amazonaws.com"); + props.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, "https://glue.us-east-2.amazonaws.com"); + props.put(SchemaReplicationSchemaRegistryConstants.REPLICATE_SCHEMA_VERSION_COUNT, 100); + props.put(AWSSchemaRegistryConstants.AVRO_RECORD_TYPE, AvroRecordType.GENERIC_RECORD.getName()); + + return props; + } + + @AfterAll + public static void tearDown() throws URISyntaxException { + log.info("Starting Clean-up of schemas created with GSR."); + GlueClient glueClientSrc = GlueClient.builder() + .credentialsProvider(awsCredentialsProvider) + .region(Region.of(SRC_REGION)) + .endpointOverride(new URI(SCHEMA_REGISTRY_SRC_ENDPOINT_OVERRIDE)) + .httpClient(UrlConnectionHttpClient.builder() + .build()) + .build(); + GlueClient glueClientDest = GlueClient.builder() + .credentialsProvider(awsCredentialsProvider) + .region(Region.of(DEST_REGION)) + .endpointOverride(new URI(SCHEMA_REGISTRY_DEST_ENDPOINT_OVERRIDE)) + .httpClient(UrlConnectionHttpClient.builder() + .build()) + .build(); + + for (String schemaName : schemasToCleanUp) { + log.info("Cleaning up schema {}..", schemaName); + DeleteSchemaRequest deleteSchemaRequest = DeleteSchemaRequest.builder() + .schemaId(SchemaId.builder() + .registryName("default-registry") + .schemaName(schemaName) + .build()) + .build(); + + glueClientSrc.deleteSchema(deleteSchemaRequest); + glueClientDest.deleteSchema(deleteSchemaRequest); + } + + log.info("Finished Cleaning up {} schemas created with GSR.", schemasToCleanUp.size()); + } + + private void assertRecordsEquality(List> producerRecords, + List> consumerRecords) { + assertThat(producerRecords.size(), is(equalTo(consumerRecords.size()))); + Map producerRecordsMap = producerRecords.stream() + .collect(Collectors.toMap(ProducerRecord::key, ProducerRecord::value)); + + for (ConsumerRecord consumerRecord : consumerRecords) { + assertThat(producerRecordsMap, hasKey(consumerRecord.key())); + if (consumerRecord.value() instanceof DynamicMessage) { + assertDynamicRecords(consumerRecord, producerRecordsMap); + } else { + assertThat(consumerRecord.value(), is(equalTo(producerRecordsMap.get(consumerRecord.key())))); + } + } + } + + private void assertRecordsEqualityV2(List> inputRecords, + List outputRecords) { + assertEquals(inputRecords.size(), outputRecords.size()); + + for (int i =0; i < inputRecords.size(); i++) { + if (outputRecords.get(i) instanceof DynamicMessage) { + assertDynamicRecords(outputRecords.get(i), inputRecords.get(i).value()); + } else { + assertEquals(inputRecords.get(i).value(), outputRecords.get(i)); + } + } + } + + private void assertDynamicRecords(Object consumerRecord, T producerRecord) { + DynamicMessage consumerDynamicMessage = (DynamicMessage) consumerRecord; + Message producerDynamicMessage = (Message) producerRecord; + //In case of DynamicMessage de-serialization, we cannot equate them to POJO records, + //so we check for their byte equality. + assertThat(consumerDynamicMessage.toByteArray(), is(producerDynamicMessage.toByteArray())); + } + + private void assertDynamicRecords(ConsumerRecord consumerRecord, Map producerRecordsMap) { + DynamicMessage consumerDynamicMessage = (DynamicMessage) consumerRecord.value(); + Message producerDynamicMessage = (Message) producerRecordsMap.get(consumerRecord.key()); + //In case of DynamicMessage de-serialization, we cannot equate them to POJO records, + //so we check for their byte equality. + assertThat(consumerDynamicMessage.toByteArray(), is(producerDynamicMessage.toByteArray())); + } +} diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ConsumerProperties.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ConsumerProperties.java new file mode 100644 index 00000000..84b005af --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ConsumerProperties.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.schemaregistry.integrationtests.properties.GlueSchemaRegistryConnectionProperties; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ConsumerProperties implements GlueSchemaRegistryConnectionProperties { + private String topicName; + private String avroRecordType; + private String protobufMessageType; +} + diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaClusterHelper.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaClusterHelper.java new file mode 100644 index 00000000..70409a40 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaClusterHelper.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +public interface KafkaClusterHelper { + String getOrCreateCluster(); + + String getSrcClusterBootstrapString(); + + String getDestClusterBootstrapString(); + + int getNumberOfPartitions(); + + short getReplicationFactor(); +} \ No newline at end of file diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaHelper.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaHelper.java new file mode 100644 index 00000000..84dab8e4 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/KafkaHelper.java @@ -0,0 +1,297 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryKafkaDeserializer; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistryKafkaSerializer; +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; + +import java.time.Duration; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; + +@Slf4j +public class KafkaHelper { + private static final Duration CONSUMER_RUNTIME = Duration.ofMillis(10000); + private final String bootstrapBrokers; + private final String clusterArn; + + public KafkaHelper(final String bootstrapString, final String clusterArn) { + this.bootstrapBrokers = bootstrapString; + this.clusterArn = clusterArn; + } + + /** + * Helper function to create test topic + * + * @param topic topic name to be created + * @param numPartitions number of numPartitions + * @param replicationFactor replicationFactor count + * @throws Exception + */ + public void createTopic(final String topic, final int numPartitions, final short replicationFactor) throws Exception { + final Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapBrokers); + properties.put("client.id", "gsr-integration-tests"); + + log.info("Creating Kafka topic {} with bootstrap {}...", topic, bootstrapBrokers); + try (AdminClient kafkaAdminClient = AdminClient.create(properties)) { + final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor); + final CreateTopicsResult createTopicsResult = kafkaAdminClient + .createTopics(Collections.singleton(newTopic)); + createTopicsResult.values().get(topic).get(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + /** + * Helper function to test producer can send messages + * + * @param topic topic to send messages to + * @param numRecords number of records to be sent + * @throws Exception + */ + public void doProduce(final String topic, final int numRecords) throws Exception { + log.info("Start producing to cluster {} with bootstrap {}...", clusterArn, bootstrapBrokers); + + final Properties properties = getKafkaProducerProperties(); + properties.put("key.serializer", StringSerializer.class.getName()); + properties.put("value.serializer", StringSerializer.class.getName()); + + try (Producer producer = new KafkaProducer<>(properties)) { + for (int i = 0; i < numRecords; i++) { + log.info("Producing record " + i); + producer.send(new ProducerRecord<>(topic, Integer.toString(i), Integer.toString(i))).get(); + } + } + + log.info("Finishing producing messages via Kafka."); + } + + /** + * Helper method to test consumption of records + * + * @param consumerProperties consumerProperties + * @return + */ + public int doConsume(final ConsumerProperties consumerProperties) { + final Properties properties = getKafkaConsumerProperties(consumerProperties, true); + properties.put("key.deserializer", StringDeserializer.class.getName()); + properties.put("value.deserializer", StringDeserializer.class.getName()); + final KafkaConsumer consumer = new KafkaConsumer<>(properties); + return consumeRecords(consumer, consumerProperties.getTopicName()).size(); + } + + /** + * Helper function to produce test AVRO records + * + * @param producerProperties producer properties + * @return list of produced records + */ + public List> doProduceRecords(final ProducerProperties producerProperties, + final List records) throws Exception { + Properties properties = getProducerProperties(producerProperties); + properties.put("key.serializer", StringSerializer.class.getName()); + properties.put("value.serializer", GlueSchemaRegistryKafkaSerializer.class.getName()); + Producer producer = new KafkaProducer<>(properties); + + return produceRecords(producer, producerProperties, records); + } + + /** + * Helper function to test consumption of records + * + * @param + */ + public List> doConsumeRecords(final ConsumerProperties consumerProperties) { + Properties properties = getConsumerProperties(consumerProperties, true); + properties.put("key.deserializer", StringDeserializer.class.getName()); + properties.put("value.deserializer", GlueSchemaRegistryKafkaDeserializer.class.getName()); + + final KafkaConsumer consumer = new KafkaConsumer<>(properties); + return consumeRecords(consumer, consumerProperties.getTopicName()); + } + + /** + * Helper function to test consumption of records using ByteArrayDeserializer + * + * @param + * @return + */ + public List> doConsumeRecordsWithByteArrayDeserializer(final ConsumerProperties consumerProperties) { + Properties properties = getConsumerProperties(consumerProperties, false); + properties.put("key.deserializer", org.apache.kafka.common.serialization.ByteArrayDeserializer.class.getName()); + properties.put("value.deserializer", org.apache.kafka.common.serialization.ByteArrayDeserializer.class.getName()); + + final KafkaConsumer consumer = new KafkaConsumer<>(properties); + return consumeRecordsAsByteArray(consumer, consumerProperties.getTopicName()); + } + + /** + * Helper function to process Kafka Streams + * + * @param producerProperties + */ + + private List> produceRecords(final Producer producer, + final ProducerProperties producerProperties, + final List records) throws Exception { + log.info("Start producing to cluster {} with bootstrap {}...", clusterArn, bootstrapBrokers); + List> producerRecords = new ArrayList<>(); + + for (int i = 0; i < records.size(); i++) { + log.info("Fetching record {} for Kafka: {}", i, (T) records.get(i)); + + final ProducerRecord producerRecord; + + // Verify and use a unique field present in the schema as a key for the producer record. + producerRecord = new ProducerRecord<>(producerProperties.getTopicName(), "message-" + i, (T) records.get(i)); + + producerRecords.add(producerRecord); + producer.send(producerRecord); + Thread.sleep(500); + log.info("Sent {} message {}", producerProperties.getDataFormat(), i); + } + producer.flush(); + log.info("Successfully produced {} messages to a topic called {}", records.size(), producerProperties.getTopicName()); + return producerRecords; + } + + private List> consumeRecords(final KafkaConsumer consumer, + final String topic) { + log.info("Start consuming from cluster {} with bootstrap {} ...", clusterArn, bootstrapBrokers); + + consumer.subscribe(Collections.singleton(topic)); + List> consumerRecords = new ArrayList<>(); + final long now = System.currentTimeMillis(); + while (System.currentTimeMillis() - now < CONSUMER_RUNTIME.toMillis()) { + final ConsumerRecords recordsReceived = consumer.poll(Duration.ofMillis(CONSUMER_RUNTIME.toMillis())); + int i = 0; + for (final ConsumerRecord record : recordsReceived) { + final String key = record.key(); + final T value = record.value(); + log.info("Received message {}: key = {}, value = {}", i, key, value); + consumerRecords.add(record); + i++; + } + } + + consumer.close(); + log.info("Finished consuming messages via Kafka."); + return consumerRecords; + } + + private List> consumeRecordsAsByteArray(final KafkaConsumer consumer, + final String topic) { + log.info("Start consuming from cluster {} with bootstrap {} ...", clusterArn, bootstrapBrokers); + + consumer.subscribe(Collections.singleton(topic)); + List> consumerRecords = new ArrayList<>(); + final long now = System.currentTimeMillis(); + while (System.currentTimeMillis() - now < CONSUMER_RUNTIME.toMillis()) { + final ConsumerRecords recordsReceived = consumer.poll(Duration.ofMillis(CONSUMER_RUNTIME.toMillis())); + for (final ConsumerRecord record : recordsReceived) { + consumerRecords.add(record); + } + } + + consumer.close(); + log.info("Finished consuming messages via Kafka."); + return consumerRecords; + } + + private Properties getProducerProperties(final ProducerProperties producerProperties) { + Properties properties = getKafkaProducerProperties(); + setSchemaRegistrySerializerProperties(properties, producerProperties); + return properties; + } + + private Properties getKafkaProducerProperties() { + Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapBrokers); + properties.put("acks", "all"); + properties.put("retries", 0); + properties.put("batch.size", 16384); + properties.put("linger.ms", 1); + properties.put("buffer.memory", 33554432); + properties.put("block.on.buffer.full", false); + properties.put("request.timeout.ms", "1000"); + return properties; + } + + private Properties getConsumerProperties(final ConsumerProperties consumerProperties, boolean shouldAddRegistryDetails) { + Properties properties = getKafkaConsumerProperties(consumerProperties, shouldAddRegistryDetails); + return properties; + } + + private Properties getKafkaConsumerProperties(final ConsumerProperties consumerProperties, boolean shouldAddRegistryDetails) { + Properties properties = new Properties(); + properties.put("bootstrap.servers", bootstrapBrokers); + properties.put("group.id", UUID.randomUUID().toString()); + properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + + if (shouldAddRegistryDetails) { + properties.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, consumerProperties.DEST_ENDPOINT); + properties.put(AWSSchemaRegistryConstants.AWS_REGION, consumerProperties.DEST_REGION); + } + + if(consumerProperties.getAvroRecordType() != null) { + properties.put(AWSSchemaRegistryConstants.AVRO_RECORD_TYPE, consumerProperties.getAvroRecordType()); + } + if(consumerProperties.getProtobufMessageType() != null) { + properties.put(AWSSchemaRegistryConstants.PROTOBUF_MESSAGE_TYPE, + consumerProperties.getProtobufMessageType()); + } + return properties; + } + + private void setSchemaRegistrySerializerProperties(final Properties properties, + final ProducerProperties producerProperties) { + properties.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, producerProperties.SRC_ENDPOINT); + properties.put(AWSSchemaRegistryConstants.AWS_REGION, producerProperties.SRC_REGION); + properties.put(AWSSchemaRegistryConstants.SCHEMA_NAME, producerProperties.getSchemaName()); + properties.put(AWSSchemaRegistryConstants.DATA_FORMAT, producerProperties.getDataFormat()); + properties.put(AWSSchemaRegistryConstants.COMPATIBILITY_SETTING, producerProperties.getCompatibilityType()); + properties.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, producerProperties.getAutoRegistrationEnabled()); + } + + /** + * Create Config map from the properties Object passed. + * + * @param properties properties of configuration elements. + * @return map of configs. + */ + private Map getMapFromPropertiesFile(Properties properties) { + return new HashMap<>(properties.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().toString(), e -> e.getValue()))); + } +} \ No newline at end of file diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/LocalKafkaClusterHelper.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/LocalKafkaClusterHelper.java new file mode 100644 index 00000000..48610bca --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/LocalKafkaClusterHelper.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +public class LocalKafkaClusterHelper implements KafkaClusterHelper { + private static final String FAKE_CLUSTER_ARN = "FAKE_CLUSTER_ARN"; + private static final String SRC_BOOTSTRAP_STRING = "127.0.0.1:9092"; + private static final String DEST_BOOTSTRAP_STRING = "127.0.0.1:9093"; + private static final int NUMBER_OF_PARTITIONS = 1; + private static final short REPLICATION_FACTOR = 1; + + @Override + public String getOrCreateCluster() { + return FAKE_CLUSTER_ARN; + } + + @Override + public String getSrcClusterBootstrapString() { + return SRC_BOOTSTRAP_STRING; + } + + @Override + public String getDestClusterBootstrapString() { + return DEST_BOOTSTRAP_STRING; + } + + @Override + public int getNumberOfPartitions() { + return NUMBER_OF_PARTITIONS; + } + + @Override + public short getReplicationFactor() { + return REPLICATION_FACTOR; + } +} + diff --git a/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ProducerProperties.java b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ProducerProperties.java new file mode 100644 index 00000000..0693cc42 --- /dev/null +++ b/integration-tests/src/test/java/com/amazonaws/services/schemaregistry/integrationtests/schemareplication/ProducerProperties.java @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. + * Licensed under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.amazonaws.services.schemaregistry.integrationtests.schemareplication; + +import com.amazonaws.services.schemaregistry.integrationtests.properties.GlueSchemaRegistryConnectionProperties; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ProducerProperties implements GlueSchemaRegistryConnectionProperties { + private String topicName; + private String schemaName; + private String dataFormat; + private String compatibilityType; + private String autoRegistrationEnabled; + // Streaming properties + private String inputTopic; + private String outputTopic; + private String recordType; // required only for AVRO or Protobuf case +} + diff --git a/pom.xml b/pom.xml index dbfe18bb..f7158f2a 100644 --- a/pom.xml +++ b/pom.xml @@ -74,6 +74,7 @@ integration-tests jsonschema-kafkaconnect-converter protobuf-kafkaconnect-converter + cross-region-replication-converter @@ -99,7 +100,7 @@ 1.7.30 2.17.1 5.6.3 - 3.3.3 + 5.14.2 1.18.26 1.1