diff --git a/src/it/java/io/github/jpmorganchase/fusion/packaging/DatasetOperationsIT.java b/src/it/java/io/github/jpmorganchase/fusion/packaging/DatasetOperationsIT.java index 7554e7a..a7f0ca7 100644 --- a/src/it/java/io/github/jpmorganchase/fusion/packaging/DatasetOperationsIT.java +++ b/src/it/java/io/github/jpmorganchase/fusion/packaging/DatasetOperationsIT.java @@ -3,6 +3,7 @@ import com.github.tomakehurst.wiremock.client.WireMock; import io.github.jpmorganchase.fusion.model.*; import io.github.jpmorganchase.fusion.test.TestUtils; +import org.hamcrest.Matchers; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -13,8 +14,7 @@ import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; import static io.github.jpmorganchase.fusion.test.TestUtils.listOf; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; public class DatasetOperationsIT extends BaseOperationsIT { @@ -150,7 +150,7 @@ public void testUpdateDatasetLineage() { // When & Then - Assertions.assertDoesNotThrow(() -> dataset.createLineage(DatasetLineage.builder() + Assertions.assertDoesNotThrow(() -> dataset.createLineage(SourceDatasets.builder() .source(new LinkedHashSet<>(Arrays.asList( DatasetReference.builder().catalog("foo").dataset("d1").build(), DatasetReference.builder().catalog("foo").dataset("d2").build(), @@ -160,6 +160,59 @@ public void testUpdateDatasetLineage() { .build())); } + @Test + public void testGetDatasetLineage(){ + // Given + wireMockRule.stubFor(WireMock.get(WireMock.urlEqualTo("/catalogs/common/datasets/SD0002/lineage")) + .willReturn(WireMock.aResponse() + .withHeader("Content-Type", "application/json") + .withStatus(200) + .withBodyFile("dataset/dataset-SD0002-get-lineage-response.json"))); + + //When + Dataset dataset = getSdk().builders().dataset() + .identifier("SD0002") + .catalogIdentifier("common") + .build(); + + DatasetLineage lineage = dataset.getLineage(); + + //Then + assertThat(lineage, notNullValue()); + assertThat(lineage.getDatasets(), notNullValue()); + assertThat(lineage.getDatasets().size(), Matchers.is(2)); + assertThat(lineage.getRelations(), notNullValue()); + assertThat(lineage.getRelations(), containsInAnyOrder( + relationship("common","SD0002", "common","SD0001"), + relationship("common","SD0003", "common","SD0002") + )); + + } + + @Test + public void testGetDatasetLineageWithNoRelationships(){ + // Given + wireMockRule.stubFor(WireMock.get(WireMock.urlEqualTo("/catalogs/common/datasets/SD0002/lineage")) + .willReturn(WireMock.aResponse() + .withHeader("Content-Type", "application/json") + .withStatus(200) + .withBodyFile("dataset/dataset-SD0002-get-lineage-empty-response.json"))); + + //When + Dataset dataset = getSdk().builders().dataset() + .identifier("SD0002") + .catalogIdentifier("common") + .build(); + + DatasetLineage lineage = dataset.getLineage(); + + //Then + assertThat(lineage, notNullValue()); + assertThat(lineage.getDatasets(), Matchers.is(Matchers.empty())); + assertThat(lineage.getRelations(), Matchers.is(Matchers.empty())); + + } + @Test public void testUpdateDatasetRetrievedFromListDatasets() { // Given @@ -264,4 +317,17 @@ public void testListDatasetsUsingIdContains() { } + private DatasetRelationship relationship(String srcCatalog, String srcDataset, String destCatalog, String destDataset) { + return DatasetRelationship.builder() + .source(DatasetReference.builder() + .catalog(srcCatalog) + .dataset(srcDataset) + .build()) + .destination(DatasetReference.builder() + .catalog(destCatalog) + .dataset(destDataset) + .build()) + .build(); + } + } diff --git a/src/main/java/io/github/jpmorganchase/fusion/Fusion.java b/src/main/java/io/github/jpmorganchase/fusion/Fusion.java index e984511..51da428 100644 --- a/src/main/java/io/github/jpmorganchase/fusion/Fusion.java +++ b/src/main/java/io/github/jpmorganchase/fusion/Fusion.java @@ -371,6 +371,22 @@ public Map> datasetResources(String catalogName, Str return this.callForMap(url); } + /** + * Get the lineage for a dataset, in the specified catalog + * Currently this will always return a lineage. + * + * @param catalogName identifier of the catalog to be queried + * @param dataset a String representing the dataset identifier to query. + * @throws APICallException if the call to the Fusion API fails + * @throws ParsingException if the response from Fusion could not be parsed successfully + * @throws OAuthException if a token could not be retrieved for authentication + */ + public DatasetLineage getLineage(String catalogName, String dataset) { + + String url = String.format("%1scatalogs/%2s/datasets/%3s/lineage", this.rootURL, catalogName, dataset); + return responseParser.parseDatasetLineage(this.api.callAPI(url), catalogName); + } + /** * Get a filtered list of the reports in the specified catalog *

diff --git a/src/main/java/io/github/jpmorganchase/fusion/model/Dataset.java b/src/main/java/io/github/jpmorganchase/fusion/model/Dataset.java index c904fd2..ba61783 100644 --- a/src/main/java/io/github/jpmorganchase/fusion/model/Dataset.java +++ b/src/main/java/io/github/jpmorganchase/fusion/model/Dataset.java @@ -67,10 +67,14 @@ protected String getApiPathForLineage() { * * @param lineage the {@code DatasetLineage} object representing the dataset lineage to be created */ - public void createLineage(DatasetLineage lineage) { + public void createLineage(SourceDatasets lineage) { getFusion().create(getApiPathForLineage(), lineage); } + public DatasetLineage getLineage() { + return getFusion().getLineage(this.getCatalogIdentifier(), this.getIdentifier()); + } + @Override public Set getRegisteredAttributes() { Set exclusions = super.getRegisteredAttributes(); diff --git a/src/main/java/io/github/jpmorganchase/fusion/model/DatasetLineage.java b/src/main/java/io/github/jpmorganchase/fusion/model/DatasetLineage.java index 3e55792..06b9afb 100644 --- a/src/main/java/io/github/jpmorganchase/fusion/model/DatasetLineage.java +++ b/src/main/java/io/github/jpmorganchase/fusion/model/DatasetLineage.java @@ -12,5 +12,6 @@ @Builder public class DatasetLineage { - Set source; + Set datasets; + Set relations; } diff --git a/src/main/java/io/github/jpmorganchase/fusion/model/DatasetRelationship.java b/src/main/java/io/github/jpmorganchase/fusion/model/DatasetRelationship.java new file mode 100644 index 0000000..74e663b --- /dev/null +++ b/src/main/java/io/github/jpmorganchase/fusion/model/DatasetRelationship.java @@ -0,0 +1,16 @@ +package io.github.jpmorganchase.fusion.model; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import lombok.Value; + +@Value +@EqualsAndHashCode +@ToString +@Builder +public class DatasetRelationship { + + DatasetReference source; + DatasetReference destination; +} diff --git a/src/main/java/io/github/jpmorganchase/fusion/model/SourceDatasets.java b/src/main/java/io/github/jpmorganchase/fusion/model/SourceDatasets.java new file mode 100644 index 0000000..8849160 --- /dev/null +++ b/src/main/java/io/github/jpmorganchase/fusion/model/SourceDatasets.java @@ -0,0 +1,16 @@ +package io.github.jpmorganchase.fusion.model; + +import java.util.Set; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import lombok.Value; + +@Value +@EqualsAndHashCode +@ToString +@Builder +public class SourceDatasets { + + Set source; +} diff --git a/src/main/java/io/github/jpmorganchase/fusion/parsing/APIResponseParser.java b/src/main/java/io/github/jpmorganchase/fusion/parsing/APIResponseParser.java index 578acb9..abd9349 100644 --- a/src/main/java/io/github/jpmorganchase/fusion/parsing/APIResponseParser.java +++ b/src/main/java/io/github/jpmorganchase/fusion/parsing/APIResponseParser.java @@ -10,6 +10,8 @@ public interface APIResponseParser { Map parseDatasetResponse(String json, String catalog); + DatasetLineage parseDatasetLineage(String json, String catalog); + Map parseReportResponse(String json, String catalog); Map parseDataFlowResponse(String json, String catalog); @@ -24,9 +26,6 @@ public interface APIResponseParser { Map parseDistributionResponse(String json); - T parseResourceFromResponse( - String json, Class resourceClass, ResourceMutationFactory mutator); - Map parseResourcesFromResponse(String json, Class resourceClass); Map parseResourcesWithVarArgsFromResponse( diff --git a/src/main/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParser.java b/src/main/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParser.java index 543c776..cc16b64 100644 --- a/src/main/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParser.java +++ b/src/main/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParser.java @@ -56,6 +56,23 @@ public Map parseDatasetResponse(String json, String catalog) { .build()); } + @Override + public DatasetLineage parseDatasetLineage(String json, String catalog) { + + Map datasets = parseResourcesWithVarArgsFromResponse( + json, Dataset.class, "datasets", (resource, mc) -> resource.toBuilder() + .varArgs(mc.getVarArgs()) + .fusion(fusion) + .build()); + + List relations = parseListOfResources(json, DatasetRelationship.class, "relations"); + + return DatasetLineage.builder() + .relations(new HashSet<>(relations)) + .datasets(new HashSet<>(datasets.values())) + .build(); + } + /** * Parses a JSON response to extract a map of reports. *

@@ -171,21 +188,16 @@ public UploadedPart parseUploadPartResponse(String json) { } @Override - public T parseResourceFromResponse( + public Map parseResourcesWithVarArgsFromResponse( String json, Class resourceClass, ResourceMutationFactory mutator) { - - Map responseMap = getMapFromJsonResponse(json); - T obj = gson.fromJson(json, resourceClass); - - return parseResourceWithVarArgs(obj.getRegisteredAttributes(), obj, responseMap, mutator); + return parseResourcesWithVarArgsFromResponse(json, resourceClass, "resources", mutator); } - @Override public Map parseResourcesWithVarArgsFromResponse( - String json, Class resourceClass, ResourceMutationFactory mutator) { + String json, Class resourceClass, String resourceAttribute, ResourceMutationFactory mutator) { - Map> untypedResources = parseResourcesUntyped(json); - JsonArray resources = getResources(json); + Map> untypedResources = parseResourcesUntyped(json, resourceAttribute); + JsonArray resources = getResources(json, resourceAttribute); List resourceList = new ArrayList<>(); for (JsonElement element : resources) { @@ -199,25 +211,34 @@ public Map parseResourcesWithVarArgsFromR @Override public Map parseResourcesFromResponse(String json, Class resourceClass) { + return parseResourcesFromResponse(json, resourceClass, "resources"); + } - JsonArray resources = getResources(json); + public List parseListOfResources(String json, Class resourceClass, String resourceAttribute) { + JsonArray resources = getResources(json, resourceAttribute); Type listType = TypeToken.getParameterized(List.class, resourceClass).getType(); - List resourceList = gson.fromJson(resources, listType); + return gson.fromJson(resources, listType); + } - return collectMapOfUniqueResources(resourceList); + public Map parseResourcesFromResponse( + String json, Class resourceClass, String resourceAttribute) { + return collectMapOfUniqueResources(parseListOfResources(json, resourceClass, resourceAttribute)); } @Override public Map> parseResourcesUntyped(String json) { + return parseResourcesUntyped(json, "resources"); + } + + public Map> parseResourcesUntyped(String json, String resourceAttribute) { Map responseMap = getMapFromJsonResponse(json); - Object resources = responseMap.get("resources"); + Object resources = responseMap.get(resourceAttribute); if (resources instanceof List) { @SuppressWarnings("unchecked") // List is always safe, compiler disagrees List resourceList = (List) resources; - if (resourceList.size() == 0) throw generateNoResourceException(); Map> resourcesMap = new HashMap<>(); resourceList.forEach((o -> { @SuppressWarnings("unchecked") // Output of GSON parsing will always be in this format @@ -232,11 +253,11 @@ public Map> parseResourcesUntyped(String json) { } } - private JsonArray getResources(String json) { + private JsonArray getResources(String json, String resourceAttribute) { JsonObject obj = JsonParser.parseString(json).getAsJsonObject(); - JsonArray array = obj.getAsJsonArray("resources"); - if (array == null || array.size() == 0) { - throw generateNoResourceException(); + JsonArray array = obj.getAsJsonArray(resourceAttribute); + if (array == null) { + array = new JsonArray(); } return array; } diff --git a/src/test/java/io/github/jpmorganchase/fusion/pact/FusionApiConsumerPactTest.java b/src/test/java/io/github/jpmorganchase/fusion/pact/FusionApiConsumerPactTest.java index 8ecd94f..aacfc3e 100644 --- a/src/test/java/io/github/jpmorganchase/fusion/pact/FusionApiConsumerPactTest.java +++ b/src/test/java/io/github/jpmorganchase/fusion/pact/FusionApiConsumerPactTest.java @@ -5,6 +5,7 @@ import static io.github.jpmorganchase.fusion.pact.util.RequestResponseHelper.getExpectation; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import au.com.dius.pact.consumer.MockServer; @@ -19,7 +20,6 @@ import io.github.jpmorganchase.fusion.model.*; import io.github.jpmorganchase.fusion.oauth.provider.FusionTokenProvider; import io.github.jpmorganchase.fusion.pact.util.FileHelper; -import io.github.jpmorganchase.fusion.parsing.ParsingException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Paths; @@ -218,11 +218,8 @@ void testListCatalogsWhenNoneAreAvailable(MockServer mockServer) { givenInstanceOfFusionSdk(mockServer); - ParsingException ex = Assertions.assertThrows(ParsingException.class, () -> fusion.listCatalogs()); - assertThat( - "Exception message is incorrect", - ex.getMessage(), - is(equalTo("Failed to parse resources from JSON, none found"))); + Map actual = fusion.listCatalogs(); + assertThat("Empty map expected", actual, is(anEmptyMap())); } @Test @@ -304,11 +301,8 @@ void testListProductsWhenNoneExist(MockServer mockServer) { givenInstanceOfFusionSdk(mockServer); - ParsingException ex = Assertions.assertThrows(ParsingException.class, () -> fusion.listProducts("common")); - assertThat( - "Exception message is incorrect", - ex.getMessage(), - is(equalTo("Failed to parse resources from JSON, none found"))); + Map actual = fusion.listProducts("common"); + assertThat("Exception message is incorrect", actual, is(anEmptyMap())); } @Test @@ -338,11 +332,8 @@ void testListDatasetsWhenNoneExist(MockServer mockServer) { givenInstanceOfFusionSdk(mockServer); - ParsingException ex = Assertions.assertThrows(ParsingException.class, () -> fusion.listDatasets("common")); - assertThat( - "Exception message is incorrect", - ex.getMessage(), - is(equalTo("Failed to parse resources from JSON, none found"))); + Map actual = fusion.listDatasets("common"); + assertThat("Empty map expected", actual, is(anEmptyMap())); } @Test @@ -395,12 +386,8 @@ void testListDatasetMembersWhenNoneExist(MockServer mockServer) { givenInstanceOfFusionSdk(mockServer); - ParsingException ex = - Assertions.assertThrows(ParsingException.class, () -> fusion.listDatasetMembers("common", "API_TEST")); - assertThat( - "Exception message is incorrect", - ex.getMessage(), - is(equalTo("Failed to parse resources from JSON, none found"))); + Map actual = fusion.listDatasetMembers("common", "API_TEST"); + assertThat("Empty map expected", actual, is(anEmptyMap())); } @Test diff --git a/src/test/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParserCatalogTest.java b/src/test/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParserCatalogTest.java index b814493..e2738d3 100644 --- a/src/test/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParserCatalogTest.java +++ b/src/test/java/io/github/jpmorganchase/fusion/parsing/GsonAPIResponseParserCatalogTest.java @@ -1,8 +1,7 @@ package io.github.jpmorganchase.fusion.parsing; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.assertThrows; import io.github.jpmorganchase.fusion.model.Catalog; @@ -73,19 +72,21 @@ public void multipleCatalogsInResourcesParseCorrectly() { } @Test - public void missingResourcesSectionInResponseCausesCorrectException() { - ParsingException thrown = - assertThrows(ParsingException.class, () -> responseParser.parseCatalogResponse(invalidCatalogJson)); + public void missingResourcesSectionInResponseReturnsEmptyMap() { - assertThat(thrown.getMessage(), is(equalTo("Failed to parse resources from JSON, none found"))); + Map catalog = responseParser.parseCatalogResponse(invalidCatalogJson); + + assertThat(catalog, is(notNullValue())); + assertThat(catalog, is(anEmptyMap())); } @Test - public void emptyResourcesSectionInResponseCausesCorrectException() { - ParsingException thrown = - assertThrows(ParsingException.class, () -> responseParser.parseCatalogResponse(noResourcesCatalogJson)); + public void emptyResourcesSectionInResponseReturnsEmptyMap() { - assertThat(thrown.getMessage(), is(equalTo("Failed to parse resources from JSON, none found"))); + Map catalog = responseParser.parseCatalogResponse(noResourcesCatalogJson); + + assertThat(catalog, is(notNullValue())); + assertThat(catalog, is(anEmptyMap())); } @Test @@ -97,11 +98,11 @@ public void missingResourcesSectionInResponseCausesCorrectExceptionForUntypedDat } @Test - public void emptyResourcesSectionInResponseCausesCorrectExceptionForUntypedData() { - ParsingException thrown = assertThrows( - ParsingException.class, () -> responseParser.parseResourcesUntyped(noResourcesCatalogJson)); + public void emptyResourcesSectionInResponseReturnsEmptyMapForUntypedData() { - assertThat(thrown.getMessage(), is(equalTo("Failed to parse resources from JSON, none found"))); + Map> actual = responseParser.parseResourcesUntyped(noResourcesCatalogJson); + + assertThat(actual, is(anEmptyMap())); } private static String loadTestResource(String resourceName) { diff --git a/src/test/resources/__files/dataset/dataset-SD0002-get-lineage-empty-response.json b/src/test/resources/__files/dataset/dataset-SD0002-get-lineage-empty-response.json new file mode 100644 index 0000000..57b4a70 --- /dev/null +++ b/src/test/resources/__files/dataset/dataset-SD0002-get-lineage-empty-response.json @@ -0,0 +1,4 @@ +{ + "datasets": [], + "relations": [] +} \ No newline at end of file diff --git a/src/test/resources/__files/dataset/dataset-SD0002-get-lineage-response.json b/src/test/resources/__files/dataset/dataset-SD0002-get-lineage-response.json new file mode 100644 index 0000000..af64919 --- /dev/null +++ b/src/test/resources/__files/dataset/dataset-SD0002-get-lineage-response.json @@ -0,0 +1,108 @@ +{ + "datasets": [ + { + "catalog": { + "@id": "common/", + "description": "A catalog of common data", + "identifier": "common", + "title": "Common", + "isInternal": false + }, + "category": [ + "Category 1" + ], + "createdDate": "2022-02-05", + "coverageStartDate": "2022-02-05", + "coverageEndDate": "2023-03-08", + "description": "Sample dataset description 1", + "frequency": "Daily", + "identifier": "SD0001", + "isThirdPartyData": false, + "isInternalOnlyDataset": false, + "language": "English", + "maintainer": "Maintainer 1", + "modifiedDate": "2023-03-08", + "publisher": "Publisher 1", + "region": [ + "North America" + ], + "source": [ + "Source System 1" + ], + "subCategory": [ + "Subcategory 1" + ], + "title": "Sample Dataset 1 | North America", + "tag": [ + "Tag1" + ], + "isRestricted": false, + "isRawData": false, + "hasSample": false, + "@id": "SD0001/" + }, + { + "catalog": { + "@id": "common/", + "description": "A catalog of common data", + "identifier": "common", + "title": "Common", + "isInternal": false + }, + "category": [ + "Category 3" + ], + "createdDate": "2022-02-05", + "coverageStartDate": "2022-02-05", + "coverageEndDate": "2023-03-08", + "description": "Sample dataset description 3", + "frequency": "Daily", + "identifier": "SD0003", + "isThirdPartyData": false, + "isInternalOnlyDataset": false, + "language": "English", + "maintainer": "Maintainer 3", + "modifiedDate": "2023-03-08", + "publisher": "Publisher 3", + "region": [ + "North America" + ], + "source": [ + "Source System 3" + ], + "subCategory": [ + "Subcategory 3" + ], + "title": "Sample Dataset 3 | North America", + "tag": [ + "Tag3" + ], + "isRestricted": false, + "isRawData": false, + "hasSample": false, + "@id": "SD0003/" + } + ], + "relations": [ + { + "source": { + "dataset": "SD0002", + "catalog": "common" + }, + "destination": { + "dataset": "SD0001", + "catalog": "common" + } + }, + { + "source": { + "dataset": "SD0003", + "catalog": "common" + }, + "destination": { + "dataset": "SD0002", + "catalog": "common" + } + } + ] +} \ No newline at end of file