From 668a428e24971adfd167105564f84af747a4244d Mon Sep 17 00:00:00 2001 From: ppel Date: Thu, 11 Jul 2024 14:29:33 +0200 Subject: [PATCH 1/3] #436475 - Added new extension for searching datasets. Modified FederatedCatalog extension and CountElementsExtension to use this new search functionality --- .../CountElementsApiExtension.java | 6 +- .../controller/CountElementsApi.java | 3 +- .../CountElementsApiController.java | 30 +++- .../service/CountElementsServiceImpl.java | 5 +- .../count-elements-sql/build.gradle.kts | 1 + .../sql/index/SqlCountElementsIndex.java | 26 ++- .../schema/BaseSqlDialectStatements.java | 47 ++--- .../index/schema/CountElementsStatements.java | 47 +++++ .../schema/postgres/SqlDatasetMapping.java | 25 +++ .../build.gradle.kts | 1 + .../schema/BaseSqlDialectStatements.java | 5 +- .../schema/SqlFederatedCatalogStatements.java | 3 +- .../inesdata-search-extension/README.md | 18 ++ .../build.gradle.kts | 10 ++ .../CriterionToWhereClauseConverterImpl.java | 167 ++++++++++++++++++ .../extension/InesdataSearchExtension.java | 29 +++ .../extension/InesdataSqlQueryStatement.java | 51 ++++++ ...rg.eclipse.edc.spi.system.ServiceExtension | 1 + settings.gradle.kts | 1 + .../index/CountElementsIndex.java | 4 +- .../service/CountElementsService.java | 4 +- 21 files changed, 442 insertions(+), 42 deletions(-) create mode 100644 extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java create mode 100644 extensions/inesdata-search-extension/README.md create mode 100644 extensions/inesdata-search-extension/build.gradle.kts create mode 100644 extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java create mode 100644 extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java create mode 100644 extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java create mode 100644 extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java index a36615c..3eeb2e3 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java @@ -10,6 +10,7 @@ import org.eclipse.edc.spi.types.TypeManager; import org.eclipse.edc.transaction.spi.TransactionContext; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; import org.eclipse.edc.web.spi.configuration.ApiContext; import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl; @@ -49,6 +50,9 @@ public class CountElementsApiExtension implements ServiceExtension { @Inject private TransactionContext transactionContext; + @Inject + private JsonObjectValidatorRegistry validator; + @Override public String name() { return NAME; @@ -72,7 +76,7 @@ public void initialize(ServiceExtensionContext context) { var managementApiTransformerRegistry = transformerRegistry.forContext("management-api"); managementApiTransformerRegistry.register(new JsonObjectFromCountElementTransformer(factory, jsonLdMapper)); - var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry); + var countElementsApiController = new CountElementsApiController(countElementsService(), managementApiTransformerRegistry, validator); webService.registerResource(ApiContext.MANAGEMENT, countElementsApiController); } } diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java index bb40db4..16534de 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApi.java @@ -8,6 +8,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.json.JsonObject; import org.eclipse.edc.api.model.ApiCoreSchema; @OpenAPIDefinition( @@ -32,5 +33,5 @@ public interface CountElementsApi { content = @Content(array = @ArraySchema(schema = @Schema(implementation = ApiCoreSchema.ApiErrorDetailSchema.class)))) } ) - long countElements(String entityType); + long countElements(String entityType, JsonObject querySpecJson); } diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java index 5c3e4b6..fc59545 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java @@ -1,32 +1,44 @@ package org.upm.inesdata.countelements.controller; +import jakarta.json.JsonObject; import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.MediaType; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; +import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; +import org.eclipse.edc.web.spi.exception.ValidationFailureException; import org.upm.inesdata.spi.countelements.service.CountElementsService; import java.util.Objects; +import static org.eclipse.edc.spi.query.QuerySpec.EDC_QUERY_SPEC_TYPE; + @Produces({MediaType.APPLICATION_JSON}) +@Consumes({ MediaType.APPLICATION_JSON }) @Path("/pagination") public class CountElementsApiController implements CountElementsApi { private final CountElementsService service; + private final JsonObjectValidatorRegistry validator; private final TypeTransformerRegistry transformerRegistry; - public CountElementsApiController(CountElementsService service, TypeTransformerRegistry transformerRegistry) { + public CountElementsApiController(CountElementsService service, TypeTransformerRegistry transformerRegistry, JsonObjectValidatorRegistry validator) { this.service = service; this.transformerRegistry = transformerRegistry; + this.validator = validator; } - @GET + @POST @Path("/count") @Override - public long countElements(@QueryParam("type") String entityType) { + public long countElements(@QueryParam("type") String entityType, JsonObject querySpecJson) { if (!Objects.equals(entityType, "asset") && !Objects.equals(entityType, "policyDefinition") && !Objects.equals(entityType, "contractDefinition") && !Objects.equals(entityType, "contractAgreement") @@ -35,7 +47,17 @@ public long countElements(@QueryParam("type") String entityType) { throw new BadRequestException("Entity type provided is not valid"); } - var count = service.countElements(entityType); + QuerySpec querySpec; + if (querySpecJson == null) { + querySpec = QuerySpec.Builder.newInstance().build(); + } else { + validator.validate(EDC_QUERY_SPEC_TYPE, querySpecJson).orElseThrow(ValidationFailureException::new); + + querySpec = transformerRegistry.transform(querySpecJson, QuerySpec.class) + .orElseThrow(InvalidRequestException::new); + } + + var count = service.countElements(entityType, querySpec); // JsonObject result = transformerRegistry.transform(count, JsonObject.class) // .orElseThrow(f -> new EdcException(f.getFailureDetail())); diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java index 91ec4cc..6332128 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/service/CountElementsServiceImpl.java @@ -1,5 +1,6 @@ package org.upm.inesdata.countelements.service; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.transaction.spi.TransactionContext; import org.upm.inesdata.spi.countelements.domain.CountElement; import org.upm.inesdata.spi.countelements.index.CountElementsIndex; @@ -15,7 +16,7 @@ public CountElementsServiceImpl(CountElementsIndex countElementsIndex, Transacti } @Override - public CountElement countElements(String entityType) { - return transactionContext.execute(() -> countElementsIndex.countElements(entityType)); + public CountElement countElements(String entityType, QuerySpec querySpec) { + return transactionContext.execute(() -> countElementsIndex.countElements(entityType, querySpec)); } } diff --git a/extensions/count-elements-sql/build.gradle.kts b/extensions/count-elements-sql/build.gradle.kts index 91b8c6a..08dd665 100644 --- a/extensions/count-elements-sql/build.gradle.kts +++ b/extensions/count-elements-sql/build.gradle.kts @@ -6,6 +6,7 @@ plugins { dependencies { api(project(":spi:count-elements-spi")) implementation(project(":extensions:count-elements-api")) + api(project(":extensions:inesdata-search-extension")) api(libs.edc.spi.core) api(libs.edc.transaction.spi) implementation(libs.edc.transaction.spi) diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java index 9f314c6..0930f73 100644 --- a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/SqlCountElementsIndex.java @@ -2,8 +2,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.eclipse.edc.spi.persistence.EdcPersistenceException; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.QueryExecutor; import org.eclipse.edc.sql.store.AbstractSqlStore; +import org.eclipse.edc.sql.translation.SqlQueryStatement; import org.eclipse.edc.transaction.datasource.spi.DataSourceRegistry; import org.eclipse.edc.transaction.spi.TransactionContext; import org.upm.inesdata.countelements.sql.index.schema.CountElementsStatements; @@ -21,20 +23,28 @@ public class SqlCountElementsIndex extends AbstractSqlStore implements CountElem private final CountElementsStatements countElementsStatements; public SqlCountElementsIndex(DataSourceRegistry dataSourceRegistry, - String dataSourceName, - TransactionContext transactionContext, - ObjectMapper objectMapper, - CountElementsStatements countElementsStatements, - QueryExecutor queryExecutor) { + String dataSourceName, + TransactionContext transactionContext, + ObjectMapper objectMapper, + CountElementsStatements countElementsStatements, + QueryExecutor queryExecutor) { super(dataSourceRegistry, dataSourceName, transactionContext, objectMapper, queryExecutor); this.countElementsStatements = Objects.requireNonNull(countElementsStatements); } @Override - public CountElement countElements(String entityType) { + public CountElement countElements(String entityType, QuerySpec querySpec) { try (var connection = getConnection()) { - var sql = countElementsStatements.getCount(entityType); - long count = queryExecutor.single(connection, true, r -> r.getLong(1), sql); + long count; + if ("federatedCatalog".equals(entityType)) { + SqlQueryStatement dataSetQueryStatement = countElementsStatements.createCountDatasetQuery(entityType, querySpec); + count = queryExecutor.single(connection, true, r -> r.getLong(1), + dataSetQueryStatement.getQueryAsString(), dataSetQueryStatement.getParameters()); + } else { + var sql = countElementsStatements.getCount(entityType); + count = queryExecutor.single(connection, true, r -> r.getLong(1), sql); + } + return CountElement.Builder.newInstance().count(count).build(); } catch (SQLException e) { throw new EdcPersistenceException(e); diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java index cedb579..18367f0 100644 --- a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/BaseSqlDialectStatements.java @@ -1,6 +1,9 @@ package org.upm.inesdata.countelements.sql.index.schema; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.upm.inesdata.countelements.sql.index.schema.postgres.SqlDatasetMapping; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; import static java.lang.String.format; @@ -15,31 +18,33 @@ public BaseSqlDialectStatements(SqlOperatorTranslator operatorTranslator) { this.operatorTranslator = operatorTranslator; } + /** + * {@inheritDoc} + * + * @see CountElementsStatements#getCount(String) + */ @Override public String getCount(String entityType) { - String tableName = null; - switch (entityType) { - case "asset": - tableName = getAssetTable(); - break; - case "policyDefinition": - tableName = getPolicyDefinitionTable(); - break; - case "contractDefinition": - tableName = getContractDefinitionTable(); - break; - case "contractAgreement": - tableName = getContractAgreementTable(); - break; - case "transferProcess": - tableName = getTransferProcessTable(); - break; - case "federatedCatalog": - tableName = getDatasetTable(); - break; - } + String tableName = switch (entityType) { + case "asset" -> getAssetTable(); + case "policyDefinition" -> getPolicyDefinitionTable(); + case "contractDefinition" -> getContractDefinitionTable(); + case "contractAgreement" -> getContractAgreementTable(); + case "transferProcess" -> getTransferProcessTable(); + case "federatedCatalog" -> getDatasetTable(); + default -> null; + }; return format("SELECT COUNT(*) FROM %s", tableName); } + /** + * {@inheritDoc} + * + * @see CountElementsStatements#createDatasetQuery(QuerySpec) + */ + @Override + public InesdataSqlQueryStatement createCountDatasetQuery(String entityType, QuerySpec querySpec) { + return new InesdataSqlQueryStatement(getCount(entityType), querySpec, new SqlDatasetMapping(this), operatorTranslator); + } } diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java index 3bec855..93d29e4 100644 --- a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/CountElementsStatements.java @@ -1,7 +1,9 @@ package org.upm.inesdata.countelements.sql.index.schema; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.statement.SqlStatements; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; /** * Defines queries used by the SqlCountElementsIndexServiceExtension. @@ -51,6 +53,51 @@ default String getDatasetTable() { return "edc_dataset"; } + /** + * Retrieves the name of the column storing dataset IDs. + * + * @return the name of the dataset ID column. + */ + default String getDatasetIdColumn() { + return "id"; + } + + /** + * Retrieves the name of the column storing offers associated with datasets. + * + * @return the name of the offers column. + */ + default String getDatasetOffersColumn() { + return "offers"; + } + + /** + * Retrieves the name of the column storing properties of datasets as JSON. + * + * @return the name of the properties column. + */ + default String getDatasetPropertiesColumn() { + return "properties"; + } + + /** + * Retrieves the name of the column storing the catalog ID associated with datasets. + * + * @return the name of the catalog ID column. + */ + default String getDatasetCatalogIdColumn() { + return "catalog_id"; + } + + /** + * Creates an SQL query statement specifically for datasets based on the provided query specification. + * + * @param querySpec the query specification defining filters, sorting, and pagination for datasets. + * @param entityType the entity type (federatedCatalog) + * @return an SQL query statement for datasets. + */ + InesdataSqlQueryStatement createCountDatasetQuery(String entityType, QuerySpec querySpec); + /** * SELECT COUNT clause. */ diff --git a/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java new file mode 100644 index 0000000..ce71b5c --- /dev/null +++ b/extensions/count-elements-sql/src/main/java/org/upm/inesdata/countelements/sql/index/schema/postgres/SqlDatasetMapping.java @@ -0,0 +1,25 @@ +package org.upm.inesdata.countelements.sql.index.schema.postgres; + +import org.eclipse.edc.sql.translation.JsonFieldTranslator; +import org.eclipse.edc.sql.translation.TranslationMapping; +import org.upm.inesdata.countelements.sql.index.schema.CountElementsStatements; + +/** + * Maps fields of a dataset of federated catalog onto the + * corresponding SQL schema (= column names) enabling access through Postgres JSON operators where applicable + */ +public class SqlDatasetMapping extends TranslationMapping { + /** + * Constructs a mapping for SQL dataset columns using the provided SQL federated catalog statements. + * This mapping specifies how dataset fields correspond to database columns. + * + * @param statements the SQL statements specific to the federated catalog schema. + */ + public SqlDatasetMapping(CountElementsStatements statements) { + add("id", statements.getDatasetIdColumn()); + add("offers", new JsonFieldTranslator(statements.getDatasetOffersColumn())); + add("properties", new JsonFieldTranslator(statements.getDatasetPropertiesColumn())); + add("catalog_id", statements.getDatasetCatalogIdColumn()); + } + +} diff --git a/extensions/federated-catalog-cache-sql/build.gradle.kts b/extensions/federated-catalog-cache-sql/build.gradle.kts index bd0dcb4..48f92fd 100644 --- a/extensions/federated-catalog-cache-sql/build.gradle.kts +++ b/extensions/federated-catalog-cache-sql/build.gradle.kts @@ -8,6 +8,7 @@ dependencies { implementation(libs.edc.federated.catalog.api) api(libs.edc.spi.core) api(libs.edc.transaction.spi) + api(project(":extensions:inesdata-search-extension")) implementation(libs.edc.transaction.spi) implementation(libs.edc.transaction.datasource.spi) implementation(libs.edc.sql.core) diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java index ef51d38..317fee0 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/BaseSqlDialectStatements.java @@ -5,6 +5,7 @@ import org.eclipse.edc.sql.translation.SqlQueryStatement; import org.upm.inesdata.federated.sql.index.schema.postgres.SqlDatasetMapping; import org.upm.inesdata.federated.sql.index.schema.postgres.SqlFederatedCatalogMapping; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; import static java.lang.String.format; @@ -133,8 +134,8 @@ public SqlQueryStatement createQuery(QuerySpec querySpec) { * @see SqlFederatedCatalogStatements#createDatasetQuery(QuerySpec) */ @Override - public SqlQueryStatement createDatasetQuery(QuerySpec querySpec) { - return new SqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator); + public InesdataSqlQueryStatement createDatasetQuery(QuerySpec querySpec) { + return new InesdataSqlQueryStatement(getSelectDatasetTemplate(), querySpec, new SqlDatasetMapping(this), operatorTranslator); } /** diff --git a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java index def47e4..eb981c0 100644 --- a/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java +++ b/extensions/federated-catalog-cache-sql/src/main/java/org/upm/inesdata/federated/sql/index/schema/SqlFederatedCatalogStatements.java @@ -4,6 +4,7 @@ import org.eclipse.edc.spi.query.QuerySpec; import org.eclipse.edc.sql.statement.SqlStatements; import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.upm.inesdata.search.extension.InesdataSqlQueryStatement; /** * SQL statements interface for managing federated catalog data. Extends {@link SqlStatements} and provides methods for @@ -250,7 +251,7 @@ default String getDistributionDatasetIdColumn() { * @param querySpec the query specification defining filters, sorting, and pagination for datasets. * @return an SQL query statement for datasets. */ - SqlQueryStatement createDatasetQuery(QuerySpec querySpec); + InesdataSqlQueryStatement createDatasetQuery(QuerySpec querySpec); /** * Retrieves the SQL template for deleting expired catalogs. diff --git a/extensions/inesdata-search-extension/README.md b/extensions/inesdata-search-extension/README.md new file mode 100644 index 0000000..51bb140 --- /dev/null +++ b/extensions/inesdata-search-extension/README.md @@ -0,0 +1,18 @@ +# Oauth2 JWT Token Authentication Service + +This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. + +To authorize a user, the roles of the provided JWT token must contain: +- a valid role from those configured in `allowedRoles` +- a role with the `connector name` + +## Configuration + +Example configuration: + +```properties +edc.api.auth.oauth2.allowedRoles.1.role=connector-admin +edc.api.auth.oauth2.allowedRoles.2.role=connector-management +``` + +The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file diff --git a/extensions/inesdata-search-extension/build.gradle.kts b/extensions/inesdata-search-extension/build.gradle.kts new file mode 100644 index 0000000..cfbb2cf --- /dev/null +++ b/extensions/inesdata-search-extension/build.gradle.kts @@ -0,0 +1,10 @@ +plugins { + `java-library` + id("com.gmv.inesdata.edc-application") +} + +dependencies { + implementation(libs.edc.sql.core) +} + + diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java new file mode 100644 index 0000000..8c70404 --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.upm.inesdata.search.extension; + +import org.eclipse.edc.spi.EdcException; +import org.eclipse.edc.spi.query.Criterion; +import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; +import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.eclipse.edc.sql.translation.TranslationMapping; +import org.eclipse.edc.sql.translation.WhereClause; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static java.util.Collections.unmodifiableCollection; + +public class CriterionToWhereClauseConverterImpl implements CriterionToWhereClauseConverter { + + private static final String GENERIC_SEARCH = "genericSearch"; + private static final String ASSET_DATA_PROPERTY = "'https://w3id.org/edc/v0.0.1/ns/assetData'"; + private static final String [] COMMON_PROPERTIES = { + "https://w3id.org/edc/v0.0.1/ns/id", + "https://w3id.org/edc/v0.0.1/ns/name", + "https://w3id.org/edc/v0.0.1/ns/version", + "https://w3id.org/edc/v0.0.1/ns/contenttype", + "https://w3id.org/edc/v0.0.1/ns/contenttype", + "http://purl.org/dc/terms/format", + "http://www.w3.org/ns/dcat#keyword", + "http://www.w3.org/ns/dcat#byteSize", + "https://w3id.org/edc/v0.0.1/ns/shortDescription", + "https://w3id.org/edc/v0.0.1/ns/assetType", + "http://purl.org/dc/terms/description" + }; + private final TranslationMapping translationMapping; + private final SqlOperatorTranslator operatorTranslator; + + public CriterionToWhereClauseConverterImpl(TranslationMapping translationMapping, SqlOperatorTranslator operatorTranslator) { + this.translationMapping = translationMapping; + this.operatorTranslator = operatorTranslator; + } + + @Override + public WhereClause convert(Criterion criterion) { + var operator = operatorTranslator.translate(criterion.getOperator().toLowerCase()); + if (operator == null) { + throw new IllegalArgumentException("The operator '%s' is not supported".formatted(criterion.getOperator())); + } + + if (!operator.rightOperandClass().isAssignableFrom(criterion.getOperandRight().getClass())) { + throw new IllegalArgumentException("The operator '%s' requires the right-hand operand to be of type %s" + .formatted(criterion.getOperator(), operator.rightOperandClass().getSimpleName())); + } + + if (criterion.getOperandLeft().toString().startsWith(ASSET_DATA_PROPERTY)) { + return generateVocabularyWhereClause(criterion); + } else if (GENERIC_SEARCH.equals(criterion.getOperandLeft().toString())) { + return generateGenericPropertiesWhereClause(criterion); + } + + var whereClause = translationMapping.getWhereClause(criterion, operator); + if (whereClause == null) { + return new WhereClause("0 = ?", 1); + } + + return whereClause; + } + + private WhereClause generateGenericPropertiesWhereClause(Criterion criterion) { + String operator = criterion.getOperator(); + String rightValue = criterion.getOperandRight().toString(); + List values = new ArrayList<>(Collections.nCopies(COMMON_PROPERTIES.length, rightValue)); + + StringBuilder sqlWhereBuilder = new StringBuilder("("); + for (int i = 0; i < COMMON_PROPERTIES.length; i++) { + sqlWhereBuilder.append("properties ->> '") + .append(COMMON_PROPERTIES[i]) + .append("' ") + .append(operator) + .append(" ?"); + if (i < COMMON_PROPERTIES.length - 1) { + sqlWhereBuilder.append(" OR "); + } + } + sqlWhereBuilder.append(")"); + + return new WhereClause(sqlWhereBuilder.toString(), unmodifiableCollection(values)); + } + + private WhereClause generateVocabularyWhereClause(Criterion criterion) { + String[] propertiesList = splitByDotOutsideQuotes(criterion.getOperandLeft().toString()); + StringBuilder sqlWhereBuilder = new StringBuilder(); + + switch (propertiesList.length) { + case 3 -> + generateNonObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); + case 4 -> + generateObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); + default -> throw new IllegalArgumentException("Invalid vocabulary argument in the operandLeft: %s" + .formatted(criterion.getOperandLeft().toString())); + } + + return new WhereClause(sqlWhereBuilder.toString(), unmodifiableCollection(new ArrayList<>())); + } + + private void generateNonObjectPropertySQL(StringBuilder sqlWhereBuilder, String[] propertiesList, String operandRight) { + sqlWhereBuilder.append("(properties::jsonb -> ") + .append(propertiesList[0]) + .append(" -> ") + .append(propertiesList[1]) + .append(")::jsonb @> '[{") + .append(propertiesList[2].replaceAll("'", "\"")) + .append(": [{\"@value\": \"") + .append(operandRight) + .append("\"}]}]'::jsonb"); + } + + private void generateObjectPropertySQL(StringBuilder sqlWhereBuilder, String[] propertiesList, String operandRight) { + sqlWhereBuilder.append("EXISTS (SELECT 1 FROM jsonb_array_elements((properties::jsonb -> ") + .append(propertiesList[0]) + .append(" -> ") + .append(propertiesList[1]) + .append(")::jsonb) AS vocab WHERE vocab -> ") + .append(propertiesList[2]) + .append(" @> '[{") + .append(propertiesList[3].replaceAll("'", "\"")) + .append(": [{\"@value\": \"") + .append(operandRight) + .append("\"}]}]')"); + } + + private String[] splitByDotOutsideQuotes(String input) { + List parts = new ArrayList<>(); + + Pattern pattern = Pattern.compile("\\.(?=(?:[^']*'[^']*')*[^']*$)"); + + Matcher matcher = pattern.matcher(input); + int start = 0; + + while (matcher.find()) { + String part = input.substring(start, matcher.start()).trim(); + parts.add(part); + start = matcher.end(); + } + + if (start < input.length()) { + String lastPart = input.substring(start).trim(); + parts.add(lastPart); + } + + return parts.toArray(new String[0]); + } +} diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java new file mode 100644 index 0000000..21b24a3 --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSearchExtension.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.upm.inesdata.search.extension; + +import org.eclipse.edc.runtime.metamodel.annotation.Extension; +import org.eclipse.edc.spi.system.ServiceExtension; + +@Extension(value = InesdataSearchExtension.NAME) +public class InesdataSearchExtension implements ServiceExtension { + + public static final String NAME = "Inesdata Search Extension"; + + @Override + public String name() { + return NAME; + } +} diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java new file mode 100644 index 0000000..a464c2a --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial API and implementation + * + */ + +package org.upm.inesdata.search.extension; + +import org.eclipse.edc.spi.query.QuerySpec; +import org.eclipse.edc.spi.query.SortOrder; +import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; +import org.eclipse.edc.sql.translation.SortFieldConverterImpl; +import org.eclipse.edc.sql.translation.SqlOperatorTranslator; +import org.eclipse.edc.sql.translation.SqlQueryStatement; +import org.eclipse.edc.sql.translation.TranslationMapping; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static java.lang.String.format; +import static java.util.stream.Collectors.joining; + +/** + * Maps a {@link QuerySpec} to a single SQL {@code SELECT ... FROM ... WHERE ...} statement. The {@code SELECT ...} part + * is passed in through the constructor, and the rest of the query is assembled dynamically, based on the + * {@link QuerySpec} and the {@link TranslationMapping}. + */ +public class InesdataSqlQueryStatement extends SqlQueryStatement{ + + /** + * Initializes this SQL Query Statement. + * + * @param selectStatement The SELECT clause, e.g. {@code SELECT * FROM your_table} + * @param query a {@link QuerySpec} that contains a query in the canonical format + * @param rootModel A {@link TranslationMapping} that enables mapping from canonical to the SQL-specific + * model/format + * @param operatorTranslator the {@link SqlOperatorTranslator} instance. + */ + public InesdataSqlQueryStatement(String selectStatement, QuerySpec query, TranslationMapping rootModel, SqlOperatorTranslator operatorTranslator) { + super(selectStatement, query, rootModel, new CriterionToWhereClauseConverterImpl(rootModel, operatorTranslator)); + } +} diff --git a/extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 0000000..67fef91 --- /dev/null +++ b/extensions/inesdata-search-extension/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.upm.inesdata.search.extension.InesdataSearchExtension diff --git a/settings.gradle.kts b/settings.gradle.kts index 49a1eef..c140da8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,6 +25,7 @@ include(":extensions:count-elements-sql") include(":extensions:extended-data-plane-public-api") include(":extensions:audit-configuration") include(":extensions:audit-event-configuration") +include(":extensions:inesdata-search-extension") // Connector include(":launchers:connector") diff --git a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java index 031a8ea..d8a14a2 100644 --- a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java +++ b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/index/CountElementsIndex.java @@ -1,6 +1,7 @@ package org.upm.inesdata.spi.countelements.index; import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint; +import org.eclipse.edc.spi.query.QuerySpec; import org.upm.inesdata.spi.countelements.domain.CountElement; /** @@ -13,7 +14,8 @@ public interface CountElementsIndex { * Counts all contract agreements * * @param entityType entity type + * @param querySpec filters * @return the number of contract agreements */ - CountElement countElements(String entityType); + CountElement countElements(String entityType, QuerySpec querySpec); } diff --git a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java index 10d3802..be0bf5a 100644 --- a/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java +++ b/spi/count-elements-spi/src/main/java/org/upm/inesdata/spi/countelements/service/CountElementsService.java @@ -1,5 +1,6 @@ package org.upm.inesdata.spi.countelements.service; +import org.eclipse.edc.spi.query.QuerySpec; import org.upm.inesdata.spi.countelements.domain.CountElement; /** @@ -11,7 +12,8 @@ public interface CountElementsService { * Gets the total number of elements of an entity. * * @param entityType entity type + * @param querySpec filters * @return the total number of elements */ - CountElement countElements(String entityType); + CountElement countElements(String entityType, QuerySpec querySpec); } From 81534904eb8581b7cf31207b6d1fa7f5c0c6e485 Mon Sep 17 00:00:00 2001 From: ppel Date: Fri, 12 Jul 2024 10:46:35 +0200 Subject: [PATCH 2/3] #436475 - Throw InvalidRequestExcpetion when invalid length of vocabulary properties --- .../countelements/CountElementsApiExtension.java | 1 - .../controller/CountElementsApiController.java | 3 +-- extensions/inesdata-search-extension/build.gradle.kts | 3 ++- .../extension/CriterionToWhereClauseConverterImpl.java | 4 ++-- .../search/extension/InesdataSqlQueryStatement.java | 10 ---------- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java index 3eeb2e3..edbf05d 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/CountElementsApiExtension.java @@ -13,7 +13,6 @@ import org.eclipse.edc.validator.spi.JsonObjectValidatorRegistry; import org.eclipse.edc.web.spi.WebService; import org.eclipse.edc.web.spi.configuration.ApiContext; -import org.eclipse.edc.web.spi.configuration.context.ManagementApiUrl; import org.upm.inesdata.countelements.controller.CountElementsApiController; import org.upm.inesdata.countelements.service.CountElementsServiceImpl; import org.upm.inesdata.countelements.transformer.JsonObjectFromCountElementTransformer; diff --git a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java index fc59545..fafde2a 100644 --- a/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java +++ b/extensions/count-elements-api/src/main/java/org/upm/inesdata/countelements/controller/CountElementsApiController.java @@ -3,7 +3,6 @@ import jakarta.json.JsonObject; import jakarta.ws.rs.BadRequestException; import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; @@ -44,7 +43,7 @@ public long countElements(@QueryParam("type") String entityType, JsonObject quer && !Objects.equals(entityType, "contractAgreement") && !Objects.equals(entityType, "transferProcess") && !Objects.equals(entityType, "federatedCatalog")) { - throw new BadRequestException("Entity type provided is not valid"); + throw new InvalidRequestException("Entity type provided is not valid: %s".formatted(entityType)); } QuerySpec querySpec; diff --git a/extensions/inesdata-search-extension/build.gradle.kts b/extensions/inesdata-search-extension/build.gradle.kts index cfbb2cf..5baaa3b 100644 --- a/extensions/inesdata-search-extension/build.gradle.kts +++ b/extensions/inesdata-search-extension/build.gradle.kts @@ -4,7 +4,8 @@ plugins { } dependencies { - implementation(libs.edc.sql.core) + api(libs.edc.sql.core) + implementation(libs.edc.web.spi) } diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java index 8c70404..ef1b007 100644 --- a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/CriterionToWhereClauseConverterImpl.java @@ -14,12 +14,12 @@ package org.upm.inesdata.search.extension; -import org.eclipse.edc.spi.EdcException; import org.eclipse.edc.spi.query.Criterion; import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; import org.eclipse.edc.sql.translation.TranslationMapping; import org.eclipse.edc.sql.translation.WhereClause; +import org.eclipse.edc.web.spi.exception.InvalidRequestException; import java.util.ArrayList; import java.util.Collections; @@ -110,7 +110,7 @@ private WhereClause generateVocabularyWhereClause(Criterion criterion) { generateNonObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); case 4 -> generateObjectPropertySQL(sqlWhereBuilder, propertiesList, criterion.getOperandRight().toString()); - default -> throw new IllegalArgumentException("Invalid vocabulary argument in the operandLeft: %s" + default -> throw new InvalidRequestException("Invalid vocabulary argument in the operandLeft: %s" .formatted(criterion.getOperandLeft().toString())); } diff --git a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java index a464c2a..8764f59 100644 --- a/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java +++ b/extensions/inesdata-search-extension/src/main/java/org/upm/inesdata/search/extension/InesdataSqlQueryStatement.java @@ -15,20 +15,10 @@ package org.upm.inesdata.search.extension; import org.eclipse.edc.spi.query.QuerySpec; -import org.eclipse.edc.spi.query.SortOrder; -import org.eclipse.edc.sql.translation.CriterionToWhereClauseConverter; -import org.eclipse.edc.sql.translation.SortFieldConverterImpl; import org.eclipse.edc.sql.translation.SqlOperatorTranslator; import org.eclipse.edc.sql.translation.SqlQueryStatement; import org.eclipse.edc.sql.translation.TranslationMapping; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import static java.lang.String.format; -import static java.util.stream.Collectors.joining; - /** * Maps a {@link QuerySpec} to a single SQL {@code SELECT ... FROM ... WHERE ...} statement. The {@code SELECT ...} part * is passed in through the constructor, and the rest of the query is assembled dynamically, based on the From 6c574dee32263bfd5408321cde1e417125f585f9 Mon Sep 17 00:00:00 2001 From: ppel Date: Mon, 15 Jul 2024 09:37:15 +0200 Subject: [PATCH 3/3] #436475 - Updated readme --- .../inesdata-search-extension/README.md | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/extensions/inesdata-search-extension/README.md b/extensions/inesdata-search-extension/README.md index 51bb140..3288d8d 100644 --- a/extensions/inesdata-search-extension/README.md +++ b/extensions/inesdata-search-extension/README.md @@ -1,18 +1,37 @@ -# Oauth2 JWT Token Authentication Service +# INESData search extension -This extension provides the capability to authorizate the request to the connector management API. The extension will access the Bearer token provided in the Authorization header and validate that it is a valid JWT-encoded bearer token. It is necessary to have the `org.eclipse.edc:oauth2-core` extension correctly configured. +This extension provides the capability to search inside the properties of an asset. +The functionality of this new search works as follows: +- To perform a search among the generic properties of the asset it is necessary to indicate 'genericSearch' as the value of the operandLeft +- To perform a search among the properties of a vocabulary, it is necessary to indicate 'https://w3id.org/edc/v0.0.1/ns/assetData' followed by the name of the vocabulary and the property to search for. An example is given in the following section. -To authorize a user, the roles of the provided JWT token must contain: -- a valid role from those configured in `allowedRoles` -- a role with the `connector name` +## Example -## Configuration - -Example configuration: - -```properties -edc.api.auth.oauth2.allowedRoles.1.role=connector-admin -edc.api.auth.oauth2.allowedRoles.2.role=connector-management +```json +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "offset": 0, + "limit": 5, + "sortOrder": "ASC", + "sortField": "id", + "filterExpression": [ + { + "operandLeft": "genericSearch", + "operator": "LIKE", + "operandRight": "%test%" + }, + { + "operandLeft": "'https://w3id.org/edc/v0.0.1/ns/assetData'.'https://w3id.org/edc/v0.0.1/ns/dcat-vocabulary'.'http://purl.org/dc/terms/language'", + "operator": "=", + "operandRight": "spanish" + }, + { + "operandLeft": "'https://w3id.org/edc/v0.0.1/ns/assetData'.'https://w3id.org/edc/v0.0.1/ns/dcat-vocabulary'.'http://purl.org/dc/terms/publisher'.'http://www.w3.org/2004/02/skos/core#notation'", + "operator": "=", + "operandRight": "notation-publisher" + } + ] +} ``` - -The `edc.api.auth.oauth2.allowedRoles` will be used by the federated catalog to retrieve the list of allowed roles that can perform requests on the managemente API connector. \ No newline at end of file