From 12e331a59fdf5022a648d217d999c5ed210dbf76 Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Mon, 25 Nov 2024 16:00:36 +0100 Subject: [PATCH 1/4] Handle the cases where the visas are a mix of Fega and non-fega visas --- .../localega/doa/mq/ExportRequestsListener.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java index 4c1ae81..f9a9077 100644 --- a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java +++ b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java @@ -9,6 +9,7 @@ import lombok.extern.slf4j.Slf4j; import no.uio.ifi.localega.doa.dto.DestinationFormat; import no.uio.ifi.localega.doa.dto.ExportRequest; +import no.uio.ifi.localega.doa.model.Dataset; import no.uio.ifi.localega.doa.model.DatasetEventLog; import no.uio.ifi.localega.doa.services.AAIService; import no.uio.ifi.localega.doa.services.MetadataService; @@ -26,6 +27,7 @@ import java.io.InputStream; import java.util.Base64; import java.util.Collection; +import java.util.Objects; /** * RabbitMQ listener that processes incoming export requests. @@ -81,7 +83,17 @@ public void listen(String message) { String stableDatasetId = metadataService.getDataset(datasetsDbTableId).getStableId(); log.info("Reference id {} mapped to dataset id {}", requestedDatasetId, stableDatasetId); requestedDatasetId = stableDatasetId; // use stable dataset id instead of reference to complete the export as normal - Collection approvedMappedDatasetIds = approvedDatasetIds.stream().map(x -> metadataService.getDataset(metadataService.findByReferenceId(x).getDatasetId()).getStableId()).toList(); + Collection approvedMappedDatasetIds = approvedDatasetIds.stream() + .map(x -> { + var reference = metadataService.findByReferenceId(x); + if (reference != null && reference.getDatasetId() != null) { + var dataset = metadataService.getDataset(reference.getDatasetId()); + return dataset != null ? dataset.getStableId() : x; + } else { + return x; + } + }) + .toList(); approvedDatasetIds = approvedMappedDatasetIds; } exportDataset(user, approvedDatasetIds, requestedDatasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); From 9a1c554e74ca4eebac31da9efaa3b98901354aca Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Mon, 25 Nov 2024 16:06:31 +0100 Subject: [PATCH 2/4] remove unused imports --- .../java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java index f9a9077..1521518 100644 --- a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java +++ b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java @@ -9,7 +9,6 @@ import lombok.extern.slf4j.Slf4j; import no.uio.ifi.localega.doa.dto.DestinationFormat; import no.uio.ifi.localega.doa.dto.ExportRequest; -import no.uio.ifi.localega.doa.model.Dataset; import no.uio.ifi.localega.doa.model.DatasetEventLog; import no.uio.ifi.localega.doa.services.AAIService; import no.uio.ifi.localega.doa.services.MetadataService; @@ -27,7 +26,6 @@ import java.io.InputStream; import java.util.Base64; import java.util.Collection; -import java.util.Objects; /** * RabbitMQ listener that processes incoming export requests. From 823880a5816dd930a9e000c7f6e2aff0fced3972 Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Tue, 26 Nov 2024 12:53:57 +0100 Subject: [PATCH 3/4] add tests for dataset reference --- docker-compose-posix-outbox.yml | 6 ++-- docker-compose-s3-outbox.yml | 6 ++-- src/main/resources/application.yml | 28 ++++++++------- .../doa/LocalEGADOAApplicationTests.java | 35 +++++++++++++++++++ test/mock_auth.py | 20 +++++++++-- 5 files changed, 74 insertions(+), 21 deletions(-) diff --git a/docker-compose-posix-outbox.yml b/docker-compose-posix-outbox.yml index 4df21a4..a92a91c 100644 --- a/docker-compose-posix-outbox.yml +++ b/docker-compose-posix-outbox.yml @@ -3,12 +3,12 @@ version: '3.3' services: db: - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.23-postgres + image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.120-postgres ports: - 5432:5432 environment: - - DB_LEGA_IN_PASSWORD=password - - DB_LEGA_OUT_PASSWORD=password + - LEGA_IN_PASSWORD=rootpasswd + - LEGA_OUT_PASSWORD=rootpasswd - POSTGRES_SERVER_CERT=/etc/ega/pg.cert - POSTGRES_SERVER_KEY=/etc/ega/pg.key - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert diff --git a/docker-compose-s3-outbox.yml b/docker-compose-s3-outbox.yml index c25eb9c..f94cce8 100644 --- a/docker-compose-s3-outbox.yml +++ b/docker-compose-s3-outbox.yml @@ -3,12 +3,12 @@ version: '3.3' services: db: - image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.47-postgres + image: ghcr.io/neicnordic/sensitive-data-archive:v0.3.120-postgres ports: - 5432:5432 environment: - - DB_LEGA_IN_PASSWORD=password - - DB_LEGA_OUT_PASSWORD=password + - LEGA_IN_PASSWORD=password + - LEGA_OUT_PASSWORD=password - POSTGRES_SERVER_CERT=/etc/ega/pg.cert - POSTGRES_SERVER_KEY=/etc/ega/pg.key - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3101336..b963bc1 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,13 +9,13 @@ logging: rest.enabled: ${REST_ENABLED:true} server.ssl: - enabled: ${SSL_ENABLED:true} + enabled: ${SSL_ENABLED:false} key-store-type: PKCS12 key-store: file:${KEYSTORE_PATH:/etc/ega/ssl/server.cert} key-store-password: ${KEYSTORE_PASSWORD} spring.rabbitmq: - host: ${BROKER_HOST:private-mq} + host: ${BROKER_HOST:129.177.177.134} port: ${BROKER_PORT:5671} virtual-host: ${BROKER_VHOST:sda} username: ${BROKER_USERNAME:admin} @@ -23,7 +23,7 @@ spring.rabbitmq: ssl: enabled: true algorithm: TLSv1.2 - validate-server-certificate: ${BROKER_VALIDATE:true} + validate-server-certificate: ${BROKER_VALIDATE:false} trust-store-type: PKCS12 key-store-type: PKCS12 @@ -31,12 +31,14 @@ outbox: enabled: ${OUTBOX_ENABLED:true} type: ${OUTBOX_TYPE:POSIX} queue: ${OUTBOX_QUEUE:exportRequests} - location: ${OUTBOX_LOCATION:/ega/outbox/p11-%s/files/} + location: ${OUTBOX_LOCATION:%s/files/} spring: datasource: - url: jdbc:postgresql://${DB_INSTANCE:db}:${DB_PORT:5432}/${POSTGRES_DB:sda}?sslmode=${SSL_MODE:verify-full}&sslrootcert=${ROOT_CERT_PATH:/etc/ega/ssl/CA.cert}&sslcert=${CERT_PATH:/etc/ega/ssl/client.cert}&sslkey=${CERT_KEY:/etc/ega/ssl/client.key} - username: ${POSTGRES_USER:postgres} +# url: jdbc:postgresql://${DB_INSTANCE:129.177.177.134}:${DB_PORT:5432}/${POSTGRES_DB:sda}?sslmode=${SSL_MODE:verify-ca}&sslrootcert=${ROOT_CERT_PATH:test/rootCA.pem}&sslcert=${CERT_PATH:test/localhost.pem}&sslkey=${CERT_KEY:test/localhost-client-key.der} + url: jdbc:postgresql://${DB_INSTANCE:129.177.177.134}:${DB_PORT:5432}/${POSTGRES_DB:sda} + + username: ${POSTGRES_USER:lega_out} password: ${POSTGRES_PASSWORD:rootpasswd} driver-class-name: org.postgresql.Driver jpa: @@ -55,13 +57,13 @@ s3: root-ca: ${S3_ROOT_CERT_PATH:/etc/ssl/certs/ca-certificates.crt} s3.out: - endpoint: ${S3_OUT_ENDPOINT:outbox} - port: ${S3_OUT_PORT:443} + endpoint: ${S3_OUT_ENDPOINT:129.177.177.134} + port: ${S3_OUT_PORT:9000} access-key: ${S3_OUT_ACCESS_KEY:minio} secret-key: ${S3_OUT_SECRET_KEY:miniostorage} region: ${S3_OUT_REGION:us-west-1} bucket: ${S3_OUT_BUCKET:lega} - secure: ${S3_OUT_SECURE:true} + secure: ${S3_OUT_SECURE:false} root-ca: ${S3_OUT_ROOT_CERT_PATH:/etc/ssl/certs/ca-certificates.crt} archive: @@ -72,14 +74,14 @@ ga4gh: # First, the public key is checked. If present, it's used for validating the token and openid-configuration-url is not used. public-key-path: ${PASSPORT_PUBLIC_KEY_PATH:/etc/ega/jwt/passport.pem} # If public key is absent, then openid-configuration-url is used to query JWK and fetch the public key. - openid-configuration-url: ${OPENID_CONFIGURATION_URL:https://login.elixir-czech.org/oidc/.well-known/openid-configuration} + openid-configuration-url: ${OPENID_CONFIGURATION_URL:http://129.177.177.134:8000/openid-configuration} # /userinfo endpoint URL for handling opaque access tokens - userinfo-endpoint-url: ${USERINFO_ENDPOINT_URL:https://login.elixir-czech.org/oidc/userinfo} + userinfo-endpoint-url: ${USERINFO_ENDPOINT_URL:http://129.177.177.134:8000/userinfo} visa: # First, the public key is checked. If present, it's used for validating the token. Otherwise, JKU is used instead (entry in the header). public-key-path: ${VISA_PUBLIC_KEY_PATH:/etc/ega/jwt/visa.pem} crypt4gh: - private-key-path: ${CRYPT4GH_PRIVATE_KEY_PATH:/etc/ega/crypt4gh/key.pem} - private-key-password-path: ${CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH:/etc/ega/crypt4gh/key.pass} + private-key-path: ${CRYPT4GH_PRIVATE_KEY_PATH:test/crypt4gh.sec.pem} + private-key-password-path: ${CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH:test/crypt4gh.pass} diff --git a/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java b/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java index 12fb76a..8361e75 100644 --- a/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java +++ b/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java @@ -78,6 +78,9 @@ public static void setup() { PreparedStatement dataset_event_released = connection.prepareStatement(prepareInsertQueryDatasetEvent("EGAD00010000919", "released", "release")); dataset_event_released.executeUpdate(); + + PreparedStatement datasetReferenceInsert = connection.prepareStatement("INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');"); + datasetReferenceInsert.executeUpdate(); connection.close(); JSONArray tokens = Unirest.get("http://localhost:8000/tokens").asJson().getBody().getArray(); @@ -272,6 +275,38 @@ void testS3ExportRequestDatasetValidToken() { } } + @SneakyThrows + @Test + void testS3ExportRequestReferenceValidToken() { + if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + Assertions.assertTrue(true); + return; + } + export("GDI-NO-10001", true); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); + Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { + byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); + Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); + } + } + + @SneakyThrows + @Test + void testPOSIXExportRequestReferenceValidToken() { + if (System.getenv("OUTBOX_TYPE").equals("S3")) { + Assertions.assertTrue(true); + return; + } + export("GDI-NO-10001", true); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { + byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); + Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); + } + } + @SneakyThrows void export(String id, boolean dataset) { String mqConnectionString = "amqps://admin:guest@localhost:5671/sda"; diff --git a/test/mock_auth.py b/test/mock_auth.py index dfd80d7..5c08b89 100644 --- a/test/mock_auth.py +++ b/test/mock_auth.py @@ -102,6 +102,20 @@ def generate_token(): "exp": 99999999999, "jti": "9fa600d6-4148-47c1-b708-36c4ba2e980e" } + passport_dataset_gdi = { + "iss": "http://129.177.177.134:8000/", + "sub": "requester@elixir-europe.org", + "ga4gh_visa_v1": { + "type": "ControlledAccessGrants", + "value": "https://www.ebi.ac.uk/ega/GDI-NO-10001", + "source": "https://ga4gh.org/duri/no_org", + "by": "dac", + "asserted": 1568699331 + }, + "iat": 1571144438, + "exp": 99999999999, + "jti": "2b322848-506b-492c-914f-47f9da967cdd" + } public_jwk = jwk.dumps(public_key, kty='RSA') private_jwk = jwk.dumps(pem, kty='RSA') dataset_encoded = jwt.encode(header, dataset_payload, private_jwk).decode('utf-8') @@ -110,8 +124,9 @@ def generate_token(): passport_status_encoded = jwt.encode(header, passport_status, private_jwk).decode('utf-8') passport_dataset1_encoded = jwt.encode(header, passport_dataset1, private_jwk).decode('utf-8') passport_dataset2_encoded = jwt.encode(header, passport_dataset2, private_jwk).decode('utf-8') + passport_dataset_gdi_encoded = jwt.encode(header, passport_dataset_gdi, private_jwk).decode('utf-8') return (public_jwk, dataset_encoded, empty_encoded, passport_terms_encoded, passport_status_encoded, - passport_dataset1_encoded, passport_dataset2_encoded) + passport_dataset1_encoded, passport_dataset2_encoded, passport_dataset_gdi_encoded) DATA = generate_token() @@ -143,7 +158,8 @@ async def userinfo(request): DATA[3], DATA[4], DATA[5], - DATA[6] + DATA[6], + DATA[7] ] } return web.json_response(data) From 2aa10951da7a19ab28b653aada9d35abf8572897 Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Tue, 26 Nov 2024 13:07:12 +0100 Subject: [PATCH 4/4] add tests for dataset reference --- .../doa/mq/ExportRequestsListener.java | 12 +------- src/main/resources/application.yml | 28 +++++++++---------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java index 1521518..4c1ae81 100644 --- a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java +++ b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java @@ -81,17 +81,7 @@ public void listen(String message) { String stableDatasetId = metadataService.getDataset(datasetsDbTableId).getStableId(); log.info("Reference id {} mapped to dataset id {}", requestedDatasetId, stableDatasetId); requestedDatasetId = stableDatasetId; // use stable dataset id instead of reference to complete the export as normal - Collection approvedMappedDatasetIds = approvedDatasetIds.stream() - .map(x -> { - var reference = metadataService.findByReferenceId(x); - if (reference != null && reference.getDatasetId() != null) { - var dataset = metadataService.getDataset(reference.getDatasetId()); - return dataset != null ? dataset.getStableId() : x; - } else { - return x; - } - }) - .toList(); + Collection approvedMappedDatasetIds = approvedDatasetIds.stream().map(x -> metadataService.getDataset(metadataService.findByReferenceId(x).getDatasetId()).getStableId()).toList(); approvedDatasetIds = approvedMappedDatasetIds; } exportDataset(user, approvedDatasetIds, requestedDatasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b963bc1..3101336 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -9,13 +9,13 @@ logging: rest.enabled: ${REST_ENABLED:true} server.ssl: - enabled: ${SSL_ENABLED:false} + enabled: ${SSL_ENABLED:true} key-store-type: PKCS12 key-store: file:${KEYSTORE_PATH:/etc/ega/ssl/server.cert} key-store-password: ${KEYSTORE_PASSWORD} spring.rabbitmq: - host: ${BROKER_HOST:129.177.177.134} + host: ${BROKER_HOST:private-mq} port: ${BROKER_PORT:5671} virtual-host: ${BROKER_VHOST:sda} username: ${BROKER_USERNAME:admin} @@ -23,7 +23,7 @@ spring.rabbitmq: ssl: enabled: true algorithm: TLSv1.2 - validate-server-certificate: ${BROKER_VALIDATE:false} + validate-server-certificate: ${BROKER_VALIDATE:true} trust-store-type: PKCS12 key-store-type: PKCS12 @@ -31,14 +31,12 @@ outbox: enabled: ${OUTBOX_ENABLED:true} type: ${OUTBOX_TYPE:POSIX} queue: ${OUTBOX_QUEUE:exportRequests} - location: ${OUTBOX_LOCATION:%s/files/} + location: ${OUTBOX_LOCATION:/ega/outbox/p11-%s/files/} spring: datasource: -# url: jdbc:postgresql://${DB_INSTANCE:129.177.177.134}:${DB_PORT:5432}/${POSTGRES_DB:sda}?sslmode=${SSL_MODE:verify-ca}&sslrootcert=${ROOT_CERT_PATH:test/rootCA.pem}&sslcert=${CERT_PATH:test/localhost.pem}&sslkey=${CERT_KEY:test/localhost-client-key.der} - url: jdbc:postgresql://${DB_INSTANCE:129.177.177.134}:${DB_PORT:5432}/${POSTGRES_DB:sda} - - username: ${POSTGRES_USER:lega_out} + url: jdbc:postgresql://${DB_INSTANCE:db}:${DB_PORT:5432}/${POSTGRES_DB:sda}?sslmode=${SSL_MODE:verify-full}&sslrootcert=${ROOT_CERT_PATH:/etc/ega/ssl/CA.cert}&sslcert=${CERT_PATH:/etc/ega/ssl/client.cert}&sslkey=${CERT_KEY:/etc/ega/ssl/client.key} + username: ${POSTGRES_USER:postgres} password: ${POSTGRES_PASSWORD:rootpasswd} driver-class-name: org.postgresql.Driver jpa: @@ -57,13 +55,13 @@ s3: root-ca: ${S3_ROOT_CERT_PATH:/etc/ssl/certs/ca-certificates.crt} s3.out: - endpoint: ${S3_OUT_ENDPOINT:129.177.177.134} - port: ${S3_OUT_PORT:9000} + endpoint: ${S3_OUT_ENDPOINT:outbox} + port: ${S3_OUT_PORT:443} access-key: ${S3_OUT_ACCESS_KEY:minio} secret-key: ${S3_OUT_SECRET_KEY:miniostorage} region: ${S3_OUT_REGION:us-west-1} bucket: ${S3_OUT_BUCKET:lega} - secure: ${S3_OUT_SECURE:false} + secure: ${S3_OUT_SECURE:true} root-ca: ${S3_OUT_ROOT_CERT_PATH:/etc/ssl/certs/ca-certificates.crt} archive: @@ -74,14 +72,14 @@ ga4gh: # First, the public key is checked. If present, it's used for validating the token and openid-configuration-url is not used. public-key-path: ${PASSPORT_PUBLIC_KEY_PATH:/etc/ega/jwt/passport.pem} # If public key is absent, then openid-configuration-url is used to query JWK and fetch the public key. - openid-configuration-url: ${OPENID_CONFIGURATION_URL:http://129.177.177.134:8000/openid-configuration} + openid-configuration-url: ${OPENID_CONFIGURATION_URL:https://login.elixir-czech.org/oidc/.well-known/openid-configuration} # /userinfo endpoint URL for handling opaque access tokens - userinfo-endpoint-url: ${USERINFO_ENDPOINT_URL:http://129.177.177.134:8000/userinfo} + userinfo-endpoint-url: ${USERINFO_ENDPOINT_URL:https://login.elixir-czech.org/oidc/userinfo} visa: # First, the public key is checked. If present, it's used for validating the token. Otherwise, JKU is used instead (entry in the header). public-key-path: ${VISA_PUBLIC_KEY_PATH:/etc/ega/jwt/visa.pem} crypt4gh: - private-key-path: ${CRYPT4GH_PRIVATE_KEY_PATH:test/crypt4gh.sec.pem} - private-key-password-path: ${CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH:test/crypt4gh.pass} + private-key-path: ${CRYPT4GH_PRIVATE_KEY_PATH:/etc/ega/crypt4gh/key.pem} + private-key-password-path: ${CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH:/etc/ega/crypt4gh/key.pass}