Skip to content

Commit

Permalink
fix: Fixing Transfer Issues (Digital-Ecosystems#57)
Browse files Browse the repository at this point in the history 
* fix: transfer bucketName always null and transfer blobName ignored

* fix: transfer bucketName always null and transfer blobName ignored

* fix: shared bucket access denied (Digital-Ecosystems#56)
  • Loading branch information
@jannotti-glaucio
jannotti-glaucio authored Dec 13, 2023
1 parent 28a00f9 commit bc919e5
Show file tree
Hide file tree
Showing 20 changed files with 244 additions and 391 deletions.
14 changes: 0 additions & 14 deletions ...-extension/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,36 +14,22 @@

package com.ionos.edc.extension.s3.api;

import io.minio.Result;
import io.minio.messages.Item;
import org.eclipse.edc.runtime.metamodel.annotation.ExtensionPoint;

import com.ionos.edc.extension.s3.connector.ionosapi.TemporaryKey;

import java.io.ByteArrayInputStream;



@ExtensionPoint
public interface S3ConnectorApi {

void s3ConnectorApi(String endpoint, String accessKey, String secretKey, String token);

void createBucket(String bucketName);

void deleteBucket(String bucketName);

boolean bucketExists(String bucketName);

void uploadFile(String bucketName, String fileName, String path);

void uploadParts(String bucketName, String fileName, ByteArrayInputStream part);

byte[] getFile(String bucketName, String fileName);

Result<Item> listItems(String bucketName);

void deleteFile(String bucketName, String fileName);

TemporaryKey createTemporaryKey();

Expand Down
132 changes: 21 additions & 111 deletions ...ension/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/api/S3ConnectorApiImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,15 @@

import io.minio.BucketExistsArgs;
import io.minio.GetObjectArgs;
import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.PutObjectArgs;
import io.minio.RemoveBucketArgs;
import io.minio.RemoveObjectArgs;
import io.minio.Result;
import io.minio.UploadObjectArgs;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidResponseException;
import io.minio.errors.ServerException;
import io.minio.errors.XmlParserException;
import io.minio.messages.Item;

import java.io.ByteArrayInputStream;
import java.io.IOException;
Expand All @@ -47,169 +41,85 @@ public class S3ConnectorApiImpl implements S3ConnectorApi {

MinioConnector minConnector = new MinioConnector();
HttpConnector ionosApi = new HttpConnector();

private MinioClient minioClient;
private String region;
private String token;
private final String region;
private final String token;


public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, String token) {
if(accessKey != null && secretKey != null && endpoint !=null)
this.minioClient = minConnector.connect(endpoint, accessKey, secretKey);
this.region = getRegion(endpoint);
this.token = token;
}

@Override
public void s3ConnectorApi(String endpoint, String accessKey, String secretKey, String token) {
if(accessKey != null && secretKey != null && endpoint !=null)
this.minioClient = minConnector.connect(endpoint, accessKey, secretKey);
public S3ConnectorApiImpl(String endpoint, String accessKey, String secretKey, String token) {
if (accessKey != null && secretKey != null && endpoint != null)
this.minioClient = minConnector.connect(endpoint, accessKey, secretKey);
this.region = getRegion(endpoint);
this.token = token;
this.token = token;
}

@Override
public void createBucket(String bucketName) {
if (!bucketExists(bucketName.toLowerCase())) {
// Make a new bucket'.
try {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName.toLowerCase()).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {
e.printStackTrace();
}
}

}

@Override
public void deleteBucket(String bucketName) {

if (bucketExists(bucketName.toLowerCase())) {
// Remove a bucket.
try {
minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName.toLowerCase()).build());
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName.toLowerCase()).region(region).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {

InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {
e.printStackTrace();
}
}
}

@Override
public void uploadFile(String bucketName, String fileName, String path) {

if (!bucketExists(bucketName.toLowerCase())){

createBucket(bucketName.toLowerCase());
}

try {

minioClient.uploadObject(UploadObjectArgs.builder().bucket(bucketName.toLowerCase()).object(fileName)
.filename(path).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {

e.printStackTrace();

}
}


@Override
public void uploadParts(String bucketName, String fileName, ByteArrayInputStream part) {

if (!bucketExists(bucketName.toLowerCase())){

if (!bucketExists(bucketName.toLowerCase())) {
createBucket(bucketName.toLowerCase());
}

try {

minioClient.putObject(PutObjectArgs.builder().bucket(bucketName).object(fileName).stream(part, part.available(), -1).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {

e.printStackTrace();
}

minioClient.putObject(PutObjectArgs.builder().bucket(bucketName.toLowerCase()).region(region).object(fileName).stream(part, part.available(), -1).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {
e.printStackTrace();
}
}

@Override
public byte[] getFile(String bucketName, String fileName) {

if (!bucketExists(bucketName.toLowerCase())) {
return null;
}

InputStream stream;
try {
stream = minioClient.getObject(GetObjectArgs.builder().bucket(bucketName).object(fileName).build());
stream = minioClient.getObject(GetObjectArgs.builder().bucket(bucketName.toLowerCase()).region(region).object(fileName).build());
return stream.readAllBytes();
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {

e.printStackTrace();
return null;
}

}

@Override
public Result<Item> listItems(String bucketName) {

Iterable<Result<Item>> results = minioClient
.listObjects(ListObjectsArgs.builder().bucket(bucketName.toLowerCase()).build());

return results.iterator().next();
}

@Override
public void deleteFile(String bucketName, String fileName) {

try {
minioClient.removeObject(RemoveObjectArgs.builder().bucket(bucketName.toLowerCase()).object(fileName).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {

e.printStackTrace();
}
}

@Override
public boolean bucketExists(String bucketName) {
boolean found = false;
try {

found = minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName.toLowerCase()).region(this.region).build());
return minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName.toLowerCase()).region(this.region).build());
} catch (InvalidKeyException | ErrorResponseException | InsufficientDataException | InternalException |
InvalidResponseException | NoSuchAlgorithmException | ServerException | XmlParserException |
IllegalArgumentException | IOException e) {

e.printStackTrace();
return false;
}
return found;
}

@Override
public TemporaryKey createTemporaryKey() {

return ionosApi.createTemporaryKey(token);

}

@Override
public void deleteTemporaryKey(String accessKey) {

ionosApi.deleteTemporaryAccount(token,accessKey);

}

private String getRegion(String endpoint) {
Expand Down
21 changes: 3 additions & 18 deletions ...core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/configuration/S3CoreExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,30 +19,20 @@
import org.eclipse.edc.runtime.metamodel.annotation.Extension;
import org.eclipse.edc.runtime.metamodel.annotation.Inject;
import org.eclipse.edc.runtime.metamodel.annotation.Provides;
import org.eclipse.edc.runtime.metamodel.annotation.Setting;
import org.eclipse.edc.spi.monitor.Monitor;
import org.eclipse.edc.spi.security.Vault;
import org.eclipse.edc.spi.system.ServiceExtension;
import org.eclipse.edc.spi.system.ServiceExtensionContext;
import org.eclipse.edc.spi.security.CertificateResolver;
import org.eclipse.edc.spi.security.PrivateKeyResolver;
//import org.eclipse.edc.spi.system.vault.NoopCertificateResolver;
//import org.eclipse.edc.spi.system.vault.NoopPrivateKeyResolver;

@Provides(S3ConnectorApi.class)
@Extension(value = S3CoreExtension.NAME)
public class S3CoreExtension implements ServiceExtension {

public static final String NAME = "IonosS3";
@Setting
private static final String IONOS_ACCESS_KEY = "edc.ionos.access.key";
@Setting
private static final String IONOS_SECRET_KEY = "edc.ionos.secret.key";
@Setting
private static final String IONOS_ENDPOINT = "edc.ionos.endpoint";
@Setting
private static final String IONOS_TOKEN = "edc.ionos.token";
@Setting
private static final String IONOS_WITH_VAULT = "edc.ionos.vault";

@Inject
private Vault vault;
Expand All @@ -57,10 +47,12 @@ public String name() {

@Override
public void initialize(ServiceExtensionContext context) {

var accessKey = vault.resolveSecret(IONOS_ACCESS_KEY);
var secretKey = vault.resolveSecret(IONOS_SECRET_KEY);
var endPoint = vault.resolveSecret(IONOS_ENDPOINT);
var token = vault.resolveSecret(IONOS_TOKEN);

if(accessKey == null || secretKey == null || endPoint ==null) {
monitor.warning("Couldn't connect or the vault didn't return values, falling back to ConfigMap Configuration");
accessKey = context.getSetting(IONOS_ACCESS_KEY, IONOS_ACCESS_KEY);
Expand All @@ -71,12 +63,5 @@ public void initialize(ServiceExtensionContext context) {

var s3Api = new S3ConnectorApiImpl(endPoint, accessKey, secretKey, token);
context.registerService(S3ConnectorApi.class, s3Api);

// var privateKeyResolver = new NoopPrivateKeyResolver();
// context.registerService(PrivateKeyResolver.class, privateKeyResolver);

// var certificateResolver = new NoopCertificateResolver();
// context.registerService(CertificateResolver.class, certificateResolver);
}

}
12 changes: 7 additions & 5 deletions ...sion/core-ionos-s3/src/main/java/com/ionos/edc/extension/s3/schema/IonosBucketSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@

package com.ionos.edc.extension.s3.schema;

import static org.eclipse.edc.spi.CoreConstants.EDC_NAMESPACE;

public interface IonosBucketSchema {
String TYPE = "IonosS3";
String STORAGE_NAME = "storage";
String BUCKET_NAME = "bucketName";
String BLOB_NAME = "blobName";
String ACCESS_KEY_ID = "accessKey";
String SECRET_ACCESS_KEY = "secretKey";
String STORAGE_NAME = EDC_NAMESPACE + "storage";
String BUCKET_NAME = EDC_NAMESPACE + "bucketName";
String BLOB_NAME = EDC_NAMESPACE + "blobName";
String ACCESS_KEY_ID = EDC_NAMESPACE + "accessKey";
String SECRET_ACCESS_KEY = EDC_NAMESPACE + "secretKey";
}
29 changes: 14 additions & 15 deletions ...ion/data-plane-ionos-s3/src/main/java/com/ionos/edc/dataplane/ionos/s3/IonosDataSink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,27 +18,33 @@
import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource;
import org.eclipse.edc.connector.dataplane.spi.pipeline.StreamResult;
import org.eclipse.edc.connector.dataplane.util.sink.ParallelSink;
import org.eclipse.edc.spi.response.ResponseStatus;
import org.jetbrains.annotations.NotNull;

import java.io.ByteArrayInputStream;
import java.util.List;
import java.util.Objects;

import static java.lang.String.format;

public class IonosDataSink extends ParallelSink {

private S3ConnectorApi s3Api;
private String bucketName;
private String blobName;
private String accessKey;
private String secretkey;

private IonosDataSink() {}

@Override
protected StreamResult<Void> transferParts(List<DataSource.Part> parts) {

for (var part : parts) {
for (DataSource.Part part : parts) {
String blobName;
if (this.blobName != null) {
blobName = this.blobName;
} else {
blobName = part.name();
}

try (var input = part.openStream()) {
s3Api.uploadParts(bucketName, blobName, new ByteArrayInputStream(input.readAllBytes()));
} catch (Exception e) {
Expand All @@ -57,6 +63,7 @@ private StreamResult<Void> uploadFailure(Exception e, String blobName) {
}

public static class Builder extends ParallelSink.Builder<Builder, IonosDataSink> {

private Builder() {
super(new IonosDataSink());
}
Expand All @@ -79,18 +86,10 @@ public Builder blobName(String blobName) {
sink.blobName = blobName;
return this;
}

public Builder accessKey(String accessKey) {
sink.accessKey = accessKey;
return this;
}

public Builder secretkey(String secretkey) {
sink.secretkey = secretkey;
return this;
}

@Override
protected void validate() {}
protected void validate() {
Objects.requireNonNull(sink.bucketName, "Bucket Name is required");
}
}
}
Loading

0 comments on commit bc919e5

Please sign in to comment.