feat: Support available credentials providers for GCP blob storage #239…

… (#245)

Co-authored-by: Aliaksandr Stsiapanay <aliaksandr_stsiapanay@epam.com>

README.md

@@ -52,8 +52,8 @@ Static settings are used on startup and cannot be changed while application is r
 | client.*                                   | -                  | Vertx HTTP client settings for outbound requests.
 | storage.provider                           | -                  | Specifies blob storage provider. Supported providers: s3, aws-s3, azureblob, google-cloud-storage, filesystem
 | storage.endpoint                           | -                  | Optional. Specifies endpoint url for s3 compatible storages
-| storage.identity                           | -                  | Blob storage access key. Can be optional for filesystem and aws-s3 providers
-| storage.credential                         | -                  | Blob storage secret key. Can be optional for filesystem and aws-s3 providers
+| storage.identity                           | -                  | Blob storage access key. Can be optional for filesystem, aws-s3, google-cloud-storage providers
+| storage.credential                         | -                  | Blob storage secret key. Can be optional for filesystem, aws-s3, google-cloud-storage providers
 | storage.bucket                             | -                  | Blob storage bucket
 | storage.overrides.*                        | -                  | Key-value pairs to override storage settings
 | storage.createBucket                       | false              | Indicates whether bucket should be created on start-up
@@ -70,6 +70,49 @@ Static settings are used on startup and cannot be changed while application is r
 | redis.clusterServersConfig.nodeAddresses   | -                  | Json array with Redis cluster server addresses, e.g. ["redis://host1:port1","redis://host2:port2"]
 | invitations.ttlInSeconds                   | 259200             | Invitation time to live in seconds
 
+### Google Cloud Storage
+
+There are two types of credentials providers supported:
+ - User credentials. You can create a service account and authenticate using its private key obtained from Developer console
+ - Temporary credentials. Application default credentials (ADC)
+
+#### User credentials
+
+You should set `storage.credential` to a path to the private key JSON file and `storage.identity` must be unset.
+See example below:
+```
+{
+  "type": "service_account",
+  "project_id": "<your_project_id>",
+  "private_key_id": "<your_project_key_id>",
+  "private_key": "-----BEGIN PRIVATE KEY-----\n<your_private_key>\n-----END PRIVATE KEY-----\n",
+  "client_email": "gcp-dial-core@<your_project_id>.iam.gserviceaccount.com",
+  "client_id": "<client_id>",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/gcp-dial-core.iam.gserviceaccount.com",
+  "universe_domain": "googleapis.com"
+}
+```
+Otherwise `storage.credential` is a private key in PEM format and `storage.identity` is a client email address.
+
+#### Temporary credentials
+
+You should follow [instructions](https://cloud.google.com/kubernetes-engine/docs/concepts/workload-identity) to setup your pod in GKE.
+`storage.credential` and `storage.identity` must be unset.
+JClouds property `jclouds.oauth.credential-type` should be set `bearerTokenCredentials`, e.g.
+
+```
+{
+  "storage": {
+    "overrides": {
+      "jclouds.oauth.credential-type": "bearerTokenCredentials"
+    }
+  }
+}
+```
+
 ### Redis
 The Redis can be used as a cache with volatile-* eviction policies:
 ```

build.gradle

@@ -50,6 +50,8 @@ dependencies {
     implementation 'org.redisson:redisson:3.25.1'
     implementation group: 'com.amazonaws', name: 'aws-java-sdk-core', version: '1.12.663'
     implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.12.663'
+    implementation group: 'com.google.auth', name: 'google-auth-library-oauth2-http', version: '1.23.0'
+
 
     runtimeOnly 'com.epam.deltix:gflog-slf4j:3.0.5'
 

src/main/java/com/epam/aidial/core/storage/BlobStorage.java

@@ -5,6 +5,8 @@
 import com.epam.aidial.core.data.MetadataBase;
 import com.epam.aidial.core.data.ResourceFolderMetadata;
 import com.epam.aidial.core.data.ResourceType;
+import com.epam.aidial.core.storage.credential.CredentialProvider;
+import com.epam.aidial.core.storage.credential.CredentialProviderFactory;
 import io.vertx.core.buffer.Buffer;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -63,7 +65,7 @@ public BlobStorage(Storage config) {
         if (overrides != null) {
             builder.overrides(overrides);
         }
-        CredentialProvider credentialProvider = getCredentialProvider(StorageProvider.from(provider), config.getIdentity(), config.getCredential());
+        CredentialProvider credentialProvider = CredentialProviderFactory.create(provider, config.getIdentity(), config.getCredential());
         builder.credentialsSupplier(credentialProvider::getCredentials);
         this.storeContext = builder.buildView(BlobStoreContext.class);
         this.blobStore = storeContext.getBlobStore();
@@ -302,14 +304,6 @@ private static ContentMetadata buildContentMetadata(String contentType) {
         return BaseMutableContentMetadata.fromContentMetadata(contentMetadata);
     }
 
-    private static CredentialProvider getCredentialProvider(StorageProvider provider, String identity, String credential) {
-        return switch (provider) {
-            case S3, AZURE_BLOB, GOOGLE_CLOUD_STORAGE -> new DefaultCredentialProvider(identity, credential);
-            case FILESYSTEM -> new DefaultCredentialProvider("identity", "credential");
-            case AWS_S3 -> new AwsCredentialProvider(identity, credential);
-        };
-    }
-
     private void createBucketIfNeeded(Storage config) {
         if (config.isCreateBucket() && !storeContext.getBlobStore().containerExists(bucketName)) {
             storeContext.getBlobStore().createContainerInLocation(null, bucketName);

src/main/java/com/epam/aidial/core/storage/BlobWriteStream.java

@@ -179,6 +179,6 @@ public synchronized void abortUpload(Throwable ex) {
             errorHandler.handle(ex);
         }
 
-        log.warn("Multipart upload aborted");
+        log.warn("Multipart upload aborted", ex);
     }
 }

src/main/java/com/epam/aidial/core/storage/AwsCredentialProvider.java → src/main/java/com/epam/aidial/core/storage/credential/AwsCredentialProvider.java

@@ -1,4 +1,4 @@
-package com.epam.aidial.core.storage;
+package com.epam.aidial.core.storage.credential;
 
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSSessionCredentials;

src/main/java/com/epam/aidial/core/storage/CredentialProvider.java → src/main/java/com/epam/aidial/core/storage/credential/CredentialProvider.java

@@ -1,4 +1,4 @@
-package com.epam.aidial.core.storage;
+package com.epam.aidial.core.storage.credential;
 
 import org.jclouds.domain.Credentials;
 

src/main/java/com/epam/aidial/core/storage/credential/CredentialProviderFactory.java

@@ -0,0 +1,17 @@
+package com.epam.aidial.core.storage.credential;
+
+import com.epam.aidial.core.storage.StorageProvider;
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class CredentialProviderFactory {
+    public static CredentialProvider create(String providerName, String identity, String credential) {
+        StorageProvider provider = StorageProvider.from(providerName);
+        return switch (provider) {
+            case S3, AZURE_BLOB -> new DefaultCredentialProvider(identity, credential);
+            case GOOGLE_CLOUD_STORAGE -> new GcpCredentialProvider(identity, credential);
+            case FILESYSTEM -> new DefaultCredentialProvider("identity", "credential");
+            case AWS_S3 -> new AwsCredentialProvider(identity, credential);
+        };
+    }
+}

src/main/java/com/epam/aidial/core/storage/DefaultCredentialProvider.java → src/main/java/com/epam/aidial/core/storage/credential/DefaultCredentialProvider.java

@@ -1,4 +1,4 @@
-package com.epam.aidial.core.storage;
+package com.epam.aidial.core.storage.credential;
 
 import org.jclouds.domain.Credentials;
 

src/main/java/com/epam/aidial/core/storage/credential/GcpCredentialProvider.java

@@ -0,0 +1,68 @@
+package com.epam.aidial.core.storage.credential;
+
+import com.google.auth.oauth2.AccessToken;
+import com.google.auth.oauth2.GoogleCredentials;
+import com.google.common.io.Files;
+import lombok.SneakyThrows;
+import org.jclouds.domain.Credentials;
+import org.jclouds.googlecloud.GoogleCredentialsFromJson;
+
+import java.io.File;
+import java.time.Instant;
+import java.util.Date;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+public class GcpCredentialProvider implements CredentialProvider {
+
+    private static final long EXPIRATION_WINDOW_IN_MS = 10_000;
+
+    private Credentials credentials;
+
+    private AccessToken accessToken;
+
+    private GoogleCredentials googleCredentials;
+
+    /**
+     *
+     * @param identity client email address
+     * @param credential could be private key or path to JSON file where the private resides
+     */
+    @SneakyThrows
+    public GcpCredentialProvider(String identity, String credential) {
+        if (identity != null && credential != null) {
+            // credential is a client email address
+            this.credentials = new Credentials(identity, credential);
+        } else if (credential != null) {
+            // credential is a path to private key JSON file
+            this.credentials = getCredentialsFromJsonKeyFile(credential);
+        } else {
+            // use temporary credential provided by GCP
+            this.googleCredentials = GoogleCredentials.getApplicationDefault();
+        }
+    }
+
+    @Override
+    public Credentials getCredentials() {
+        if (credentials != null) {
+            return credentials;
+        }
+        return getTemporaryCredentials();
+    }
+
+    @SneakyThrows
+    private synchronized Credentials getTemporaryCredentials() {
+        Date expireAt = Date.from(Instant.ofEpochMilli(System.currentTimeMillis() - EXPIRATION_WINDOW_IN_MS));
+        if (accessToken == null || expireAt.after(accessToken.getExpirationTime())) {
+            accessToken = googleCredentials.refreshAccessToken();
+        }
+        return new Credentials("", accessToken.getTokenValue());
+    }
+
+    @SneakyThrows
+    private static Credentials getCredentialsFromJsonKeyFile(String filename) {
+        String fileContents = Files.asCharSource(new File(filename), UTF_8).read();
+        GoogleCredentialsFromJson credentialSupplier = new GoogleCredentialsFromJson(fileContents);
+        return credentialSupplier.get();
+    }
+}