[#6504] Supports s3 path-style-access properties in Iceberg REST server

apache · Feb 27, 2025 · 5da5bb5 · 5da5bb5
1 parent 1650c20
commit 5da5bb5
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 5 deletions.
diff --git a/...common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/...common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java
@@ -41,6 +41,7 @@ public class IcebergConstants {
   public static final String ICEBERG_S3_ACCESS_KEY_ID = "s3.access-key-id";
   public static final String ICEBERG_S3_SECRET_ACCESS_KEY = "s3.secret-access-key";
   public static final String ICEBERG_S3_TOKEN = "s3.session-token";
+  public static final String ICEBERG_S3_PATH_STYLE_ACCESS = "s3.path-style-access";
   public static final String AWS_S3_REGION = "client.region";
 
   public static final String ICEBERG_OSS_ENDPOINT = "oss.endpoint";

diff --git a/.../src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java b/.../src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergPropertiesUtils.java
@@ -52,6 +52,8 @@ public class IcebergPropertiesUtils {
     map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, IcebergConstants.ICEBERG_S3_ACCESS_KEY_ID);
     map.put(
         S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, IcebergConstants.ICEBERG_S3_SECRET_ACCESS_KEY);
+    map.put(
+        S3Properties.GRAVITINO_S3_PATH_STYLE_ACCESS, IcebergConstants.ICEBERG_S3_PATH_STYLE_ACCESS);
     // OSS
     map.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, IcebergConstants.ICEBERG_OSS_ENDPOINT);
     map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, IcebergConstants.ICEBERG_OSS_ACCESS_KEY_ID);

diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/storage/S3Properties.java
@@ -40,5 +40,8 @@ public class S3Properties {
   // The S3 credentials provider class name.
   public static final String GRAVITINO_S3_CREDS_PROVIDER = "s3-creds-provider";
 
+  // The S3 path style access flag.
+  public static final String GRAVITINO_S3_PATH_STYLE_ACCESS = "s3-path-style-access";
+
   private S3Properties() {}
 }
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
@@ -107,11 +107,12 @@ Please refer to [Credential vending](./security/credential-vending.md) for more
 
 #### S3 configuration
 
-| Configuration item                                 | Description                                                                                                                                                                                                         | Default value | Required                                       | Since Version    |
-|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------------------------|------------------|
-| `gravitino.iceberg-rest.io-impl`                   | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3.                                                                                                                     | (none)        | No                                             | 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-endpoint`               | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none)        | No                                             | 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-region`                 | The region of the S3 service, like `us-west-2`.                                                                                                                                                                     | (none)        | No                                             | 0.6.0-incubating |
+| Configuration item                            | Description                                                                                                                                                                                                         | Default value | Required                                       | Since Version    |
+|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------------------------|------------------|
+| `gravitino.iceberg-rest.io-impl`              | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3.                                                                                                                     | (none)        | No                                             | 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-endpoint`          | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none)        | No                                             | 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-region`            | The region of the S3 service, like `us-west-2`.                                                                                                                                                                     | (none)        | No                                             | 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-path-style-access` | Whether to use path style access for S3.                                                                                                                                                                            | true          | No                                             | 0.9.0-incubating |
 
 For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`, you could config it directly by `gravitino.iceberg-rest.s3.sse.type`.
 

diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java
@@ -144,6 +144,13 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig {
           .stringConf()
           .create();
 
+  public static final ConfigEntry<Boolean> S3_PATH_STYLE_ACCESS =
+      new ConfigBuilder(S3Properties.GRAVITINO_S3_PATH_STYLE_ACCESS)
+          .doc("Whether to use path style access for S3")
+          .version(ConfigConstants.VERSION_0_9_0)
+          .booleanConf()
+          .createWithDefault(true);
+
   public static final ConfigEntry<String> OSS_ENDPOINT =
       new ConfigBuilder(OSSProperties.GRAVITINO_OSS_ENDPOINT)
           .doc("The endpoint of Aliyun OSS service")