From 80bf87114a0d4e556ae66484695dff651b35528d Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Tue, 12 Dec 2023 21:15:35 -0600 Subject: [PATCH 01/51] [MINOR] fix(docs): Fix the wrong link in the `README.md` (#1128) ### What changes were proposed in this pull request? Fix the wrong link in the `README.md` ### Why are the changes needed? If we don't have this pr, It will report an error if we click the lick ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? I click the link locally. Co-authored-by: Heng Qin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc4e099353c..ef6030e2722 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ To stop the Gravitino server, please run: ### Using Trino with Gravitino -Gravitino provides a Trino connector to access the metadata in Gravitino. To use Trino with Gravitino, please follow the [trino-gravitino-connector doc](docs/trino-gravitino-connector.md). +Gravitino provides a Trino connector to access the metadata in Gravitino. To use Trino with Gravitino, please follow the [trino-gravitino-connector doc](docs/trino-connector/index.md). ## Development guide From 94aac034ac06a6bdab84ce9777dbed85263caecf Mon Sep 17 00:00:00 2001 From: FANNG Date: Wed, 13 Dec 2023 15:39:24 +0800 Subject: [PATCH 02/51] [MINOR] docs(lakehouse-iceberg): improvement lakehouse iceberg document (#1131) ### What changes were proposed in this pull request? 1. add "Manage Metadata Using Gravitino" link to lakehouse iceberg document 2. merge info blocks ### Why are the changes needed? more user friendly ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? just docs --- docs/apache-hive-catalog.md | 4 +++- docs/iceberg-rest-service.md | 3 --- docs/jdbc-mysql-catalog.md | 6 ++++-- docs/jdbc-postgresql-catalog.md | 11 +++++++++-- docs/lakehouse-iceberg-catalog.md | 19 ++++++++----------- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md index 3c0d5bf7aaf..c900bd7dcdb 100644 --- a/docs/apache-hive-catalog.md +++ b/docs/apache-hive-catalog.md @@ -36,7 +36,7 @@ The Hive catalog supports to create, update, and delete databases and tables in ### Catalog operations -see [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations). +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. ## Schema @@ -136,6 +136,8 @@ Hive automatically adds and manages some reserved properties. Users aren't allow ### Table operations +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details. + #### Alter operations Gravitino has already defined a unified set of [metadata operation interfaces](./manage-metadata-using-gravitino#alter-a-table), and almost all [Hive Alter operations](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-AlterTable/Partition/Column) have corresponding table update request which enable you to change the struct of an existing table. diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 4e6f266b71b..4908bdd6e55 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -20,9 +20,6 @@ The Gravitino Iceberg REST Server follows the [Apache Iceberg REST API specifica :::info Builds with Apache Iceberg `1.3.1`. The Apache Iceberg table format version is `1` by default. -::: - -:::info Builds with Hadoop 2.10.x, there may compatibility issue when accessing Hadoop 3.x clusters. ::: diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index 7d54fd976cd..235ab6a629b 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -44,7 +44,7 @@ You must download the corresponding JDBC driver to the `catalogs/jdbc-mysql/libs ### Catalog operations -see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#catalogs-operations). +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. ## Schema @@ -61,7 +61,7 @@ see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#catalogs ### Schema operations -see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#schemas-operations). +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations) for more details. ## Table @@ -102,6 +102,8 @@ MySQL doesn't support Gravitino `Boolean` `Fixed` `Struct` `List` `Map` `Timesta ### Table operations +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details. + #### Alter table operations Supports operations: diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index 675c6590b1d..dfc8a80c9ee 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -50,7 +50,7 @@ In PostgreSQL, the database corresponds to the Gravitino catalog, and the schema ### Catalog operations -see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#catalogs-operations). +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. ## Schema @@ -62,13 +62,17 @@ see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#catalogs - Supports cascade drop schema. ### Schema properties + - Doesn't are schema property settings. ### Schema operations -see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#schemas-operations). + +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations) for more details. ## Table + ### Table capabilities + - Gravitino table corresponds to the PostgreSQL table. - Supports DDL operation for PostgreSQL tables. - Doesn't support setting certain column properties, such as default value and check constraints. @@ -76,6 +80,7 @@ see [Manage metadata using Gravitino](./manage-metadata-using-gravitino#schemas- - Doesn't support table property settings. #### Table column types + | Gravitino Type | PostgreSQL Type | |----------------|-------------------------------| | `Boolean` | `boolean` | @@ -105,6 +110,8 @@ PostgreSQL doesn't support Gravitino `Fixed` `Struct` `List` `Map` `IntervalDay` ### Table operations +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details. + #### Alter table operations Supports operations: diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 95637f2f4fb..5e5a2cef70d 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -17,9 +17,6 @@ Gravitino provides the ability to manage Apache Iceberg metadata. :::info Builds with Apache Iceberg `1.3.1`. The Apache Iceberg table format version is `1` by default. -::: - -:::info Builds with Hadoop 2.10.x, there may compatibility issue when accessing Hadoop 3.x clusters. ::: @@ -58,7 +55,9 @@ You must download the corresponding JDBC driver to the `catalogs/lakehouse-icebe ### Catalog operations -## Schema +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. + +## Schema ### Schema capabilities @@ -70,7 +69,9 @@ You could put properties except `comment`. ### Schema operations -## Table +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations) for more details. + +## Table ### Table capabilities @@ -88,9 +89,6 @@ Supports transforms: :::info Iceberg doesn't support multi fields in `BucketTransform`. -::: - -:::info Iceberg doesn't support `ApplyTransform`, `RangeTransform` and `ListTransform`. ::: @@ -163,6 +161,8 @@ The Gravitino server doesn't allow passing the following reserved fields. ### Table operations +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details. + #### Alter table operations Supports operations: @@ -181,9 +181,6 @@ Supports operations: :::info The default column position is `LAST` when you add a column. If you add a non nullability column, there may be compatibility issues. -::: - -:::info Iceberg just supports updating primitive types. ::: From aab86ed7347b876fcdb369dd88be8cb66b6cd9d3 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Wed, 13 Dec 2023 03:10:46 -0600 Subject: [PATCH 03/51] [#1136] improvement(docs): Add the `md` suffix for every link in the markdown (#1137) ### What changes were proposed in this pull request? Add the `md` suffix for every link in the markdown ### Why are the changes needed? Fix: #1136 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Tested in Docunausars and markdown locally. --------- Co-authored-by: Heng Qin --- docs/apache-hive-catalog.md | 6 +-- docs/getting-started.md | 14 +++--- docs/gravitino-server-config.md | 16 +++---- docs/how-to-install.md | 6 +-- docs/how-to-test.md | 2 +- docs/iceberg-rest-service.md | 4 +- docs/index.md | 52 +++++++++++----------- docs/jdbc-mysql-catalog.md | 4 +- docs/jdbc-postgresql-catalog.md | 4 +- docs/lakehouse-iceberg-catalog.md | 2 +- docs/manage-metadata-using-gravitino.md | 44 +++++++++--------- docs/publish-docker-images.md | 2 +- docs/security.md | 4 +- docs/trino-connector/catalog-hive.md | 2 +- docs/trino-connector/catalog-iceberg.md | 2 +- docs/trino-connector/catalog-mysql.md | 2 +- docs/trino-connector/catalog-postgresql.md | 2 +- docs/trino-connector/index.md | 20 ++++----- docs/trino-connector/installation.md | 6 +-- docs/trino-connector/supported-catalog.md | 10 ++--- docs/trino-connector/trino-connector.md | 2 +- 21 files changed, 103 insertions(+), 103 deletions(-) diff --git a/docs/apache-hive-catalog.md b/docs/apache-hive-catalog.md index c900bd7dcdb..399f5df4ebd 100644 --- a/docs/apache-hive-catalog.md +++ b/docs/apache-hive-catalog.md @@ -36,7 +36,7 @@ The Hive catalog supports to create, update, and delete databases and tables in ### Catalog operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details. ## Schema @@ -55,7 +55,7 @@ The following table lists predefined schema properties for the Hive database. Ad ### Schema operations -see [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations). +see [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations). ## Table @@ -140,7 +140,7 @@ Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravit #### Alter operations -Gravitino has already defined a unified set of [metadata operation interfaces](./manage-metadata-using-gravitino#alter-a-table), and almost all [Hive Alter operations](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-AlterTable/Partition/Column) have corresponding table update request which enable you to change the struct of an existing table. +Gravitino has already defined a unified set of [metadata operation interfaces](./manage-metadata-using-gravitino.md#alter-a-table), and almost all [Hive Alter operations](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-AlterTable/Partition/Column) have corresponding table update request which enable you to change the struct of an existing table. The following table lists the mapping relationship between Hive Alter operations and Gravitino table update request. ##### Alter table diff --git a/docs/getting-started.md b/docs/getting-started.md index 77026643217..37c0d55d93e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -62,10 +62,10 @@ To begin using Gravitino on AWS, follow these steps: 5. Install Gravitino on the instance: You can install Gravitino from the binary release package or Docker image, please follow the - [how-to-install](./how-to-install) to install Gravitino. + [how-to-install](./how-to-install.md) to install Gravitino. Or, you can install Gravitino from scratch, please follow the documentation of - [how-to-build](./how-to-build) and [how-to-install](./how-to-install) to install Gravitino. + [how-to-build](./how-to-build.md) and [how-to-install](./how-to-install.md) to install Gravitino. 6. Start Gravitino using the gravitino.sh script: @@ -109,10 +109,10 @@ To begin using Gravitino on GCP, follow these steps: 5. Install Gravitino on the instance: You can install Gravitino from the binary release package or Docker image, please follow the - [how-to-install](./how-to-install) to install Gravitino. + [how-to-install](./how-to-install.md) to install Gravitino. Or, you can install Gravitino from scratch, please follow the documentation of - [how-to-build](./how-to-build) and [how-to-install](./how-to-install) to install Gravitino. + [how-to-build](./how-to-build.md) and [how-to-install](./how-to-install.md) to install Gravitino. 6. Start Gravitino using the gravitino.sh script: @@ -136,10 +136,10 @@ To use Gravitino locally on macOS or Linux, follow similar steps: 2. Install Gravitino: You can install Gravitino from the binary release package or Docker image, please follow the - [how-to-install](./how-to-install) to install Gravitino. + [how-to-install](./how-to-install.md) to install Gravitino. Or, you can install Gravitino from scratch, please follow the documentation of - [how-to-build](./how-to-build) and [how-to-install](./how-to-install) to install Gravitino. + [how-to-build](./how-to-build.md) and [how-to-install](./how-to-install.md) to install Gravitino. 3. Start Gravitino using the gravitino.sh script: @@ -191,7 +191,7 @@ newgrp docker You can install and run all the programs as Docker containers by using the [gravitino-playground](https://github.com/datastrato/gravitino-playground). For the details of -how to run the playground, please see [how-to-use-the-playground](./how-to-use-the-playground) +how to run the playground, please see [how-to-use-the-playground](./how-to-use-the-playground.md) ## Installing Gravitino playground locally diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 62be1668c23..8ee97a166cf 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -64,11 +64,11 @@ It's highly recommend that you change the default value of `gravitino.entity.sto |-------------------------------|--------------------------------------------------------------------------------------------------------------------------------|---------------|---------------| | `gravitino.auxService.names ` | The auxiliary service name of the Gravitino Iceberg REST server, use **`iceberg-rest`** for the Gravitino Iceberg REST server. | (none) | 0.2.0 | -Please refer to [Iceberg REST catalog service](iceberg-rest-service) for Iceberg REST catalog service configurations. +Please refer to [Iceberg REST catalog service](iceberg-rest-service.md) for Iceberg REST catalog service configurations. ### Security configuration -Please refer to [security](security) for HTTPS and Authentication configurations. +Please refer to [security](security.md) for HTTPS and Authentication configurations. ## Gravitino catalog properties configuration @@ -88,12 +88,12 @@ Explicit specifications take precedence over the formal configurations. These rules only apply on the catalog properties, doesn't affect on the schema or table properties. ::: -| catalog provider | catalog properties | catalog properties configuration file path | -|---------------------|--------------------------------------------------------------------------------------|----------------------------------------------------------| -| `hive` | [Hive catalog properties](apache-hive-catalog#catalog-properties) | `catalogs/hive/conf/hive.conf` | -| `lakehouse-iceberg` | [Lakehouse Iceberg catalog properties](lakehouse-iceberg-catalog#catalog-properties) | `catalogs/lakehouse-iceberg/conf/lakehouse-iceberg.conf` | -| `jdbc-mysql` | [MySQL catalog properties](jdbc-mysql-catalog#catalog-properties) | `catalogs/jdbc-mysql/conf/jdbc-mysql.conf` | -| `jdbc-postgresql` | [PostgreSQL catalog properties](jdbc-postgresql-catalog#catalog-properties) | `catalogs/jdbc-postgresql/conf/jdbc-postgresql.conf` | +| catalog provider | catalog properties | catalog properties configuration file path | +|---------------------|-----------------------------------------------------------------------------------------|----------------------------------------------------------| +| `hive` | [Hive catalog properties](apache-hive-catalog.md#catalog-properties) | `catalogs/hive/conf/hive.conf` | +| `lakehouse-iceberg` | [Lakehouse Iceberg catalog properties](lakehouse-iceberg-catalog.md#catalog-properties) | `catalogs/lakehouse-iceberg/conf/lakehouse-iceberg.conf` | +| `jdbc-mysql` | [MySQL catalog properties](jdbc-mysql-catalog.md#catalog-properties) | `catalogs/jdbc-mysql/conf/jdbc-mysql.conf` | +| `jdbc-postgresql` | [PostgreSQL catalog properties](jdbc-postgresql-catalog.md#catalog-properties) | `catalogs/jdbc-postgresql/conf/jdbc-postgresql.conf` | :::info Gravitino server automatically add catalog properties configuration dir to classpath. diff --git a/docs/how-to-install.md b/docs/how-to-install.md index 41fb0580495..6f3c5beec37 100644 --- a/docs/how-to-install.md +++ b/docs/how-to-install.md @@ -16,7 +16,7 @@ configured correctly. To confirm the Java version, you can simply run `${JAVA_HO Before installing Gravitino, make sure you have Gravitino binary distribution package. You can download the latest Gravitino binary distribution Package from [GitHub](https://github.com/datastrato/gravitino/releases), -or you can build it yourself by following the instructions in [How to Build Gravitino](./how-to-build). +or you can build it yourself by following the instructions in [How to Build Gravitino](./how-to-build.md). If you build Gravitino yourself by `./gradlew compileDistribution` command, you can find the Gravitino binary distribution package in `distribution/package` directory. @@ -49,7 +49,7 @@ The Gravitino binary distribution package contains the following files: The Gravitino server configuration file is `conf/gravitino.conf`. You can configure the Gravitino server by modifying this file. Basic configurations are already added to this file, all the -configurations list in [Gravitino Server Configurations](./gravitino-server-config). +configurations list in [Gravitino Server Configurations](./gravitino-server-config.md). #### Configure Gravitino server log @@ -142,4 +142,4 @@ compose file. For the details, you can review the [Gravitino playground repository](https://github.com/datastrato/gravitino-playground) and -[playground example](./how-to-use-the-playground). +[playground example](./how-to-use-the-playground.md). diff --git a/docs/how-to-test.md b/docs/how-to-test.md index c25b83533fb..42238e5c50d 100644 --- a/docs/how-to-test.md +++ b/docs/how-to-test.md @@ -12,7 +12,7 @@ are end-to-end tests that covers the whole system. :::note before test * If you want to run the complete integration test suites, you need to install Docker in your environment. -* Please refer to [How to build Gravitino](./how-to-build) for more details to make sure you have +* Please refer to [How to build Gravitino](./how-to-build.md) for more details to make sure you have a build environment ready. * [OrbStack](https://orbstack.dev/) is highly recommended to replace Docker Desktop for macOS. OrbStack automatically configures the network between Docker containers. diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 4908bdd6e55..73484883c9f 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -16,7 +16,7 @@ The Gravitino Iceberg REST Server follows the [Apache Iceberg REST API specifica - Supports the Apache Iceberg REST API defined in Iceberg 1.3.1, supports all namespace and table interfaces. `Token`, `ReportMetrics`, and `Config` interfaces aren't supported yet. - Works as a catalog proxy, supporting `HiveCatalog` and `JDBCCatalog`. - When writing to HDFS, the Gravitino Iceberg REST catalog service can only operate as the specified HDFS user and - doesn't support proxying to other HDFS users. See [How to access Apache Hadoop](gravitino-server-config) for more details. + doesn't support proxying to other HDFS users. See [How to access Apache Hadoop](gravitino-server-config.md) for more details. :::info Builds with Apache Iceberg `1.3.1`. The Apache Iceberg table format version is `1` by default. @@ -25,7 +25,7 @@ Builds with Hadoop 2.10.x, there may compatibility issue when accessing Hadoop 3 ## How to start the Gravitino Iceberg REST catalog service -Deploy the Gravitino server to the `GRAVITINO_HOME` directory. You can find the configuration options in [`$GRAVITINO_HOME/conf/gravitino.conf`](gravitino-server-config). +Deploy the Gravitino server to the `GRAVITINO_HOME` directory. You can find the configuration options in [`$GRAVITINO_HOME/conf/gravitino.conf`](gravitino-server-config.md). ### Gravitino Iceberg REST catalog service configuration diff --git a/docs/index.md b/docs/index.md index c19b263eac5..88ed5e8fd35 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,37 +11,37 @@ Gravitino is a high-performance, geo-distributed, and federated metadata lake. I metadata directly in different sources, types, and regions. It also provides users with unified metadata access for data and AI assets. -[Learn more](./overview)→ +[Learn more](./overview.md)→ ## Downloading You can get Graviton from the [GitHub release page](https://github.com/datastrato/gravitino/releases), -or you can build Gravitino from source, please see [How to build Gravitino](./how-to-build). +or you can build Gravitino from source, please see [How to build Gravitino](./how-to-build.md). Gravitino runs on both Linux and macOS, and requires Java 8. Gravitino trino-connector runs with Trino, and requires Java 17. This should include JVMs on x86_64 and ARM64. It's easy to run locally on one machine --- all you need is to have `java` installed on your system `PATH`, or the `JAVA_HOME` environment variable pointing to a Java installation. -See [How to install Gravitino](./how-to-install) to learn how to install Gravitino server. +See [How to install Gravitino](./how-to-install.md) to learn how to install Gravitino server. Gravitino provides Docker image on [Docker Hub](https://hub.docker.com/u/datastrato). Please pull the image and run it. For the details of Gravitino Docker image, please see -[Dock image details](./docker-image-details). +[Dock image details](./docker-image-details.md). Gravitino also provides a playground to experience the whole Gravitino system with other components. Please see the [Gravitino playground repository](https://github.com/datastrato/gravitino-playground) -and [How to use the playground](./how-to-use-the-playground). +and [How to use the playground](./how-to-use-the-playground.md). ## Getting started -To get started with Gravitino, please see [Getting started](./getting-started) for the details. +To get started with Gravitino, please see [Getting started](./getting-started.md) for the details. -* [Getting started locally](./getting-started#getting-started-locally): a quick guide to start +* [Getting started locally](./getting-started.md#getting-started-locally): a quick guide to start and use Gravitino locally. -* [Running on Amazon Web Services](./getting-started#getting-started-on-amazon-web-services): a +* [Running on Amazon Web Services](./getting-started.md#getting-started-on-amazon-web-services): a quick guide to start and use Gravitino on AWS. -* [Running on Google Cloud Platform](./getting-started#getting-started-on-google-cloud-platform): +* [Running on Google Cloud Platform](./getting-started.md#getting-started-on-google-cloud-platform): a quick guide to start and use Gravitino on GCP. ## Gravitino playground @@ -49,21 +49,21 @@ To get started with Gravitino, please see [Getting started](./getting-started) f To experience Gravitino with other components simply, Gravitino provides a playground to run. It integrates Apache Hadoop, Apache Hive, Trino, MySQL, PostgreSQL, and Gravitino together as a complete environment. To experience the whole features, please also see -[Getting started](./getting-started) and [How to use the Gravitino playground](./how-to-use-the-playground) +[Getting started](./getting-started.md) and [How to use the Gravitino playground](./how-to-use-the-playground.md) to learn how to use the playground. -* [Install Gravitino playground on AWS or GCP](./getting-started#installing-gravitino-playground-on-aws-or-google-cloud-platform): +* [Install Gravitino playground on AWS or GCP](./getting-started.md#installing-gravitino-playground-on-aws-or-google-cloud-platform): a quick guide to start and use Gravitino playground on AWS or GCP. -* [Install Gravitino playground locally](./getting-started#installing-gravitino-playground-locally): +* [Install Gravitino playground locally](./getting-started.md#installing-gravitino-playground-locally): a quick guide to start and use Gravitino playground locally. -* [How to use the Gravitino playground](./how-to-use-the-playground): provides an example of how +* [How to use the Gravitino playground](./how-to-use-the-playground.md): provides an example of how to use Gravitino and other components together. ## Where to go from here ### Programming guides -* [Manage metadata using Gravitino](./manage-metadata-using-gravitino): provides the complete +* [Manage metadata using Gravitino](./manage-metadata-using-gravitino.md): provides the complete functionalities of Gravitino metadata management. Including metalake, catalog, schema and table management. * [Gravitino Open API](pathname:///docs/0.3.0/api/rest/index.html): provides the complete Open API definition of @@ -74,24 +74,24 @@ to learn how to use the playground. Gravitino provides several ways to configure and manage the Gravitino server. Please see: -* [How to customize Gravitino server configurations](./gravitino-server-config): provides the +* [How to customize Gravitino server configurations](./gravitino-server-config.md): provides the complete Gravitino server configurations. -* [Security](./security): provides the security configurations for Gravitino, including HTTPS +* [Security](./security.md): provides the security configurations for Gravitino, including HTTPS and OAuth2 configurations. -* [Gravitino metrics](./metrics): provides the metrics configurations and detailed metrics list +* [Gravitino metrics](./metrics.md): provides the metrics configurations and detailed metrics list of Gravitino server. ### Catalog details Gravitino supports different catalogs to manage the metadata in different sources. Please see: -* [Lakehouse Iceberg catalog](./lakehouse-iceberg-catalog): a complete guide to use Gravitino +* [Lakehouse Iceberg catalog](./lakehouse-iceberg-catalog.md): a complete guide to use Gravitino manage Apache Iceberg data. -* [How to set up Gravitino Apache Iceberg REST catalog service](./iceberg-rest-service): a +* [How to set up Gravitino Apache Iceberg REST catalog service](./iceberg-rest-service.md): a complete guide to use Gravitino as Apache Iceberg REST catalog service. -* [Apache Hive catalog](./apache-hive-catalog): a complete guide to use Gravitino manage Apache Hive data. -* [JDBC MySQL catalog](./jdbc-mysql-catalog): a complete guide to use Gravitino manage MySQL data. -* [JDBC PostgreSQL catalog](./jdbc-postgresql-catalog): a complete guide to use Gravitino manage PostgreSQL data. +* [Apache Hive catalog](./apache-hive-catalog.md): a complete guide to use Gravitino manage Apache Hive data. +* [JDBC MySQL catalog](./jdbc-mysql-catalog.md): a complete guide to use Gravitino manage MySQL data. +* [JDBC PostgreSQL catalog](./jdbc-postgresql-catalog.md): a complete guide to use Gravitino manage PostgreSQL data. ### Trino connector @@ -103,11 +103,11 @@ way. to use the Trino connector, please see: ### Development guides -* [How to build Gravitino](./how-to-build): a complete guide to build Gravitino from +* [How to build Gravitino](./how-to-build.md): a complete guide to build Gravitino from source. -* [How to test Gravitino](./how-to-test): a complete guide to run Gravitino unit tests and +* [How to test Gravitino](./how-to-test.md): a complete guide to run Gravitino unit tests and integration tests. -* [How to sign and verify a Gravitino releases](./how-to-sign-releases): a guide to sign and verify +* [How to sign and verify a Gravitino releases](./how-to-sign-releases.md): a guide to sign and verify a Gravitino release. -* [Publish Docker images](./publish-docker-images): a guide to publish Gravitino Docker images, +* [Publish Docker images](./publish-docker-images.md): a guide to publish Gravitino Docker images, also list the change logs of Gravitino CI Docker images and release images. diff --git a/docs/jdbc-mysql-catalog.md b/docs/jdbc-mysql-catalog.md index 235ab6a629b..b8745e5cc19 100644 --- a/docs/jdbc-mysql-catalog.md +++ b/docs/jdbc-mysql-catalog.md @@ -44,7 +44,7 @@ You must download the corresponding JDBC driver to the `catalogs/jdbc-mysql/libs ### Catalog operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details. ## Schema @@ -61,7 +61,7 @@ Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravit ### Schema operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations) for more details. ## Table diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md index dfc8a80c9ee..35e52b419f6 100644 --- a/docs/jdbc-postgresql-catalog.md +++ b/docs/jdbc-postgresql-catalog.md @@ -50,7 +50,7 @@ In PostgreSQL, the database corresponds to the Gravitino catalog, and the schema ### Catalog operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details. ## Schema @@ -67,7 +67,7 @@ Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravit ### Schema operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations) for more details. ## Table diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 5e5a2cef70d..1f1ede4c16c 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -193,5 +193,5 @@ If you update a nullability column to non nullability, there may be compatibilit You can place `core-site.xml` and `hdfs-site.xml` in the `catalogs/lakehouse-iceberg/conf` directory to automatically load as the default HDFS configuration. :::caution -When writing to HDFS, the Gravitino Iceberg REST server can only operate as the specified HDFS user and doesn't support proxying to other HDFS users. See [How to access Apache Hadoop](gravitino-server-config) for more details. +When writing to HDFS, the Gravitino Iceberg REST server can only operate as the specified HDFS user and doesn't support proxying to other HDFS users. See [How to access Apache Hadoop](gravitino-server-config.md) for more details. ::: diff --git a/docs/manage-metadata-using-gravitino.md b/docs/manage-metadata-using-gravitino.md index 7bf494b023a..e807e8a0b80 100644 --- a/docs/manage-metadata-using-gravitino.md +++ b/docs/manage-metadata-using-gravitino.md @@ -15,10 +15,10 @@ like metalakes, catalogs, schemas, and tables. This page includes the following In this document, Gravitino uses Apache Hive catalog as an example to show how to manage metadata by Gravitino. Other catalogs are similar to Hive catalog, but they may have some differences, especially in catalog property, table property and column type. For more details, please refer to the related doc. -- [**Apache Hive**](./apache-hive-catalog) -- [**MySQL**](./jdbc-postgresql-catalog) -- [**PostgreSQL**](./jdbc-mysql-catalog) -- [**Apache Iceberg**](./lakehouse-iceberg-catalog) +- [**Apache Hive**](./apache-hive-catalog.md) +- [**MySQL**](./jdbc-postgresql-catalog.md) +- [**PostgreSQL**](./jdbc-mysql-catalog.md) +- [**Apache Iceberg**](./lakehouse-iceberg-catalog.md) Assuming Gravitino has just started, and the host and port is `http://localhost:8090`. @@ -244,12 +244,12 @@ Catalog catalog = gravitinoMetaLake.createCatalog( Currently, Gravitino supports the following catalog providers: -| Catalog provider | Catalog property | -|---------------------|-----------------------------------------------------------------------------| -| `hive` | [Hive catalog property](./apache-hive-catalog#catalog-properties) | -| `lakehouse-iceberg` | [Iceberg catalog property](./lakehouse-iceberg-catalog#catalog-properties) | -| `jdbc-mysql` | [MySQL catalog property](./jdbc-mysql-catalog#catalog-properties) | -| `jdbc-postgresql` | [PostgreSQL catalog property](./jdbc-postgresql-catalog#catalog-properties) | +| Catalog provider | Catalog property | +|---------------------|--------------------------------------------------------------------------------| +| `hive` | [Hive catalog property](./apache-hive-catalog.md#catalog-properties) | +| `lakehouse-iceberg` | [Iceberg catalog property](./lakehouse-iceberg-catalog.md#catalog-properties) | +| `jdbc-mysql` | [MySQL catalog property](./jdbc-mysql-catalog.md#catalog-properties) | +| `jdbc-postgresql` | [PostgreSQL catalog property](./jdbc-postgresql-catalog.md#catalog-properties) | ### Load a catalog @@ -448,12 +448,12 @@ Schema schema = supportsSchemas.createSchema( Currently, Gravitino supports the following schema property: -| Catalog provider | Schema property | -|---------------------|---------------------------------------------------------------------------| -| `hive` | [Hive schema property](./apache-hive-catalog#schema-properties) | -| `lakehouse-iceberg` | [Iceberg scheme property](./lakehouse-iceberg-catalog#schema-properties) | -| `jdbc-mysql` | [MySQL schema property](./jdbc-mysql-catalog#schema-properties) | -| `jdbc-postgresql` | [PostgreSQL schema property](./jdbc-postgresql-catalog#schema-properties) | +| Catalog provider | Schema property | +|---------------------|------------------------------------------------------------------------------| +| `hive` | [Hive schema property](./apache-hive-catalog.md#schema-properties) | +| `lakehouse-iceberg` | [Iceberg scheme property](./lakehouse-iceberg-catalog.md#schema-properties) | +| `jdbc-mysql` | [MySQL schema property](./jdbc-mysql-catalog.md#schema-properties) | +| `jdbc-postgresql` | [PostgreSQL schema property](./jdbc-postgresql-catalog.md#schema-properties) | ### Load a schema @@ -717,12 +717,12 @@ The related java doc is [here](pathname:///docs/0.3.0/api/java/com/datastrato/gr The following is the table property that Gravitino supports: -| Catalog provider | Table property | Type mapping | -|---------------------|-------------------------------------------------------------------------|-------------------------------------------------------------------------| -| `hive` | [Hive table property](./apache-hive-catalog#table-properties) | [Hive type mapping](./apache-hive-catalog#table-column-types) | -| `lakehouse-iceberg` | [Iceberg table property](./lakehouse-iceberg-catalog#table-properties) | [Iceberg type mapping](./lakehouse-iceberg-catalog#table-column-types) | -| `jdbc-mysql` | [MySQL table property](./jdbc-mysql-catalog#table-properties) | [MySQL type mapping](./jdbc-mysql-catalog#table-column-types) | -| `jdbc-postgresql` | [PostgreSQL table property](./jdbc-postgresql-catalog#table-properties) | [PostgreSQL type mapping](./jdbc-postgresql-catalog#table-column-types) | +| Catalog provider | Table property | Type mapping | +|---------------------|----------------------------------------------------------------------------|----------------------------------------------------------------------------| +| `hive` | [Hive table property](./apache-hive-catalog.md#table-properties) | [Hive type mapping](./apache-hive-catalog.md#table-column-types) | +| `lakehouse-iceberg` | [Iceberg table property](./lakehouse-iceberg-catalog.md#table-properties) | [Iceberg type mapping](./lakehouse-iceberg-catalog.md#table-column-types) | +| `jdbc-mysql` | [MySQL table property](./jdbc-mysql-catalog.md#table-properties) | [MySQL type mapping](./jdbc-mysql-catalog.md#table-column-types) | +| `jdbc-postgresql` | [PostgreSQL table property](./jdbc-postgresql-catalog.md#table-properties) | [PostgreSQL type mapping](./jdbc-postgresql-catalog.md#table-column-types) | In addition to the basic settings, Gravitino supports the following features: diff --git a/docs/publish-docker-images.md b/docs/publish-docker-images.md index e12953d39ff..4fc4f8c4027 100644 --- a/docs/publish-docker-images.md +++ b/docs/publish-docker-images.md @@ -33,4 +33,4 @@ You can use GitHub actions to publish Docker images to the Docker Hub repository ## More details of Gravitino Docker images -+ [Gravitino Docker images](docker-image-details) ++ [Gravitino Docker images](docker-image-details.md) diff --git a/docs/security.md b/docs/security.md index bc7d83fca85..9919c9d6fe8 100644 --- a/docs/security.md +++ b/docs/security.md @@ -110,7 +110,7 @@ You can follow the steps to set up an OAuth mode Gravitino server. 5. Copy the public key and remove the character `\n` and you can get the default signing key of Gravitino server. -6. You can refer to the [Configurations](gravitino-server-config) and append the configurations to the conf/gravitino.conf. +6. You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. ```text gravitino.authenticator oauth @@ -220,7 +220,7 @@ bin/keytool -export -alias localhost -keystore localhost.jks -file localhost.cr bin/keytool -import -alias localhost -keystore jre/lib/security/cacerts -file localhost.crt -storepass changeit -noprompt ``` -5. You can refer to the [Configurations](gravitino-server-config) and append the configurations to the conf/gravitino.conf. +5. You can refer to the [Configurations](gravitino-server-config.md) and append the configurations to the conf/gravitino.conf. Configuration doesn't support to resolve environment variable, so you should replace `${JAVA_HOME}` with the actual value. Then, You can start the Gravitino server. diff --git a/docs/trino-connector/catalog-hive.md b/docs/trino-connector/catalog-hive.md index e91b718ac2e..d00997ae983 100644 --- a/docs/trino-connector/catalog-hive.md +++ b/docs/trino-connector/catalog-hive.md @@ -99,7 +99,7 @@ CREATE TABLE "metalake.catalog".dbname.tabname First, you need to create a metalake and catalog in Gravitino. For example, create a new metalake named `test` and create a new catalog named `hive_test` using the `hive` provider. And configure the Metalake `test` into the `Graviton connector`. -For More information about the Hive catalog, please refer to [Hive catalog](../apache-hive-catalog). +For More information about the Hive catalog, please refer to [Hive catalog](../apache-hive-catalog.md). ```bash curl -X POST -H "Content-Type: application/json" \ diff --git a/docs/trino-connector/catalog-iceberg.md b/docs/trino-connector/catalog-iceberg.md index 0c10ae03783..3692a1428d9 100644 --- a/docs/trino-connector/catalog-iceberg.md +++ b/docs/trino-connector/catalog-iceberg.md @@ -51,7 +51,7 @@ Iceberg's tables and schemas do not support properties. First, you need to create a metalake and catalog in Gravitino. For example, create a new metalake named `test` and create a new catalog named `iceberg_test` using the `lakehouse-iceberg` provider. And configure the Metalake `test` into the `Graviton connector`. -For More information about the Iceberg catalog, please refer to [Iceberg catalog](../lakehouse-iceberg-catalog). +For More information about the Iceberg catalog, please refer to [Iceberg catalog](../lakehouse-iceberg-catalog.md). ```bash curl -X POST -H "Content-Type: application/json" \ diff --git a/docs/trino-connector/catalog-mysql.md b/docs/trino-connector/catalog-mysql.md index bbef7ee76af..6aa8568df22 100644 --- a/docs/trino-connector/catalog-mysql.md +++ b/docs/trino-connector/catalog-mysql.md @@ -45,7 +45,7 @@ MySQL's tables and schemas cannot support properties. First, you need to create a metalake and catalog in Gravitino. For example, create a new metalake named `test` and create a new catalog named `mysql_test` using the `jdbc-mysql` provider. And configure the Metalake `test` into the `Graviton connector`. -For More information about the MySQL catalog, please refer to [MySql catalog](../docs/jdbc-mysql-catalog). +For More information about the MySQL catalog, please refer to [MySQL catalog](../jdbc-mysql-catalog.md). ```bash curl -X POST -H "Content-Type: application/json" \ diff --git a/docs/trino-connector/catalog-postgresql.md b/docs/trino-connector/catalog-postgresql.md index 2af7cd8c5cb..e841b1e7962 100644 --- a/docs/trino-connector/catalog-postgresql.md +++ b/docs/trino-connector/catalog-postgresql.md @@ -46,7 +46,7 @@ PostgreSQL's tables and schemas cannot support properties. First, you need to create a metalake and catalog in Gravitino. For example, create a new metalake named `test` and create a new catalog named `postgresql_test` using the `jdbc-postgresql` provider. And configure the Metalake `test` into the `Graviton connector`. -For More information about the PostgreSQL catalog, please refer to [PostgreSQL catalog](../docs/jdbc-postgressql-catalog). +For More information about the PostgreSQL catalog, please refer to [PostgreSQL catalog](../jdbc-postgresql-catalog.md). ```bash curl -X POST -H "Content-Type: application/json" \ diff --git a/docs/trino-connector/index.md b/docs/trino-connector/index.md index 528cdc58c8a..3a1bd676904 100644 --- a/docs/trino-connector/index.md +++ b/docs/trino-connector/index.md @@ -8,13 +8,13 @@ This software is licensed under the Apache License version 2." Gravitino connector index: -- [Trino Support](trino-connector) - - [Requirements](requirements) - - [Installation](installation) - - [Configuration](configuration) - - [Supported catalogs](supported-catalog) - - [Hive](catalog-hive) - - [Iceberg](catalog-iceberg) - - [MySQL](catalog-mysql) - - [PostgreSQL](catalog-postgresql) - - [Supported SQL](sql-support) \ No newline at end of file +- [Trino Support](trino-connector.md) + - [Requirements](requirements.md) + - [Installation](installation.md) + - [Configuration](configuration.md) + - [Supported catalogs](supported-catalog.md) + - [Hive](catalog-hive.md) + - [Iceberg](catalog-iceberg.md) + - [MySQL](catalog-mysql.md) + - [PostgreSQL](catalog-postgresql.md) + - [Supported SQL](sql-support.md) \ No newline at end of file diff --git a/docs/trino-connector/installation.md b/docs/trino-connector/installation.md index 6f724d0238f..1fd897ba281 100644 --- a/docs/trino-connector/installation.md +++ b/docs/trino-connector/installation.md @@ -20,7 +20,7 @@ Please refer to the [Deploying Trino documentation](https://trino.io/docs/curren Alternatively, you can build the Gravitino connector package from the sources and obtain the `gravitino-trino-connector-.tar.gz` file in the `$PROJECT/distribution` directory. -Please refer to the [Gravitino Development documentation](../how-to-build) +Please refer to the [Gravitino Development documentation](../how-to-build.md) ## Example @@ -68,7 +68,7 @@ Now you can see the Gravitino connector directory in the plugin directory. ### Configuring the Gravitino connector Assuming you have now started the Gravitino server on the host `gravitino-server-host` and can access it properly. -And you have created a metalake named `test`. If not, please refer to the [Gravitino Getting-started](../getting-started) +And you have created a metalake named `test`. If not, please refer to the [Gravitino Getting-started](../getting-started.md) Add catalog configuration to the Trino configuration file `/etc/trino/catalog/gravitino.properties`. @@ -116,7 +116,7 @@ system You can see the `gravitino` catalog in the result. This signifies the successful installation of the Gravitino connector. Assuming you have created a catalog named `test.jdbc-mysql` in the Gravitino server. -If you don't have it, please refer to [Create a Catalog](../manage-metadata-using-gravitino#create-a-catalog) +If you don't have it, please refer to [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog) Then you can use the Trino CLI to connect to the Trino container and run a query. diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index be35c39c124..4b3f0e7068b 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -8,16 +8,16 @@ This software is licensed under the Apache License version 2." The catalogs currently supported by the Gravitino connector are as follows: -- [Hive](catalog-hive) -- [Iceberg](catalog-iceberg) -- [MySQL](catalog-mysql) -- [PostgreSQL](catalog-postgresql) +- [Hive](catalog-hive.md) +- [Iceberg](catalog-iceberg.md) +- [MySQL](catalog-mysql.md) +- [PostgreSQL](catalog-postgresql.md) ## Create catalog Trino currently does not support creating Gravitino managed catalogs. If you need to create a catalog, please refer to: -- [Create a Catalog](../manage-metadata-using-gravitino#create-a-catalog) +- [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog) ## Data type mapping diff --git a/docs/trino-connector/trino-connector.md b/docs/trino-connector/trino-connector.md index aef383756cd..efea55cba0e 100644 --- a/docs/trino-connector/trino-connector.md +++ b/docs/trino-connector/trino-connector.md @@ -20,7 +20,7 @@ The loading of Gravitino's catalogs into Trino follows the naming convention: ``` Regarding `metalake` and `catalog`, -you can refer to [Create a Metalake](../manage-metadata-using-gravitino#create-a-metalake), [Create a Catalog](../manage-metadata-using-gravitino#create-a-catalog). +you can refer to [Create a Metalake](../manage-metadata-using-gravitino.md#create-a-metalake), [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog). Usage in queries is as follows: From b33141d265edb45913916b5bd8f4656ad17c237f Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Wed, 13 Dec 2023 19:12:24 +0800 Subject: [PATCH 04/51] [Minor] improvement(docs): Optimized Trino related docs. (#1139) ### What changes were proposed in this pull request? Optimize the docs of Gravitino Trino connector. ### Why are the changes needed? Better user experience. ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --------- Co-authored-by: yuhui --- docs/lakehouse-iceberg-catalog.md | 6 ++-- docs/trino-connector/catalog-hive.md | 14 ++++++--- docs/trino-connector/catalog-iceberg.md | 14 ++++++--- docs/trino-connector/catalog-mysql.md | 14 ++++++--- docs/trino-connector/catalog-postgresql.md | 13 +++++---- docs/trino-connector/configuration.md | 9 +++--- docs/trino-connector/installation.md | 34 +++++++++------------- docs/trino-connector/supported-catalog.md | 5 ++-- docs/trino-connector/trino-connector.md | 8 +++-- 9 files changed, 67 insertions(+), 50 deletions(-) diff --git a/docs/lakehouse-iceberg-catalog.md b/docs/lakehouse-iceberg-catalog.md index 1f1ede4c16c..4ee38c3e1c5 100644 --- a/docs/lakehouse-iceberg-catalog.md +++ b/docs/lakehouse-iceberg-catalog.md @@ -55,7 +55,7 @@ You must download the corresponding JDBC driver to the `catalogs/lakehouse-icebe ### Catalog operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#catalogs-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#catalogs-operations) for more details. ## Schema @@ -69,7 +69,7 @@ You could put properties except `comment`. ### Schema operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#schemas-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#schemas-operations) for more details. ## Table @@ -161,7 +161,7 @@ The Gravitino server doesn't allow passing the following reserved fields. ### Table operations -Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino#tables-operations) for more details. +Please refer to [Manage Metadata Using Gravitino](./manage-metadata-using-gravitino.md#tables-operations) for more details. #### Alter table operations diff --git a/docs/trino-connector/catalog-hive.md b/docs/trino-connector/catalog-hive.md index d00997ae983..d8cb484942f 100644 --- a/docs/trino-connector/catalog-hive.md +++ b/docs/trino-connector/catalog-hive.md @@ -96,10 +96,10 @@ CREATE TABLE "metalake.catalog".dbname.tabname ## Basic usage examples -First, you need to create a metalake and catalog in Gravitino. -For example, create a new metalake named `test` and create a new catalog named `hive_test` using the `hive` provider. -And configure the Metalake `test` into the `Graviton connector`. -For More information about the Hive catalog, please refer to [Hive catalog](../apache-hive-catalog.md). +You need to do the following steps before you can use the Hive catalog in Trino through Gravitino. + +- Create a metalake and catalog in Gravitino. Assuming that the metalake name is `test` and the catalog name is `hive_test`, +then you can use the following code to create them in Gravitino: ```bash curl -X POST -H "Content-Type: application/json" \ @@ -122,6 +122,12 @@ curl -X POST \ }' http://gravitino-host:8090/api/metalakes/test/catalogs ``` +For More information about the Hive catalog, please refer to [Hive catalog](../apache-hive-catalog.md). + +- Set the value of configuration `gravitino.metalake` to the metalake you have created, named 'test', and start the Trino container. + +Use the Trino CLI to connect to the Trino container and run a query. + Listing all Gravitino managed catalogs: ```sql diff --git a/docs/trino-connector/catalog-iceberg.md b/docs/trino-connector/catalog-iceberg.md index 3692a1428d9..6064d34cc24 100644 --- a/docs/trino-connector/catalog-iceberg.md +++ b/docs/trino-connector/catalog-iceberg.md @@ -48,10 +48,10 @@ Iceberg's tables and schemas do not support properties. ## Basic usage examples -First, you need to create a metalake and catalog in Gravitino. -For example, create a new metalake named `test` and create a new catalog named `iceberg_test` using the `lakehouse-iceberg` provider. -And configure the Metalake `test` into the `Graviton connector`. -For More information about the Iceberg catalog, please refer to [Iceberg catalog](../lakehouse-iceberg-catalog.md). +You need to do the following steps before you can use the Iceberg catalog in Trino through Gravitino. + +- Create a metalake and catalog in Gravitino. Assuming that the metalake name is `test` and the catalog name is `iceberg_test`, +then you can use the following code to create them in Gravitino: ```bash curl -X POST -H "Content-Type: application/json" \ @@ -75,6 +75,12 @@ curl -X POST -H "Content-Type: application/json" \ }' http://gravitino-host:8090/api/metalakes/test/catalogs ``` +For More information about the Iceberg catalog, please refer to [Iceberg catalog](../lakehouse-iceberg-catalog.md). + +- Set the value of configuration `gravitino.metalake` to the metalake you have created, named 'test', and start the Trino container. + +Use the Trino CLI to connect to the Trino container and run a query. + Listing all Gravitino managed catalogs: ```sql diff --git a/docs/trino-connector/catalog-mysql.md b/docs/trino-connector/catalog-mysql.md index 6aa8568df22..964539461d7 100644 --- a/docs/trino-connector/catalog-mysql.md +++ b/docs/trino-connector/catalog-mysql.md @@ -42,10 +42,10 @@ MySQL's tables and schemas cannot support properties. ## Basic usage examples -First, you need to create a metalake and catalog in Gravitino. -For example, create a new metalake named `test` and create a new catalog named `mysql_test` using the `jdbc-mysql` provider. -And configure the Metalake `test` into the `Graviton connector`. -For More information about the MySQL catalog, please refer to [MySQL catalog](../jdbc-mysql-catalog.md). +You need to do the following steps before you can use the MySQL catalog in Trino through Gravitino. + +- Create a metalake and catalog in Gravitino. Assuming that the metalake name is `test` and the catalog name is `mysql_test`, +then you can use the following code to create them in Gravitino: ```bash curl -X POST -H "Content-Type: application/json" \ @@ -70,6 +70,12 @@ curl -X POST -H "Content-Type: application/json" \ }' http://gravitino-host:8090/api/metalakes/test/catalogs ``` +For More information about the MySQL catalog, please refer to [MySQL catalog](../jdbc-mysql-catalog.md). + +- Set the value of configuration `gravitino.metalake` to the metalake you have created, named 'test', and start the Trino container. + +Use the Trino CLI to connect to the Trino container and run a query. + Listing all Gravitino managed catalogs: ```sql diff --git a/docs/trino-connector/catalog-postgresql.md b/docs/trino-connector/catalog-postgresql.md index e841b1e7962..3a1e96fb7c7 100644 --- a/docs/trino-connector/catalog-postgresql.md +++ b/docs/trino-connector/catalog-postgresql.md @@ -43,10 +43,9 @@ PostgreSQL's tables and schemas cannot support properties. ## Basic usage examples -First, you need to create a metalake and catalog in Gravitino. -For example, create a new metalake named `test` and create a new catalog named `postgresql_test` using the `jdbc-postgresql` provider. -And configure the Metalake `test` into the `Graviton connector`. -For More information about the PostgreSQL catalog, please refer to [PostgreSQL catalog](../jdbc-postgresql-catalog.md). +You need to do the following steps before you can use the PostgreSQL catalog in Trino through Gravitino. + +- Create a metalake and catalog in Gravitino. Assuming that the metalake name is `test` and the catalog name is `postgresql_test`, then you can use the following code to create them in Gravitino: ```bash curl -X POST -H "Content-Type: application/json" \ @@ -70,8 +69,12 @@ curl -X POST -H "Content-Type: application/json" \ "jdbc-driver": "org.postgresql.Driver" } }' http://gravitino-host:8090/api/metalakes/test/catalogs - ``` +For more information about the PostgreSQL catalog, please refer to [PostgreSQL catalog](../jdbc-postgresql-catalog.md). + +- Set the value of configuration `gravitino.metalake` to the metalake you have created, named 'test', and start the Trino container. + +Use the Trino CLI to connect to the Trino container and run a query. Listing all Gravitino managed catalogs: diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index 8141c9ba533..471723aa8cf 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -6,7 +6,8 @@ license: "Copyright 2023 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2." --- -| Property | Type | Default Value | Description | Required | Since Version | -|--------------------|--------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| -| gravitino.url | string | http://localhost:8090 | The `gravitino.url` defines the connection URL for the Gravitino server. If not set, the default value is `http://localhost:8090`. If the Gravitino server is not starting, Trino can still start normally. Once the Gravitino server is up and running, the Gravitino connector will automatically connect. You can find error details in the Trino logs. | Yes | 0.2.0 | -| gravitino.metalake | string | (none) | The `gravitino.metalake` defines the metalake used. You can create it beforehand or later on. If not set, Trino might throw an error upon startup. It must be set to a valid metalake name. If it's not created, the Gravitino connector continues checking until it's created. Once created, it will load the catalogs, schemas, and tables into Trino and maintain synchronization. | Yes | 0.2.0 | \ No newline at end of file +| Property | Type | Default Value | Description | Required | Since Version | +|--------------------|--------|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| +| connector.name | string | gravitino | The `connector.name` defines the name of Gravitino connector, this value is always 'gravitino'. | Yes | 0.2.0 | +| gravitino.metalake | string | (none) | The `gravitino.metalake` defines the metalake used. You can create it beforehand or later on. If not set, Trino might throw an error upon startup. It must be set to a valid metalake name. If it's not created, the Gravitino connector continues checking until it's created. Once created, it will load the catalogs, schemas, and tables into Trino and maintain synchronization. | Yes | 0.2.0 | +| gravitino.url | string | http://localhost:8090 | The `gravitino.url` defines the connection URL for the Gravitino server. If not set, the default value is `http://localhost:8090`. If the Gravitino server is not starting, Trino can still start normally. Once the Gravitino server is up and running, the Gravitino connector will automatically connect. You can find error details in the Trino logs. | Yes | 0.2.0 | diff --git a/docs/trino-connector/installation.md b/docs/trino-connector/installation.md index 1fd897ba281..02a1d2d65b5 100644 --- a/docs/trino-connector/installation.md +++ b/docs/trino-connector/installation.md @@ -6,13 +6,12 @@ license: "Copyright 2023 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2." --- -To install the Gravitino connector, first deploy the Trino environment, and then install the Gravitino connector plugin into Trino. -Please refer to the [Deploying Trino documentation](https://trino.io/docs/current/installation/deployment.html). +To install the Gravitino connector, you should first deploy the Trino environment, and then install the Gravitino connector plugin into Trino. +Please refer to the [Deploying Trino documentation](https://trino.io/docs/current/installation/deployment.html) and do the following steps: -1. Download the Gravitino connector tarball and unpack it. +1. [Download](https://github.com/datastrato/gravitino/releases) the Gravitino connector tarball and unpack it. The tarball contains a single top-level directory `gravitino-trino-connector-`, which called the connector directory. - [Download the gravitino-connector](https://github.com/datastrato/gravitino/releases). 2. Copy the connector directory to the Trino's plugin directory. Normally, the directory location is `Trino-server-/plugin`, and the directory contains other catalogs used by Trino. 3. Add Trino JVM arguments `-Dlog4j.configurationFile=file:////etc/trino/log4j2.properties` to enable logging for the Gravitino connector. @@ -67,10 +66,9 @@ Now you can see the Gravitino connector directory in the plugin directory. ### Configuring the Gravitino connector -Assuming you have now started the Gravitino server on the host `gravitino-server-host` and can access it properly. -And you have created a metalake named `test`. If not, please refer to the [Gravitino Getting-started](../getting-started.md) +Assuming you have now started the Gravitino server on the host `gravitino-server-host` and already created a metalake named `test`, if those have not been prepared, please refer to the [Gravitino Getting-started](../getting-started.md). -Add catalog configuration to the Trino configuration file `/etc/trino/catalog/gravitino.properties`. +To configure Gravitino connector correctly, you need to put the following configurations to the Trino configuration file `/etc/trino/catalog/gravitino.properties`. ```text connector.name=gravitino @@ -78,19 +76,19 @@ gravitino.url=http://gravitino-server-host:8090 gravitino.metalake=test ``` -The `gravitino.name` defines which Gravitino connector is used. It must be `gravitino`. -The `gravitino.metalake` defines which metalake are used. It should exist in the Gravitino server. -The `gravitino.uri` defines the connection information about Gravitino server. Make sure your container can access the Gravitino server. +- The `gravitino.name` defines which Gravitino connector is used. It must be `gravitino`. +- The `gravitino.metalake` defines which metalake are used. It should exist in the Gravitino server. +- The `gravitino.uri` defines the connection information about Gravitino server. Make sure your container can access the Gravitino server. -If you don't have the `test` metalake. You can create a new metalake named `test`. +Full configurations for Gravitino connector can be seen [here](configuration.md) -Create a new metalake named `test` by the following command. +If you haven't created the metalake named `test`, you can use the following command to create it. ```shell curl -X POST -H "Content-Type: application/json" -d '{"name":"test","comment":"comment","properties":{}}' http://gravitino-server-host:8090/api/metalakes ``` -Restart the Trino container to load the Gravitino connector. +And then restart the Trino container to load the Gravitino connector. ```shell docker restart trino-gravitino @@ -113,12 +111,9 @@ tpch system ``` -You can see the `gravitino` catalog in the result. This signifies the successful installation of the Gravitino connector. +You can see the `gravitino` catalog in the result set. This signifies the successful installation of the Gravitino connector. -Assuming you have created a catalog named `test.jdbc-mysql` in the Gravitino server. -If you don't have it, please refer to [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog) - -Then you can use the Trino CLI to connect to the Trino container and run a query. +Assuming you have created a catalog named `test.jdbc-mysql` in the Gravitino server, or please refer to [Create a Catalog](../manage-metadata-using-gravitino#create-a-catalog). Then you can use the Trino CLI to connect to the Trino container and run a query like this. ```text docker exec -it trino trino @@ -134,5 +129,4 @@ system test.jdbc-mysql ``` -The catalog named 'test.jdbc-mysql' is your created catalog by gravitino server. -You can use it to access the mysql database like other Trino catalogs. \ No newline at end of file +The catalog named 'test.jdbc-mysql' is the catalog that you created by gravitino server, and you can use it to access the mysql database like other Trino catalogs. \ No newline at end of file diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index 4b3f0e7068b..b2602f809e0 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -19,10 +19,9 @@ Trino currently does not support creating Gravitino managed catalogs. If you need to create a catalog, please refer to: - [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog) -## Data type mapping +## Data type mapping between Trino and Gravitino -All types of catalogs support the following data type conversions. -Data Type Conversion between Trino and Gravitino is as following: +All types of catalogs support the following data type conversions between Trino and Gravitino: | Gravitino Type | Trino Type | |----------------|------------| diff --git a/docs/trino-connector/trino-connector.md b/docs/trino-connector/trino-connector.md index efea55cba0e..6a5fbca9262 100644 --- a/docs/trino-connector/trino-connector.md +++ b/docs/trino-connector/trino-connector.md @@ -7,11 +7,13 @@ This software is licensed under the Apache License version 2." --- Trino can manage and access data using the Trino connector provided by `Gravitino`, commonly referred to as the `Gravitino connector`. -After configuring the Gravitino connector in Trino, it can automatically load catalog metadata from Gravitino, allowing users to directly access these catalogs in Trino. +After configuring the Gravitino connector in Trino, Trino can automatically load catalog metadata from Gravitino, allowing users to directly access these catalogs in Trino. Once integrated with Gravitino, Trino can operate on all Gravitino data without requiring additional configuration. -When metadata such as catalogs, schemas, or tables change in Gravitino, Trino can also update itself through Gravitino. - +:::node +Once metadata such as catalogs, schemas, or tables are changed in Gravitino, Trino can update itself through Gravitino, this process usually takes +about 3~10 seconds. +::: The loading of Gravitino's catalogs into Trino follows the naming convention: From a06931259f2e5ff8b85eefe3f3a2075b40f3ba28 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Wed, 13 Dec 2023 20:44:14 +0800 Subject: [PATCH 05/51] [MINOR] fix(docs): fix ./gradlew spotlessApply of Coding standards (#1146) ### What changes were proposed in this pull request? Update `./gradlew spotlessApply` of Coding standards in `Contributing.md`. ### Why are the changes needed? There is no `grawdlew` file in project, which cause that developer could not follow coding standards. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fea7490f2a1..1f67b50ee40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,7 +97,7 @@ When adding new code or fixing a bug be sure to add unit tests to provide covera Spotless checks code formatting. If your code isn't correctly formatted, the build fails. To correctly format your code please use Spotless. ```bash -./grawdlew spotlessApply +./gradlew spotlessApply ``` All files must have a license header and the build fails if any files are missing license headers. If you are adding third-party code be sure to understand how to add the third-party license to Gravitino LICENSE and NOTICE files. From 927a449495b513ef95907b90c80a110ba935759f Mon Sep 17 00:00:00 2001 From: mchades Date: Wed, 13 Dec 2023 21:09:36 +0800 Subject: [PATCH 06/51] [MINOR] fix(docs): Fix link of REST API page (#1147) ### What changes were proposed in this pull request? Fix link of REST API page ### Why are the changes needed? link of REST API page will be changed ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? no need --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 88ed5e8fd35..3deb69c86cd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -66,7 +66,7 @@ to learn how to use the playground. * [Manage metadata using Gravitino](./manage-metadata-using-gravitino.md): provides the complete functionalities of Gravitino metadata management. Including metalake, catalog, schema and table management. -* [Gravitino Open API](pathname:///docs/0.3.0/api/rest/index.html): provides the complete Open API definition of +* [Gravitino Open API](./api/rest/gravitino-rest-api): provides the complete Open API definition of Gravitino. * [Gravitino Javadoc](pathname:///docs/0.3.0/api/java/index.html): provides the Javadoc for Gravitino API. From c304f8f3c691ca944f6930dc6799507cd75b3dc5 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Thu, 14 Dec 2023 09:39:32 +0800 Subject: [PATCH 07/51] [#1052] fix(common): position name should support upper-case when add a column (#1145) ### What changes were proposed in this pull request? Position name supports upper-case when add a column for `FIRST` and `DEFAULT`. ### Why are the changes needed? Position name should support upper-case when add a column. For example, the correct value of position is first NOT FIRST currently. ``` curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" -d '{ "updates": [ { "@type": "removeProperty", "property": "key2" }, { "@type": "setProperty", "property": "key3", "value": "value3" }, { "@type": "addColumn", "fieldName": [ "position" ], "type": "varchar(20)", "comment": "Position of user", "position": "FIRST" } ] }' http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/schema/tables/table ``` ``` Unknown json column position: "FIRST" (through reference chain: com.datastrato.gravitino.dto.requests.TableUpdatesRequest["updates"]->java.util.ArrayList[2])% ``` Fix: #1052 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? `TestTableUpdatesRequest#testAddTableColumnRequest` --- .../com/datastrato/gravitino/json/JsonUtils.java | 8 ++++++-- .../dto/requests/TestTableUpdatesRequest.java | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java index 38186c452de..ed9ec3ea1a3 100644 --- a/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java +++ b/common/src/main/java/com/datastrato/gravitino/json/JsonUtils.java @@ -704,9 +704,13 @@ public TableChange.ColumnPosition deserialize(JsonParser p, DeserializationConte node != null && !node.isNull(), "Cannot parse column position from invalid JSON: %s", node); - if (node.isTextual() && node.asText().equals(POSITION_FIRST)) { + if (node.isTextual() + && (node.asText().equals(POSITION_FIRST) + || node.asText().equals(POSITION_FIRST.toUpperCase()))) { return TableChange.ColumnPosition.first(); - } else if (node.isTextual() && node.asText().equals(POSITION_DEFAULT)) { + } else if (node.isTextual() + && (node.asText().equalsIgnoreCase(POSITION_DEFAULT) + || node.asText().equalsIgnoreCase(POSITION_DEFAULT.toUpperCase()))) { return TableChange.ColumnPosition.defaultPos(); } else if (node.isObject()) { String afterColumn = getString(POSITION_AFTER, node); diff --git a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java index 78a24cabc66..6800012437f 100644 --- a/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java +++ b/common/src/test/java/com/datastrato/gravitino/dto/requests/TestTableUpdatesRequest.java @@ -151,6 +151,13 @@ public void testAddTableColumnRequest() throws JsonProcessingException { + "}"; Assertions.assertEquals( JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(jsonString)); + Assertions.assertTrue( + JsonUtils.objectMapper() + .readValue( + jsonString.replace("first", "FIRST"), + TableUpdateRequest.AddTableColumnRequest.class) + .getPosition() + instanceof TableChange.First); // test default position addTableColumnRequest = @@ -170,5 +177,12 @@ public void testAddTableColumnRequest() throws JsonProcessingException { + "}"; Assertions.assertEquals( JsonUtils.objectMapper().readTree(expected), JsonUtils.objectMapper().readTree(jsonString)); + Assertions.assertTrue( + JsonUtils.objectMapper() + .readValue( + jsonString.replace("default", "DEFAULT"), + TableUpdateRequest.AddTableColumnRequest.class) + .getPosition() + instanceof TableChange.Default); } } From df87087466fb8d6eb2aea98a75634e42064f3662 Mon Sep 17 00:00:00 2001 From: FANNG Date: Thu, 14 Dec 2023 16:47:59 +0800 Subject: [PATCH 08/51] [#874] refactor(lakehouse-iceberg) improve IcebergRESTServiceIT (#1143) ### What changes were proposed in this pull request? 1. make getCatalogConfig abstract, Hive&Jdbc&Memory implement these interfaces. 2. remove `gravitino-docker-it` tag from IcebergRESTServiceIT. ### Why are the changes needed? After #711, It takes too much time to test IcebergRESTServiceIT, there's no need to start the docker container for the memory catalog Fix: #874 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing UT --- .../iceberg/IcebergRESTHiveCatalogIT.java | 48 ++++++ .../iceberg/IcebergRESTJdbcCatalogIT.java | 66 ++++++++ .../iceberg/IcebergRESTServiceBaseIT.java | 143 ++---------------- .../iceberg/IcebergRESTServiceIT.java | 28 +++- 4 files changed, 153 insertions(+), 132 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTHiveCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTHiveCatalogIT.java index 70a4267aaf6..dba80ebaa66 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTHiveCatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTHiveCatalogIT.java @@ -4,7 +4,15 @@ */ package com.datastrato.gravitino.integration.test.catalog.lakehouse.iceberg; +import com.datastrato.gravitino.aux.AuxiliaryServiceManager; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.HiveContainer; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import java.util.HashMap; +import java.util.Map; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -15,7 +23,47 @@ @Tag("gravitino-docker-it") @TestInstance(Lifecycle.PER_CLASS) public class IcebergRESTHiveCatalogIT extends IcebergRESTServiceIT { + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + public IcebergRESTHiveCatalogIT() { catalogType = IcebergCatalogBackend.HIVE; } + + @Override + void initEnv() { + containerSuite.startHiveContainer(); + } + + @Override + Map getCatalogConfig() { + Map customConfigs = new HashMap<>(); + customConfigs.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_BACKEND.getKey(), + IcebergCatalogBackend.HIVE.toString().toLowerCase()); + + customConfigs.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_URI.getKey(), + String.format( + "thrift://%s:%d", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT)); + + customConfigs.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_WAREHOUSE.getKey(), + GravitinoITUtils.genRandomName( + String.format( + "hdfs://%s:%d/user/hive/warehouse-hive", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT))); + return customConfigs; + } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java index b70b9250b5a..36088027d5a 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java @@ -5,7 +5,15 @@ package com.datastrato.gravitino.integration.test.catalog.lakehouse.iceberg; +import com.datastrato.gravitino.aux.AuxiliaryServiceManager; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService; +import com.datastrato.gravitino.integration.test.container.ContainerSuite; +import com.datastrato.gravitino.integration.test.container.HiveContainer; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import java.util.HashMap; +import java.util.Map; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; @@ -13,7 +21,65 @@ @Tag("gravitino-docker-it") @TestInstance(Lifecycle.PER_CLASS) public class IcebergRESTJdbcCatalogIT extends IcebergRESTServiceIT { + private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + public IcebergRESTJdbcCatalogIT() { catalogType = IcebergCatalogBackend.JDBC; } + + @Override + void initEnv() { + containerSuite.startHiveContainer(); + } + + public Map getCatalogConfig() { + Map configMap = new HashMap<>(); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_BACKEND.getKey(), + IcebergCatalogBackend.JDBC.toString().toLowerCase()); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_URI.getKey(), + "jdbc:sqlite::memory:"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_USER.getKey(), + "iceberg"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_PASSWORD.getKey(), + "iceberg"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_INIT_TABLES.getKey(), + "true"); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_WAREHOUSE.getKey(), + GravitinoITUtils.genRandomName( + String.format( + "hdfs://%s:%d/user/hive/warehouse-jdbc-sqlite", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT))); + return configMap; + } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java index efa5b5e0abe..7fb56fa8276 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceBaseIT.java @@ -10,16 +10,12 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService; -import com.datastrato.gravitino.integration.test.container.ContainerSuite; -import com.datastrato.gravitino.integration.test.container.HiveContainer; import com.datastrato.gravitino.integration.test.util.AbstractIT; -import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.server.web.JettyServerConfig; import com.datastrato.gravitino.utils.MapUtils; import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,8 +27,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,31 +34,33 @@ *

Referred from spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java */ -@TestInstance(Lifecycle.PER_CLASS) -public class IcebergRESTServiceBaseIT extends AbstractIT { +public abstract class IcebergRESTServiceBaseIT extends AbstractIT { public static final Logger LOG = LoggerFactory.getLogger(IcebergRESTServiceBaseIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); private SparkSession sparkSession; protected IcebergCatalogBackend catalogType = IcebergCatalogBackend.MEMORY; @BeforeAll void initIcebergTestEnv() throws Exception { - containerSuite.startHiveContainer(); + // Start Gravitino docker container + initEnv(); + // Inject Iceberg REST service config to gravitino.conf registerIcebergCatalogConfig(); + // Start Gravitino server AbstractIT.startIntegrationTest(); + // Start Spark session initSparkEnv(); - LOG.info("gravitino and spark env started,{}", catalogType); + LOG.info("Gravitino and Spark env started,{}", catalogType); } @AfterAll void stopIcebergTestEnv() throws Exception { stopSparkEnv(); AbstractIT.stopIntegrationTest(); - LOG.info("gravitino and spark env stopped,{}", catalogType); + LOG.info("Gravitino and Spark env stopped,{}", catalogType); } // AbstractIT#startIntegrationTest() is static, so we couldn't inject catalog info - // if startIntegrationTest() is auto invoked by Junit. so here we override + // if startIntegrationTest() is auto invoked by Junit. So here we override // startIntegrationTest() to disable the auto invoke by junit. @BeforeAll public static void startIntegrationTest() {} @@ -76,129 +72,16 @@ boolean catalogTypeNotMemory() { return !catalogType.equals(IcebergCatalogBackend.MEMORY); } - private void registerIcebergCatalogConfig() { - Map icebergConfigs; + abstract void initEnv(); - switch (catalogType) { - case HIVE: - icebergConfigs = getIcebergHiveCatalogConfigs(); - break; - case JDBC: - icebergConfigs = getIcebergJdbcCatalogConfigs(); - break; - case MEMORY: - icebergConfigs = getIcebergMemoryCatalogConfigs(); - break; - default: - throw new RuntimeException("Not support Iceberg catalog type:" + catalogType); - } + abstract Map getCatalogConfig(); + private void registerIcebergCatalogConfig() { + Map icebergConfigs = getCatalogConfig(); AbstractIT.registerCustomConfigs(icebergConfigs); LOG.info("Iceberg REST service config registered," + StringUtils.join(icebergConfigs)); } - private static Map getIcebergMemoryCatalogConfigs() { - Map configMap = new HashMap<>(); - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_BACKEND.getKey(), - IcebergCatalogBackend.MEMORY.toString().toLowerCase()); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_WAREHOUSE.getKey(), - "/tmp/"); - return configMap; - } - - private static Map getIcebergJdbcCatalogConfigs() { - Map configMap = new HashMap<>(); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_BACKEND.getKey(), - IcebergCatalogBackend.JDBC.toString().toLowerCase()); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_URI.getKey(), - "jdbc:sqlite::memory:"); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.JDBC_USER.getKey(), - "iceberg"); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.JDBC_PASSWORD.getKey(), - "iceberg"); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.JDBC_INIT_TABLES.getKey(), - "true"); - - configMap.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_WAREHOUSE.getKey(), - GravitinoITUtils.genRandomName( - String.format( - "hdfs://%s:%d/user/hive/warehouse-jdbc-sqlite", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT))); - - return configMap; - } - - private static Map getIcebergHiveCatalogConfigs() { - Map customConfigs = new HashMap<>(); - customConfigs.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_BACKEND.getKey(), - IcebergCatalogBackend.HIVE.toString().toLowerCase()); - - customConfigs.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_URI.getKey(), - String.format( - "thrift://%s:%d", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HIVE_METASTORE_PORT)); - - customConfigs.put( - AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX - + IcebergRESTService.SERVICE_NAME - + "." - + IcebergConfig.CATALOG_WAREHOUSE.getKey(), - GravitinoITUtils.genRandomName( - String.format( - "hdfs://%s:%d/user/hive/warehouse-hive", - containerSuite.getHiveContainer().getContainerIpAddress(), - HiveContainer.HDFS_DEFAULTFS_PORT))); - return customConfigs; - } - private static IcebergConfig buildIcebergConfig(Config config) { Map m = config.getConfigsWithPrefix(AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceIT.java index f57ff8c822e..aa749042fee 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTServiceIT.java @@ -5,10 +5,14 @@ package com.datastrato.gravitino.integration.test.catalog.lakehouse.iceberg; +import com.datastrato.gravitino.aux.AuxiliaryServiceManager; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergRESTService; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -23,14 +27,12 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.junit.jupiter.api.condition.EnabledIf; @TestInstance(Lifecycle.PER_CLASS) -@Tag("gravitino-docker-it") public class IcebergRESTServiceIT extends IcebergRESTServiceBaseIT { private static final String ICEBERG_REST_NS_PREFIX = "iceberg_rest_"; @@ -48,6 +50,28 @@ void cleanup() { purgeAllIcebergTestNamespaces(); } + @Override + void initEnv() {} + + @Override + Map getCatalogConfig() { + Map configMap = new HashMap<>(); + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_BACKEND.getKey(), + IcebergCatalogBackend.MEMORY.toString().toLowerCase()); + + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.CATALOG_WAREHOUSE.getKey(), + "/tmp/"); + return configMap; + } + private void purgeTable(String namespace, String table) { sql(String.format("DROP TABLE %s.%s PURGE", namespace, table)); } From 4a3eca5c27dcdb7d8f3abe12fce2432316d612a5 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Thu, 14 Dec 2023 16:52:25 +0800 Subject: [PATCH 09/51] [Minor] improvement(docs): Optimize Trino related docs and fix some minor mistakes in docs (#1152) ### What changes were proposed in this pull request? Refine some descriptions and correct mistakes in Trino-related docs. ### Why are the changes needed? Make users more aware of the documents. ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- docs/trino-connector/configuration.md | 10 +++++----- docs/trino-connector/supported-catalog.md | 14 ++++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index 471723aa8cf..711985f7a67 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -6,8 +6,8 @@ license: "Copyright 2023 Datastrato Pvt Ltd. This software is licensed under the Apache License version 2." --- -| Property | Type | Default Value | Description | Required | Since Version | -|--------------------|--------|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| -| connector.name | string | gravitino | The `connector.name` defines the name of Gravitino connector, this value is always 'gravitino'. | Yes | 0.2.0 | -| gravitino.metalake | string | (none) | The `gravitino.metalake` defines the metalake used. You can create it beforehand or later on. If not set, Trino might throw an error upon startup. It must be set to a valid metalake name. If it's not created, the Gravitino connector continues checking until it's created. Once created, it will load the catalogs, schemas, and tables into Trino and maintain synchronization. | Yes | 0.2.0 | -| gravitino.url | string | http://localhost:8090 | The `gravitino.url` defines the connection URL for the Gravitino server. If not set, the default value is `http://localhost:8090`. If the Gravitino server is not starting, Trino can still start normally. Once the Gravitino server is up and running, the Gravitino connector will automatically connect. You can find error details in the Trino logs. | Yes | 0.2.0 | +| Property | Type | Default Value | Description | Required | Since Version | +|--------------------|--------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|---------------| +| connector.name | string | (none) | The `connector.name` defines the name of Trino connector, this value is always 'gravitino'. | Yes | 0.2.0 | +| gravitino.metalake | string | (none) | The `gravitino.metalake` defines which metalake in Gravitino server the Trino connector uses. Trino connector should set it at start, the value of `gravitino.metalake` needs to be a valid name, Trino connector can detect and load the metalake with catalogs, schemas and tables once created and keep in sync. | Yes | 0.2.0 | +| gravitino.url | string | http://localhost:8090 | The `gravitino.url` defines the connection URL of the Gravitino server, the default value is `http://localhost:8090`. Trino connector can detect and connect to Gravitino server once it is ready, no need to start Gravitino server beforehand. | Yes | 0.2.0 | diff --git a/docs/trino-connector/supported-catalog.md b/docs/trino-connector/supported-catalog.md index b2602f809e0..4019ae0cbe7 100644 --- a/docs/trino-connector/supported-catalog.md +++ b/docs/trino-connector/supported-catalog.md @@ -15,13 +15,12 @@ The catalogs currently supported by the Gravitino connector are as follows: ## Create catalog -Trino currently does not support creating Gravitino managed catalogs. -If you need to create a catalog, please refer to: -- [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog) +Trino currently does not support creating Gravitino managed catalogs, if you need to create a catalog, please refer to: [Create a Catalog](../manage-metadata-using-gravitino.md#create-a-catalog). ## Data type mapping between Trino and Gravitino -All types of catalogs support the following data type conversions between Trino and Gravitino: +Gravitino connector supports the following data type conversions between Trino and Gravitino currently. Depending on the detailed catalog, Gravitino may not support some data types conversion for this specific catalog, for example, +Hive does not support `TIME` data type. | Gravitino Type | Trino Type | |----------------|------------| @@ -36,8 +35,11 @@ All types of catalogs support the following data type conversions between Trino | StringType | VARCHAR | | VarcharType | VARCHAR | | BinaryType | VARBINARY | -| TimestampType | TIMESTAMP | | DateType | DATE | +| TimeType | TIME | +| TimestampType | TIMESTAMP | | ArrayType | ARRAY | | MapType | MAP | -| StructType | ROW | \ No newline at end of file +| StructType | ROW | + +For more about Trino data types, please refer to [Trino data types](https://trino.io/docs/current/language/types.html) and Gravitino data types, please refer to [Gravitino data types](../manage-metadata-using-gravitino.md#gravitino-table-column-type). \ No newline at end of file From f10142e880d7aa23765bd5a0abbf02d3189549d5 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 18 Dec 2023 10:44:39 +0800 Subject: [PATCH 10/51] [#1166] fix(docs): Fix typos `graviton` and syntax problems in docs (#1170) ### What changes were proposed in this pull request? - Fix typo `Graviton` in `index.md` - Fix link mistakes in `index.md` and other sentence problems ### Why are the changes needed? To Improve the professionalism and rigor of documents Fixed #1166 ### Does this PR introduce _any_ user-facing change? N/A. ### How was this patch tested? N/A. --- docs/how-to-test.md | 10 +++++----- docs/iceberg-rest-service.md | 2 +- docs/index.md | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/how-to-test.md b/docs/how-to-test.md index 42238e5c50d..8d08c554539 100644 --- a/docs/how-to-test.md +++ b/docs/how-to-test.md @@ -7,7 +7,7 @@ This software is licensed under the Apache License version 2." Gravitino has two types of tests: unit tests and integration tests. Unit tests are mainly focused on the functionalities of the specific class, module, or component. Integration tests -are end-to-end tests that covers the whole system. +are end-to-end tests that cover the whole system. :::note before test * If you want to run the complete integration test suites, you need to install Docker in your @@ -31,15 +31,15 @@ To run the unit test, you can simply run the following command: ./gradlew test -PskipITs ``` -This command runs all the unit tests and skip the integration tests. +This command runs all the unit tests and skips the integration tests. ## Run the integration tests -Gravitino has two modes to run the integration tests, the `embedded` and `deploy` modes. +Gravitino has two modes to run the integration tests, the default `embedded` mode and `deploy` mode. * With the `embedded` mode, the integration test starts an embedded `MiniGravitino` server within the same process of the integration test to run the integration tests. -* With the `deploy` mode, the user has to deploy a Gravitino binary package beforehand, the +* With the `deploy` mode, the user has to build (`./gradlew compileDistribution`) a Gravitino binary package beforehand, the integration test launches and connects to the local Gravitino server to run the integration tests. @@ -153,7 +153,7 @@ server code, follow these steps: If a test fails, you can retrieve valuable information from the logs and test report. Test reports are in the `./build/reports` directory. The integration test logs are in the `./integrate-test/build` directory. In deploy mode, Gravitino server logs are in the `./distribution/package/logs/` directory. In the event of a test failure within the GitHub workflow, the system generates archived logs and test reports. To obtain the archive, follow these steps: -1. Click the `detail` link associated with the failed integrate test in the pull request. This redirects you to the job page. +1. Click the `detail` link associated with the failed integration test in the pull request. This redirects you to the job page. ![pr page Image](assets/test-fail-pr.png) diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 73484883c9f..86fd757de73 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -61,7 +61,7 @@ The Gravitino Iceberg REST catalog service using memory catalog for default. You | `gravitino.auxService.iceberg-rest.uri` | The Hive metadata address, such as `thrift://127.0.0.1:9083`. | (none) | Yes | 0.2.0 | | `gravitino.auxService.iceberg-rest.warehouse ` | The warehouse directory of the Hive catalog, such as `/user/hive/warehouse-hive/`. | (none) | Yes | 0.2.0 | -#### JDBC catalog configuration +#### Iceberg JDBC backend configuration | Configuration item | Description | Default value | Required | Since Version | |-----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------| diff --git a/docs/index.md b/docs/index.md index 3deb69c86cd..52a8178ff77 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,8 +15,8 @@ metadata access for data and AI assets. ## Downloading -You can get Graviton from the [GitHub release page](https://github.com/datastrato/gravitino/releases), -or you can build Gravitino from source, please see [How to build Gravitino](./how-to-build.md). +You can get Gravitino from the [GitHub release page](https://github.com/datastrato/gravitino/releases), +or you can build Gravitino from source code, please see [How to build Gravitino](./how-to-build.md). Gravitino runs on both Linux and macOS, and requires Java 8. Gravitino trino-connector runs with Trino, and requires Java 17. This should include JVMs on x86_64 and @@ -46,7 +46,7 @@ To get started with Gravitino, please see [Getting started](./getting-started.md ## Gravitino playground -To experience Gravitino with other components simply, Gravitino provides a playground to run. It +To experience Gravitino with other components easily, Gravitino provides a playground to run. It integrates Apache Hadoop, Apache Hive, Trino, MySQL, PostgreSQL, and Gravitino together as a complete environment. To experience the whole features, please also see [Getting started](./getting-started.md) and [How to use the Gravitino playground](./how-to-use-the-playground.md) @@ -98,7 +98,7 @@ Gravitino supports different catalogs to manage the metadata in different source Gravitino provides a Trino connector to connect to Gravitino to manage the metadata in a unified way. to use the Trino connector, please see: -* [How to use Gravitino Trino connector](./trino-connector/index): a complete guide to use Gravitino +* [How to use Gravitino Trino connector](./trino-connector/index.md): a complete guide to use Gravitino Trino connector. ### Development guides From f3e1baed3a8739edbd4b048dbb28c7010d3b2bd0 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Mon, 18 Dec 2023 13:52:20 +0800 Subject: [PATCH 11/51] [#877] improvement(web): optimize clean task of build.gradle.kts for web to delete node_modules folder (#1173) ### What changes were proposed in this pull request? Optimize clean task of `build.gradle.kts` for web to delete `node_modules` folder which reduces space after gradle clean. ### Why are the changes needed? The `node_modules` folder contains libraries downloaded from npm that takes up a lot of space. Fix: #877 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Test with gradle clean. --- web/build.gradle.kts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/build.gradle.kts b/web/build.gradle.kts index b29f093893d..3dbac5c32f6 100644 --- a/web/build.gradle.kts +++ b/web/build.gradle.kts @@ -52,7 +52,10 @@ tasks { } clean { + delete(".node") delete("build") delete("dist") + delete("node_modules") + delete("yarn-error.log") } } From 20558b6e3b270e0df88eaaeb2f4e4a2ada3b39d4 Mon Sep 17 00:00:00 2001 From: Yuhui Date: Mon, 18 Dec 2023 17:13:18 +0800 Subject: [PATCH 12/51] [#1154] fix(trino-connector): Fix the issue with joins causing errors in PostgreSQL. (#1177) ### What changes were proposed in this pull request? Fix the issue with joins causing errors in PostgreSQL. Error message is "Cannot cast com.datastrato.gravitino.trino.connector.GravitinoColumnHandle to io.trino.plugin.jdbc.JdbcColumnHandle" ### Why are the changes needed? Fix: #1154 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? UT --- .../jdbc-postgresql/00003_join_pushdown.sql | 61 +++++++++++++++++++ .../jdbc-postgresql/00003_join_pushdown.txt | 28 +++++++++ .../connector/GravitinoSplitManager.java | 3 +- 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.sql create mode 100644 integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.txt diff --git a/integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.sql b/integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.sql new file mode 100644 index 00000000000..46395154373 --- /dev/null +++ b/integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.sql @@ -0,0 +1,61 @@ +CREATE SCHEMA "test.jdbc-postgresql".gt_db1; + +use "test.jdbc-postgresql".gt_db1; + +CREATE TABLE "test.jdbc-postgresql".gt_db1.employee_performance ( + employee_id integer, + evaluation_date date, + rating integer +) +COMMENT 'comment'; + +CREATE TABLE "test.jdbc-postgresql".gt_db1.employees ( + employee_id integer, + department_id integer, + job_title varchar(100), + given_name varchar(100), + family_name varchar(100), + birth_date date, + hire_date date +) +COMMENT 'comment'; + +INSERT INTO "test.jdbc-postgresql".gt_db1.employee_performance (employee_id, evaluation_date, rating) VALUES +(1, DATE '2018-02-24', 4), +(1, DATE '2016-12-25', 7), +(1, DATE '2023-04-07', 4), +(3, DATE '2012-11-08', 7), +(3, DATE '2019-09-15', 2), +(3, DATE '2017-06-21', 8), +(3, DATE '2019-07-16', 4), +(3, DATE '2015-10-06', 4), +(3, DATE '2021-01-05', 6), +(3, DATE '2014-10-24', 4); + +INSERT INTO "test.jdbc-postgresql".gt_db1.employees (employee_id, department_id, job_title, given_name, family_name, birth_date, hire_date) VALUES +(1, 1, 'Manager', 'Gregory', 'Smith', DATE '1968-04-15', DATE '2014-06-04'), +(2, 1, 'Sales Assistant', 'Owen', 'Rivers', DATE '1988-08-13', DATE '2021-02-05'), +(3, 1, 'Programmer', 'Avram', 'Lawrence', DATE '1969-11-21', DATE '2010-09-29'), +(4, 1, 'Sales Assistant', 'Burton', 'Everett', DATE '2001-12-07', DATE '2016-06-25'), +(5, 1, 'Sales Assistant', 'Cedric', 'Barlow', DATE '1972-02-02', DATE '2012-08-15'), +(6, 2, 'Sales Assistant', 'Jasper', 'Mack', DATE '2002-03-29', DATE '2020-09-13'), +(7, 1, 'Sales Assistant', 'Felicia', 'Robinson', DATE '1973-08-21', DATE '2023-05-14'), +(8, 3, 'Sales Assistant', 'Mason', 'Steele', DATE '1964-05-19', DATE '2019-02-06'), +(9, 3, 'Programmer', 'Bernard', 'Cameron', DATE '1995-08-27', DATE '2018-07-12'), +(10, 2, 'Programmer', 'Chelsea', 'Wade', DATE '2007-01-29', DATE '2016-04-16'); + +SELECT + given_name, + family_name, + rating +FROM "test.jdbc-postgresql".gt_db1.employee_performance AS p +JOIN "test.jdbc-postgresql".gt_db1.employees AS e + ON p.employee_id = e.employee_id +ORDER BY +rating DESC +LIMIT 10; + +drop table "test.jdbc-postgresql".gt_db1.employee_performance; +drop table "test.jdbc-postgresql".gt_db1.employees; + +drop schema "test.jdbc-postgresql".gt_db1; \ No newline at end of file diff --git a/integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.txt b/integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.txt new file mode 100644 index 00000000000..4b75b585b91 --- /dev/null +++ b/integration-test/src/test/resources/trino-queries/catalogs/jdbc-postgresql/00003_join_pushdown.txt @@ -0,0 +1,28 @@ +CREATE SCHEMA + +USE + +CREATE TABLE + +CREATE TABLE + +INSERT: 10 rows + +INSERT: 10 rows + +"Avram","Lawrence","8" +"Avram","Lawrence","7" +"Gregory","Smith","7" +"Avram","Lawrence","6" +"Avram","Lawrence","4" +"Avram","Lawrence","4" +"Gregory","Smith","4" +"Gregory","Smith","4" +"Avram","Lawrence","4" +"Avram","Lawrence","2" + +DROP TABLE + +DROP TABLE + +DROP SCHEMA \ No newline at end of file diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java index 2468ad5b78b..ef7532b2f8f 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/GravitinoSplitManager.java @@ -30,11 +30,12 @@ public ConnectorSplitSource getSplits( GravitinoTransactionHandle gravitinoTransactionHandle = (GravitinoTransactionHandle) transaction; + // TODO(yuhui) add dynamic filter return internalSplitManager.getSplits( gravitinoTransactionHandle.getInternalTransactionHandle(), session, gravitinoTableHandle.getInternalTableHandle(), - dynamicFilter, + DynamicFilter.EMPTY, constraint); } } From 9450e5c1520e1f8889251bda1a236d4d805f00af Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 18 Dec 2023 17:35:20 +0800 Subject: [PATCH 13/51] [#1047] improvement(core): Fix the issue of StringIdentifier in metalake property (#1163) ### What changes were proposed in this pull request? Remove the `stringId` from metalake property map. ### Why are the changes needed? `gravitino.identifier` is generated internally and should not be visible to users. Fix: #1047 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Existing UTs can cover it. --- .../datastrato/gravitino/dto/MetalakeDTO.java | 39 ++++++++++++++++++- .../gravitino/StringIdentifier.java | 22 +++++++++++ .../gravitino/meta/BaseMetalake.java | 3 +- .../gravitino/meta/MetalakeManager.java | 5 +-- .../storage/kv/KvGarbageCollector.java | 2 +- .../gravitino/meta/TestMetalakeManager.java | 4 +- 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java b/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java index 3aabfd60c32..ef4cd90612b 100644 --- a/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java +++ b/common/src/main/java/com/datastrato/gravitino/dto/MetalakeDTO.java @@ -7,14 +7,13 @@ import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.Metalake; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import java.util.Map; import javax.annotation.Nullable; -import lombok.EqualsAndHashCode; import lombok.ToString; /** Represents a Metalake Data Transfer Object (DTO) that implements the Metalake interface. */ -@EqualsAndHashCode @ToString public class MetalakeDTO implements Metalake { @@ -132,4 +131,40 @@ public MetalakeDTO build() { return new MetalakeDTO(name, comment, properties, audit); } } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MetalakeDTO that = (MetalakeDTO) o; + return Objects.equal(name, that.name) + && Objects.equal(comment, that.comment) + && propertyEqual(properties, that.properties) + && Objects.equal(audit, that.audit); + } + + private boolean propertyEqual(Map p1, Map p2) { + if (p1 == null && p2 == null) { + return true; + } + + if (p1 != null && p1.isEmpty() && p2 == null) { + return true; + } + + if (p2 != null && p2.isEmpty() && p1 == null) { + return true; + } + + return java.util.Objects.equals(p1, p2); + } + + @Override + public int hashCode() { + return Objects.hashCode(name, comment, audit); + } } diff --git a/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java b/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java index 9f40c7dab87..942ce27ab53 100644 --- a/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java +++ b/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java @@ -7,6 +7,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -108,6 +109,27 @@ public static Map addToProperties( .build(); } + /** + * Remove StringIdentifier from properties. + * + * @param properties the properties to remove the string identifier from. + * @return the properties with the string identifier removed. + */ + public static Map removeIdFromProperties(Map properties) { + if (properties == null) { + return null; + } + + if (!properties.containsKey(ID_KEY)) { + return properties; + } + + Map copy = Maps.newHashMap(properties); + copy.remove(ID_KEY); + + return ImmutableMap.builder().putAll(copy).build(); + } + public static StringIdentifier fromProperties(Map properties) { if (properties == null) { return null; diff --git a/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java b/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java index 69cdef18d6b..f6de4bf2a54 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java @@ -10,6 +10,7 @@ import com.datastrato.gravitino.Field; import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.Metalake; +import com.datastrato.gravitino.StringIdentifier; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -126,7 +127,7 @@ public EntityType type() { */ @Override public Map properties() { - return properties; + return StringIdentifier.removeIdFromProperties(properties); } /** Builder class for creating instances of {@link BaseMetalake}. */ diff --git a/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java b/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java index f7214d58ed7..5ca502f367a 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java @@ -52,9 +52,8 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { @Override public BaseMetalake[] listMetalakes() { try { - return store - .list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE) - .toArray(new BaseMetalake[0]); + return store.list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE).stream() + .toArray(BaseMetalake[]::new); } catch (IOException ioe) { LOG.error("Listing Metalakes failed due to storage issues.", ioe); throw new RuntimeException(ioe); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java index 15d3bd25b1f..b481e635411 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvGarbageCollector.java @@ -44,7 +44,7 @@ public final class KvGarbageCollector implements Closeable { new ScheduledThreadPoolExecutor( 2, r -> { - Thread t = new Thread(r, "KvEntityStore-Garbage-Collector-%d"); + Thread t = new Thread(r, "KvEntityStore-Garbage-Collector"); t.setDaemon(true); return t; }, diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java index 7caf69d602e..eede4fb2c69 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java @@ -173,8 +173,6 @@ private void testProperties(Map expectedProps, Map Date: Mon, 18 Dec 2023 05:09:41 -0600 Subject: [PATCH 14/51] [MINOR] fix(docs): Fix a typo (#1189) ### What changes were proposed in this pull request? Fix a typo ### Why are the changes needed? Just a typo ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No need. Co-authored-by: Heng Qin --- docs/security.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/security.md b/docs/security.md index 9919c9d6fe8..69df5475dc8 100644 --- a/docs/security.md +++ b/docs/security.md @@ -173,7 +173,7 @@ Both Gravitino server and Iceberg REST service can configure HTTPS. | `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.uxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | From dbdc1c9a6cbf31cff9d56c2e93a8bf8c1a9b13cd Mon Sep 17 00:00:00 2001 From: Clearvive <143773256+Clearvive@users.noreply.github.com> Date: Mon, 18 Dec 2023 19:52:15 +0800 Subject: [PATCH 15/51] [#1171] bugfix(postgresql): Fix bug where the same table name can be loaded under multiple schemas. (#1183) ### What changes were proposed in this pull request? Fix a bug where the same table name can be loaded under multiple schemas ### Why are the changes needed? Fix: #1171 ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? UT Co-authored-by: Clearvive --- .../operation/PostgreSqlTableOperations.java | 26 ++++++++++++------- .../TestPostgreSqlTableOperations.java | 17 +++++++++--- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index af971586ee2..ce471880a9e 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -49,17 +49,17 @@ public class PostgreSqlTableOperations extends JdbcTableOperations { + " pg_namespace AS n ON n.oid = c.relnamespace\n" + "WHERE \n" + " a.attnum > 0 \n" - + " AND c.relname = ?"; + + " AND c.relname = ? AND n.nspname = ?"; private static final String SHOW_COLUMN_INFO_SQL = - "select * FROM information_schema.columns WHERE table_name = ? order by ordinal_position"; + "select * FROM information_schema.columns WHERE table_name = ? AND table_schema = ? order by ordinal_position"; private static final String SHOW_TABLE_COMMENT_SQL = "SELECT tb.table_name, d.description\n" + "FROM information_schema.tables tb\n" + " JOIN pg_class c ON c.relname = tb.table_name\n" + " LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = '0'\n" - + "WHERE tb.table_name = ?;"; + + "WHERE tb.table_name = ? AND table_schema = ?;"; private String database; @@ -80,15 +80,18 @@ public void initialize( public JdbcTable load(String schema, String tableName) throws NoSuchTableException { try (Connection connection = getConnection(schema)) { // The first step is to obtain the comment information of the column. - Map columnCommentMap = selectColumnComment(tableName, connection); + Map columnCommentMap = selectColumnComment(schema, tableName, connection); // The second step is to obtain the column information of the table. List jdbcColumns = selectColumnInfoAndExecute( - tableName, connection, (builder, s) -> builder.withComment(columnCommentMap.get(s))); + schema, + tableName, + connection, + (builder, s) -> builder.withComment(columnCommentMap.get(s))); // The third step is to obtain the comment information of the table. - String comment = selectTableComment(tableName, connection); + String comment = selectTableComment(schema, tableName, connection); return new JdbcTable.Builder() .withName(tableName) .withColumns(jdbcColumns.toArray(new JdbcColumn[0])) @@ -102,6 +105,7 @@ public JdbcTable load(String schema, String tableName) throws NoSuchTableExcepti } private List selectColumnInfoAndExecute( + String schemaName, String tableName, Connection connection, BiConsumer builderConsumer) @@ -109,6 +113,7 @@ private List selectColumnInfoAndExecute( List jdbcColumns = new ArrayList<>(); try (PreparedStatement preparedStatement = connection.prepareStatement(SHOW_COLUMN_INFO_SQL)) { preparedStatement.setString(1, tableName); + preparedStatement.setString(2, schemaName); try (ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { ColDataType colDataType = new ColDataType(); @@ -153,10 +158,12 @@ private static List getArgList(ResultSet resultSet) throws SQLException return result; } - private String selectTableComment(String tableName, Connection connection) throws SQLException { + private String selectTableComment(String schema, String tableName, Connection connection) + throws SQLException { try (PreparedStatement preparedStatement = connection.prepareStatement(SHOW_TABLE_COMMENT_SQL)) { preparedStatement.setString(1, tableName); + preparedStatement.setString(2, schema); try (ResultSet resultSet = preparedStatement.executeQuery()) { if (resultSet.next()) { return resultSet.getString("description"); @@ -170,13 +177,14 @@ private String selectTableComment(String tableName, Connection connection) throw * @return Returns the column names and comments of the table * @throws SQLException */ - private Map selectColumnComment(String tableName, Connection connection) - throws SQLException { + private Map selectColumnComment( + String schema, String tableName, Connection connection) throws SQLException { Map columnCommentMap = new HashMap<>(); try (PreparedStatement preparedStatement = connection.prepareStatement(SHOW_COLUMN_COMMENT_SQL)) { preparedStatement.setString(1, tableName); + preparedStatement.setString(2, schema); try (ResultSet resultSet = preparedStatement.executeQuery()) { while (resultSet.next()) { String comment = resultSet.getString("comment"); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java index 0da2a7e874f..afa96964797 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java @@ -350,6 +350,14 @@ public void testCreateMultipleTable() throws SQLException { List tableNames = TABLE_OPERATIONS.listTables(TEST_DB_NAME); Assertions.assertFalse(tableNames.contains(table_1)); + Assertions.assertThrows( + NoSuchTableException.class, () -> TABLE_OPERATIONS.load(TEST_DB_NAME, table_1)); + + Assertions.assertThrows( + NoSuchTableException.class, () -> TABLE_OPERATIONS.load("other_schema", table_1)); + Assertions.assertThrows( + NoSuchTableException.class, () -> postgreSqlTableOperations.load("other_schema", table_1)); + String table_2 = "table_multiple_2"; TABLE_OPERATIONS.create( TEST_DB_NAME, @@ -369,10 +377,11 @@ public void testCreateMultipleTable() throws SQLException { Assertions.assertFalse(tableNames.contains(table_2)); Assertions.assertThrows( - NoSuchTableException.class, - () -> { - postgreSqlTableOperations.load(TEST_DB_NAME, table_2); - }); + NoSuchTableException.class, () -> postgreSqlTableOperations.load(TEST_DB_NAME, table_2)); + Assertions.assertThrows( + NoSuchTableException.class, () -> postgreSqlTableOperations.load("other_schema", table_2)); + Assertions.assertThrows( + NoSuchTableException.class, () -> TABLE_OPERATIONS.load("other_schema", table_2)); postgreSqlTableOperations.purge(TEST_DB_NAME, table_1); Assertions.assertThrows( From 2477685aa93c277eb632106581e4f993ed85ff61 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Tue, 19 Dec 2023 17:36:33 +0800 Subject: [PATCH 16/51] [#1179] feat(build): Make Gravitino build and run against JDK8, 11, 17 (#1199) ### What changes were proposed in this pull request? This PR proposes to add multiple JDK version support for building and running Gravitino server, including JDK8, 11, and 17. Users could specify the version they want to build in `gradle.properties`, like: ``` jdkVersion = 8 or jdkVersion = 11 or jdkVersion = 17 ``` Or users could specify the version by `./gradlew build -PjdkVersion=8`. Our generated jars are JDK 8 version compatible, so it can run against Java runtime 8, 11, and 17. ### Why are the changes needed? To make Gravitino build, test and run against multiple JDK versions. Fix: #1179 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? This was compiled, tested, run on JDK8, 11, and 17 locally. --- build.gradle.kts | 46 +++++++++++++++++-- .../catalog/hive/dyn/DynConstructors.java | 4 +- .../gravitino/catalog/hive/dyn/DynFields.java | 4 +- .../catalog/hive/dyn/DynMethods.java | 4 +- .../gravitino/EntitySerDeFactory.java | 2 +- .../gravitino/EntityStoreFactory.java | 2 +- .../aux/AuxiliaryServiceManager.java | 2 +- .../gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/proto/ProtoEntitySerDe.java | 2 +- .../gravitino/storage/kv/KvEntityStore.java | 3 +- docs/trino-connector/index.md | 2 +- gradle.properties | 4 ++ .../gravitino/server/web/JettyServer.java | 4 +- 13 files changed, 63 insertions(+), 18 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ed1b59048d2..ab9c74558a3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -32,7 +32,7 @@ plugins { alias(libs.plugins.spotless) } else { throw GradleException( - "Gravitino Gradle current doesn't support " + + "Gravitino Gradle toolchain current doesn't support " + "Java version: ${JavaVersion.current()}. Please use JDK8 to 17." ) } @@ -44,6 +44,42 @@ plugins { id("org.cyclonedx.bom") version "1.5.0" // Newer version fail due to our setup } +if (extra["jdkVersion"] !in listOf("8", "11", "17")) { + throw GradleException( + "Gravitino current doesn't support building with " + + "Java version: ${extra["jdkVersion"]}. Please use JDK8, 11 or 17." + ) +} + +project.extra["extraJvmArgs"] = if (extra["jdkVersion"] in listOf("8", "11")) { + listOf() +} else { + listOf( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens", "java.base/java.io=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens", "java.base/java.lang=ALL-UNNAMED", + "--add-opens", "java.base/java.math=ALL-UNNAMED", + "--add-opens", "java.base/java.net=ALL-UNNAMED", + "--add-opens", "java.base/java.nio=ALL-UNNAMED", + "--add-opens", "java.base/java.text=ALL-UNNAMED", + "--add-opens", "java.base/java.time=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens", "java.base/java.util.regex=ALL-UNNAMED", + "--add-opens", "java.base/java.util=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens", "java.base/jdk.internal.reflect=ALL-UNNAMED", + "--add-opens", "java.sql/java.sql=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens", "java.base/sun.security.action=ALL-UNNAMED", + "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED" + ) +} + licenseReport { renderers = arrayOf(InventoryHtmlReportRenderer("report.html", "Backend")) filters = arrayOf(LicenseBundleNormalizer()) @@ -129,7 +165,9 @@ subprojects { if (project.name == "trino-connector") { languageVersion.set(JavaLanguageVersion.of(17)) } else { - languageVersion.set(JavaLanguageVersion.of(8)) + languageVersion.set(JavaLanguageVersion.of(extra["jdkVersion"].toString().toInt())) + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 } } } @@ -208,7 +246,7 @@ subprojects { sign(publishing.publications) } - tasks.configureEach { + tasks.withType { testLogging { exceptionFormat = TestExceptionFormat.FULL showExceptions = true @@ -224,6 +262,8 @@ subprojects { } else { useJUnitPlatform() } + + jvmArgs(project.property("extraJvmArgs") as List<*>) finalizedBy(tasks.getByName("jacocoTestReport")) } } diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java index 9a254951a4e..593df47eb86 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynConstructors.java @@ -23,7 +23,6 @@ import com.google.common.collect.Maps; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Map; @@ -245,6 +244,7 @@ public Builder hiddenImpl(String className, Class... types) { return this; } + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, Class... types) { // don't do any work if an implementation has been found if (ctor != null) { @@ -253,7 +253,7 @@ public Builder hiddenImpl(Class targetClass, Class... types) { try { Constructor hidden = targetClass.getDeclaredConstructor(types); - AccessController.doPrivileged(new MakeAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeAccessible(hidden)); ctor = new Ctor(hidden, targetClass); } catch (SecurityException e) { // unusable diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java index 3c8f61cba8a..0d7f67d98dc 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynFields.java @@ -25,7 +25,6 @@ import com.google.common.collect.Sets; import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Set; @@ -302,6 +301,7 @@ public Builder hiddenImpl(String className, String fieldName) { * @see Class#forName(String) * @see Class#getField(String) */ + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, String fieldName) { // don't do any work if an implementation has been found if (field != null || targetClass == null) { @@ -310,7 +310,7 @@ public Builder hiddenImpl(Class targetClass, String fieldName) { try { Field hidden = targetClass.getDeclaredField(fieldName); - AccessController.doPrivileged(new MakeFieldAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeFieldAccessible(hidden)); this.field = new UnboundField(hidden, fieldName); } catch (SecurityException | NoSuchFieldException e) { // unusable diff --git a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java index 7d07585ba34..cc27dbb002c 100644 --- a/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java +++ b/catalogs/catalog-hive/src/main/java/com/datastrato/gravitino/catalog/hive/dyn/DynMethods.java @@ -23,7 +23,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.Arrays; @@ -393,6 +392,7 @@ public Builder hiddenImpl(String className, Class... argClasses) { * @see Class#forName(String) * @see Class#getMethod(String, Class[]) */ + @SuppressWarnings("removal") public Builder hiddenImpl(Class targetClass, String methodName, Class... argClasses) { // don't do any work if an implementation has been found if (method != null) { @@ -401,7 +401,7 @@ public Builder hiddenImpl(Class targetClass, String methodName, Class... a try { Method hidden = targetClass.getDeclaredMethod(methodName, argClasses); - AccessController.doPrivileged(new MakeAccessible(hidden)); + java.security.AccessController.doPrivileged(new MakeAccessible(hidden)); this.method = new UnboundMethod(hidden, name); } catch (SecurityException | NoSuchMethodException e) { // unusable or not the right implementation diff --git a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java index a5472f2e261..fc59fe77357 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntitySerDeFactory.java @@ -48,7 +48,7 @@ public static EntitySerDe createEntitySerDe(String name) { String className = ENTITY_SERDES.getOrDefault(name, name); try { - return (EntitySerDe) Class.forName(className).newInstance(); + return (EntitySerDe) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to create EntitySerDe by name {}.", name, e); throw new RuntimeException("Failed to create EntitySerDe: " + name, e); diff --git a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java index 2bac6badeed..8f1a0367821 100644 --- a/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java +++ b/core/src/main/java/com/datastrato/gravitino/EntityStoreFactory.java @@ -36,7 +36,7 @@ public static EntityStore createEntityStore(Config config) { String className = ENTITY_STORES.getOrDefault(name, name); try { - return (EntityStore) Class.forName(className).newInstance(); + return (EntityStore) Class.forName(className).getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to create and initialize EntityStore by name {}.", name, e); throw new RuntimeException("Failed to create and initialize EntityStore: " + name, e); diff --git a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java index cc1ce3d1167..05fb03f3c28 100644 --- a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java +++ b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java @@ -71,7 +71,7 @@ public GravitinoAuxiliaryService loadAuxService( try { Class providerClz = lookupAuxService(auxServiceName, cl); - return providerClz.newInstance(); + return providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 4f41a0e7c32..9fb32d2c9b3 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -494,7 +494,7 @@ private BaseCatalog createCatalogInstance(IsolatedClassLoader classLoader, St try { Class providerClz = lookupCatalogProvider(provider, cl); - return (BaseCatalog) providerClz.newInstance(); + return (BaseCatalog) providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { LOG.error("Failed to load catalog with provider: {}", provider, e); throw new RuntimeException(e); diff --git a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java index dd93680d976..0026e6df918 100644 --- a/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java +++ b/core/src/main/java/com/datastrato/gravitino/proto/ProtoEntitySerDe.java @@ -93,7 +93,7 @@ private ProtoSerDe getProtoSerde( Class> serdeClazz = (Class>) loadClass(ENTITY_TO_SERDE.get(k.getCanonicalName()), classLoader); - return serdeClazz.newInstance(); + return serdeClazz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException( "Failed to instantiate serde class " + k.getCanonicalName(), e); diff --git a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java index 8db173f834e..d985e7c2f9c 100644 --- a/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java +++ b/core/src/main/java/com/datastrato/gravitino/storage/kv/KvEntityStore.java @@ -421,7 +421,8 @@ private static KvBackend createKvEntityBackend(Config config) { } try { - KvBackend kvBackend = (KvBackend) Class.forName(className).newInstance(); + KvBackend kvBackend = + (KvBackend) Class.forName(className).getDeclaredConstructor().newInstance(); kvBackend.initialize(config); return kvBackend; } catch (Exception e) { diff --git a/docs/trino-connector/index.md b/docs/trino-connector/index.md index 3a1bd676904..4c1d2e9f5be 100644 --- a/docs/trino-connector/index.md +++ b/docs/trino-connector/index.md @@ -17,4 +17,4 @@ Gravitino connector index: - [Iceberg](catalog-iceberg.md) - [MySQL](catalog-mysql.md) - [PostgreSQL](catalog-postgresql.md) - - [Supported SQL](sql-support.md) \ No newline at end of file + - [Supported SQL](sql-support.md) diff --git a/gradle.properties b/gradle.properties index 819f90e6cc7..0dbdc6de0a2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -14,3 +14,7 @@ version = 0.4.0-SNAPSHOT #sonatype credentials SONATYPE_USER = admin SONATYPE_PASSWORD = password + +#jdkVersion is used to specify the version of JDK to build and test against Gravitino, current +# supported version is 8, 11, and 17. +jdkVersion = 8 diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index cd6592d2ee8..f111af3a0a3 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -16,7 +16,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.AccessController; import java.security.PrivilegedAction; import java.util.EnumSet; import java.util.Optional; @@ -393,6 +392,7 @@ private ServerConnector createServerConnector( return new ServerConnector(server, null, serverExecutor, null, -1, -1, connectionFactories); } + @SuppressWarnings("removal") private ThreadPool createThreadPool(int minThreads, int maxThreads, int threadPoolWorkQueueSize) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); @@ -404,7 +404,7 @@ private ThreadPool createThreadPool(int minThreads, int maxThreads, int threadPo @Override public Thread newThread(Runnable runnable) { - return AccessController.doPrivileged( + return java.security.AccessController.doPrivileged( new PrivilegedAction() { @Override public Thread run() { From 0ef2899fe26870a824fcfb259d8b2dc603cf4acd Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Tue, 19 Dec 2023 17:40:38 +0800 Subject: [PATCH 17/51] [#995] improvement(test): Remove annotation Order to alleviate test sequence dependency (#1197) ### What changes were proposed in this pull request? Remove Junit5 annotation `Order` in test. ### Why are the changes needed? We need to remove sequence dependencies between test cases so that we can run them in parallelism. Fix: #995 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Only changes to the test code have been made. --- .../gravitino/catalog/TestCatalogManager.java | 7 ---- .../storage/kv/TestEntityKeyEncoding.java | 6 ---- .../storage/kv/TestKvNameMappingService.java | 7 ---- .../test/catalog/hive/CatalogHiveIT.java | 28 +++++++++++----- .../catalog/jdbc/mysql/CatalogMysqlIT.java | 3 -- .../jdbc/postgresql/CatalogPostgreSqlIT.java | 3 -- .../lakehouse/iceberg/CatalogIcebergIT.java | 3 -- .../test/container/TrinoContainer.java | 22 ++----------- .../test/trino/TrinoConnectorIT.java | 33 ++++--------------- 9 files changed, 28 insertions(+), 84 deletions(-) diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java index 231f97ee17e..cfa6a24be1e 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java @@ -34,13 +34,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.mockito.Mockito; -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class TestCatalogManager { private static CatalogManager catalogManager; @@ -136,7 +132,6 @@ void testCreateWithHiveProperty() throws IOException { } @Test - @Order(1) void testLoadTable() throws IOException { NameIdentifier ident = NameIdentifier.of("metalake", "test444"); // key1 is required; @@ -161,7 +156,6 @@ void testLoadTable() throws IOException { } @Test - @Order(2) void testPropertyValidationInAlter() throws IOException { // key1 is required and immutable and do not have default value, is not hidden and not reserved // key2 is required and mutable and do not have default value, is not hidden and not reserved @@ -228,7 +222,6 @@ void testPropertyValidationInAlter() throws IOException { } @Test - @Order(3) void testPropertyValidationInCreate() throws IOException { // key1 is required and immutable and do not have default value, is not hidden and not reserved // key2 is required and mutable and do not have default value, is not hidden and not reserved diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java index 929922ffb33..89cb2d16e27 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestEntityKeyEncoding.java @@ -29,16 +29,12 @@ import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.ClassOrderer.OrderAnnotation; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestClassOrder; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; import org.mockito.Mockito; @TestInstance(Lifecycle.PER_CLASS) -@TestClassOrder(OrderAnnotation.class) public class TestEntityKeyEncoding { private Config getConfig() throws IOException { File baseDir = new File(System.getProperty("java.io.tmpdir")); @@ -75,7 +71,6 @@ private IdGenerator getIdGeneratorAndSpy(BinaryEntityKeyEncoder entityKeyEncoder } @Test - @Order(1) public void testIdentifierEncoding() throws IOException, IllegalAccessException, NoSuchFieldException { Config config = getConfig(); @@ -192,7 +187,6 @@ public void testIdentifierEncoding() } @Test - @Order(10) public void testNamespaceEncoding() throws IOException, IllegalAccessException, NoSuchFieldException { Config config = getConfig(); diff --git a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java index eb1dda6b4f2..38ddd83db75 100644 --- a/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java +++ b/core/src/test/java/com/datastrato/gravitino/storage/kv/TestKvNameMappingService.java @@ -23,13 +23,9 @@ import java.lang.reflect.Field; import java.nio.file.Files; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.ClassOrderer.OrderAnnotation; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestClassOrder; import org.mockito.Mockito; -@TestClassOrder(OrderAnnotation.class) public class TestKvNameMappingService { private Config getConfig() throws IOException { File baseDir = new File(System.getProperty("java.io.tmpdir")); @@ -63,7 +59,6 @@ private IdGenerator getIdGeneratorByReflection(NameMappingService nameMappingSer } @Test - @Order(1) public void testGetIdByName() throws Exception { try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { NameMappingService nameMappingService = kvEntityStore.nameMappingService; @@ -85,7 +80,6 @@ public void testGetIdByName() throws Exception { } @Test - @Order(2) public void testUpdateName() throws Exception { try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { NameMappingService nameMappingService = kvEntityStore.nameMappingService; @@ -111,7 +105,6 @@ public void testUpdateName() throws Exception { } @Test - @Order(3) public void testBindAndUnBind() throws Exception { try (KvEntityStore kvEntityStore = getKvEntityStore(getConfig())) { KvNameMappingService nameMappingService = diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java index 1e0fe8aa2c5..e0041d4edbb 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java @@ -79,16 +79,12 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Tag("gravitino-docker-it") -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CatalogHiveIT extends AbstractIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogHiveIT.class); public static final String metalakeName = @@ -960,9 +956,9 @@ void testLoadEntityWithSamePrefix() { } @Test - // Make sure it will be executed at last. - @Order(Integer.MAX_VALUE) void testAlterEntityName() { + String metalakeName = GravitinoITUtils.genRandomName("CatalogHiveIT_metalake"); + client.createMetalake(NameIdentifier.of(metalakeName), "", ImmutableMap.of()); final GravitinoMetaLake metalake = client.loadMetalake(NameIdentifier.of(metalakeName)); String newMetalakeName = GravitinoITUtils.genRandomName("CatalogHiveIT_metalake_new"); @@ -984,6 +980,14 @@ void testAlterEntityName() { () -> client.loadMetalake(NameIdentifier.of(newMetalakeName))); } + String catalogName = GravitinoITUtils.genRandomName("CatalogHiveIT_catalog"); + metalake.createCatalog( + NameIdentifier.of(metalakeName, catalogName), + Catalog.Type.RELATIONAL, + provider, + "comment", + ImmutableMap.of(METASTORE_URIS, HIVE_METASTORE_URIS)); + Catalog catalog = metalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); // Test rename catalog String newCatalogName = GravitinoITUtils.genRandomName("CatalogHiveIT_catalog_new"); @@ -1007,7 +1011,16 @@ void testAlterEntityName() { } // Schema does not have the rename operation. + final String schemaName = GravitinoITUtils.genRandomName("CatalogHiveIT_schema"); + catalog + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), "", ImmutableMap.of()); + final Catalog cata = catalog; + // Now try to rename table + final String tableName = GravitinoITUtils.genRandomName("CatalogHiveIT_table"); + final String newTableName = GravitinoITUtils.genRandomName("CatalogHiveIT_table_new"); ColumnDTO[] columns = createColumns(); catalog .asTableCatalog() @@ -1018,9 +1031,6 @@ void testAlterEntityName() { createProperties(), new Transform[0]); - final Catalog cata = catalog; - // Now try to rename table - final String newTableName = GravitinoITUtils.genRandomName("CatalogHiveIT_table_new"); for (int i = 0; i < 2; i++) { // The table to be renamed does not exist Assertions.assertThrows( diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index 3236a473827..597dd12bf7f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -48,14 +48,11 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.testcontainers.containers.MySQLContainer; @Tag("gravitino-docker-it") -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CatalogMysqlIT extends AbstractIT { public static String metalakeName = GravitinoITUtils.genRandomName("mysql_it_metalake"); public static String catalogName = GravitinoITUtils.genRandomName("mysql_it_catalog"); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index 23f564ec37e..e9288e7028f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -44,14 +44,11 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.testcontainers.containers.PostgreSQLContainer; @Tag("gravitino-docker-it") -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CatalogPostgreSqlIT extends AbstractIT { public static String metalakeName = GravitinoITUtils.genRandomName("postgresql_it_metalake"); public static String catalogName = GravitinoITUtils.genRandomName("postgresql_it_catalog"); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java index 36e63ccea50..b679eecf75d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java @@ -67,13 +67,10 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; @Tag("gravitino-docker-it") -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class CatalogIcebergIT extends AbstractIT { public static String metalakeName = GravitinoITUtils.genRandomName("iceberg_it_metalake"); public static String catalogName = GravitinoITUtils.genRandomName("iceberg_it_catalog"); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java index 4dfd8942fe2..a3f31afe841 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/TrinoContainer.java @@ -32,10 +32,6 @@ public class TrinoContainer extends BaseContainer { static Connection trinoJdbcConnection = null; - public static final String TRINO_CONF_GRAVITINO_URI = "gravitino.uri"; - public static final String TRINO_CONF_GRAVITINO_METALAKE = "gravitino.metalake"; - public static final String TRINO_CONF_HIVE_METASTORE_URI = "hive.metastore.uri"; - public static final String TRINO_CONTAINER_CONF_DIR = "/etc/trino"; public static final String TRINO_CONTAINER_PLUGIN_GRAVITINO_DIR = "/usr/lib/trino/plugin/gravitino"; @@ -50,11 +46,7 @@ protected TrinoContainer( Map extraHosts, Map filesToMount, Map envVars, - Optional network, - String trinoConfDir, - InetSocketAddress gravitinoServerAddress, - String metalakeName, - String hiveContainerIP) { + Optional network) { super(image, hostName, ports, extraHosts, filesToMount, envVars, network); } @@ -229,17 +221,7 @@ public TrinoContainer.Builder withHiveContainerIP(String hiveContainerIP) { @Override public TrinoContainer build() { return new TrinoContainer( - image, - hostName, - exposePorts, - extraHosts, - filesToMount, - envVars, - network, - trinoConfDir, - gravitinoServerAddress, - metalakeName, - hiveContainerIP); + image, hostName, exposePorts, extraHosts, filesToMount, envVars, network); } } } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 810fceddce6..9dc3029087d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -36,16 +36,12 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Tag("gravitino-docker-it") -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class TrinoConnectorIT extends AbstractIT { public static final Logger LOG = LoggerFactory.getLogger(TrinoConnectorIT.class); @@ -142,7 +138,6 @@ public static void createSchema() throws TException, InterruptedException { } @Test - @Order(1) public void testShowSchemas() { String sql = String.format( @@ -153,7 +148,6 @@ public void testShowSchemas() { } @Test - @Order(2) public void testCreateTable() throws TException, InterruptedException { String sql3 = String.format( @@ -178,11 +172,11 @@ public void testCreateTable() throws TException, InterruptedException { hiveClientPool.run(client -> client.getTable(databaseName, tab1Name)); Assertions.assertEquals(databaseName, hiveTab1.getDbName()); Assertions.assertEquals(tab1Name, hiveTab1.getTableName()); + + testShowTable(); } - @Order(3) - @Test - public void testShowTable() { + void testShowTable() { String sql = String.format( "SHOW TABLES FROM \"%s.%s\".%s LIKE '%s'", @@ -192,8 +186,6 @@ public void testShowTable() { Assertions.assertEquals(queryData.get(0).get(0), tab1Name); } - @Test - @Order(4) public void testScenarioTable1() throws TException, InterruptedException { String sql3 = String.format( @@ -260,8 +252,6 @@ public void testScenarioTable1() throws TException, InterruptedException { Assertions.assertEquals(table1Data, table1QueryData); } - @Test - @Order(5) public void testScenarioTable2() throws TException, InterruptedException { String sql4 = String.format( @@ -326,8 +316,10 @@ public void testScenarioTable2() throws TException, InterruptedException { } @Test - @Order(6) - public void testScenarioJoinTwoTable() { + public void testScenarioJoinTwoTable() throws TException, InterruptedException { + testScenarioTable1(); + testScenarioTable2(); + String sql9 = String.format( "SELECT * FROM (SELECT t1.user_name as user_name, gender, age, phone, consumer, recharge, event_time FROM \"%1$s.%2$s\".%3$s.%4$s AS t1\n" @@ -355,7 +347,6 @@ public void testScenarioJoinTwoTable() { } @Test - @Order(7) void testHiveSchemaCreatedByTrino() { String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); @@ -373,7 +364,6 @@ void testHiveSchemaCreatedByTrino() { } @Test - @Order(8) void testHiveTableCreatedByTrino() { String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("table").toLowerCase(); @@ -400,7 +390,6 @@ void testHiveTableCreatedByTrino() { } @Test - @Order(9) void testHiveSchemaCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase(); String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); @@ -551,7 +540,6 @@ private ColumnDTO[] createFullTypeColumns() { } @Test - @Order(10) void testHiveTableCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase(); String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); @@ -624,7 +612,6 @@ void testHiveTableCreatedByGravitino() throws InterruptedException { } @Test - @Order(11) void testHiveCatalogCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase(); GravitinoMetaLake createdMetalake = client.loadMetalake(NameIdentifier.of(metalakeName)); @@ -663,7 +650,6 @@ void testHiveCatalogCreatedByGravitino() throws InterruptedException { } @Test - @Order(12) void testWrongHiveCatalogProperty() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase(); GravitinoMetaLake createdMetalake = client.loadMetalake(NameIdentifier.of(metalakeName)); @@ -697,7 +683,6 @@ void testWrongHiveCatalogProperty() throws InterruptedException { Assertions.assertTrue(containerSuite.getTrinoContainer().executeQuerySQL(sql).isEmpty()); } - @Order(13) @Test void testIcebergTableAndSchemaCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("catalog").toLowerCase(); @@ -767,7 +752,6 @@ void testIcebergTableAndSchemaCreatedByGravitino() throws InterruptedException { } @Test - @Order(14) void testIcebergTableAndSchemaCreatedByTrino() { String schemaName = GravitinoITUtils.genRandomName("schema").toLowerCase(); String tableName = GravitinoITUtils.genRandomName("table").toLowerCase(); @@ -790,7 +774,6 @@ void testIcebergTableAndSchemaCreatedByTrino() { } @Test - @Order(15) void testIcebergCatalogCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("iceberg_catalog").toLowerCase(); GravitinoMetaLake createdMetalake = client.loadMetalake(NameIdentifier.of(metalakeName)); @@ -837,7 +820,6 @@ void testIcebergCatalogCreatedByGravitino() throws InterruptedException { } @Test - @Order(16) void testMySQLCatalogCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); GravitinoMetaLake createdMetalake = client.loadMetalake(NameIdentifier.of(metalakeName)); @@ -878,7 +860,6 @@ void testMySQLCatalogCreatedByGravitino() throws InterruptedException { } @Test - @Order(17) void testMySQLTableCreatedByGravitino() throws InterruptedException { String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); String schemaName = GravitinoITUtils.genRandomName("mysql_schema").toLowerCase(); From 4bafbd01bcc476fe791325bad09a2ecac91e16ba Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Tue, 19 Dec 2023 19:44:49 +0800 Subject: [PATCH 18/51] [#1156] improvement(trino-connector): Support synchronization for drop catalog operation between Trino and Gravitino (#1157) ### What changes were proposed in this pull request? Trino connector will drop catalogs that have been deleted in Gravitino server. ### Why are the changes needed? We need to ensure strict synchronization for catalogs between Trino and Gravitino Fix: #1156 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Add test `testDropCatalogAndCreateAgain` in `TrinoConnectorIT`. --- .../test/trino/TrinoConnectorIT.java | 67 +++++++++++++++++++ .../catalog/CatalogConnectorManager.java | 19 +++++- .../connector/catalog/CatalogInjector.java | 41 ++++++++++++ .../trino/connector/GravitinoMockServer.java | 10 ++- .../connector/TestGravitinoConnector.java | 14 +++- 5 files changed, 146 insertions(+), 5 deletions(-) diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index 9dc3029087d..b8255d61202 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -435,6 +435,31 @@ void testHiveSchemaCreatedByGravitino() throws InterruptedException { "location = 'hdfs://localhost:9000/user/hive/warehouse/hive_schema_1223445.db'")); } + private static boolean checkTrinoHasRemoved(String sql, long maxWaitTimeSec) { + long current = System.currentTimeMillis(); + while (System.currentTimeMillis() - current <= maxWaitTimeSec * 1000) { + try { + ArrayList> lists = + containerSuite.getTrinoContainer().executeQuerySQL(sql); + if (lists.isEmpty()) { + return true; + } + + LOG.info("Catalog has not synchronized yet, wait 200ms and retry. The SQL is '{}'", sql); + } catch (Exception e) { + LOG.warn("Failed to execute sql: {}", sql, e); + } + + try { + Thread.sleep(200); + } catch (InterruptedException e) { + LOG.warn("Failed to sleep 200ms", e); + } + } + + return false; + } + private static boolean checkTrinoHasLoaded(String sql, long maxWaitTimeSec) throws InterruptedException { long current = System.currentTimeMillis(); @@ -935,6 +960,48 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { } } + @Test + void testDropCatalogAndCreateAgain() throws InterruptedException { + String catalogName = GravitinoITUtils.genRandomName("mysql_catalog").toLowerCase(); + GravitinoMetaLake createdMetalake = client.loadMetalake(NameIdentifier.of(metalakeName)); + String[] command = { + "mysql", + "-h127.0.0.1", + "-uroot", + "-pds123", // username and password are referred from Hive dockerfile. + "-e", + "grant all privileges on *.* to root@'%' identified by 'ds123'" + }; + + // There exists a mysql instance in Hive the container. + containerSuite.getHiveContainer().executeInContainer(command); + String hiveHost = containerSuite.getHiveContainer().getContainerIpAddress(); + + // Create the catalog and drop it for 3 times to test the create/drop catalog function work + // well. + for (int i = 0; i < 3; i++) { + createdMetalake.createCatalog( + NameIdentifier.of(metalakeName, catalogName), + Catalog.Type.RELATIONAL, + "jdbc-mysql", + "comment", + ImmutableMap.builder() + .put("jdbc-user", "root") + .put("jdbc-password", "ds123") + .put("jdbc-url", String.format("jdbc:mysql://%s:3306?useSSL=false", hiveHost)) + .build()); + + String sql = String.format("show catalogs like '%s.%s'", metalakeName, catalogName); + boolean success = checkTrinoHasLoaded(sql, 30); + Assertions.assertTrue(success, "Trino should load the catalog: " + sql); + + createdMetalake.dropCatalog(NameIdentifier.of(metalakeName, catalogName)); + // We need to test we can't load this catalog any more by Trino. + success = checkTrinoHasRemoved(sql, 30); + Assertions.assertFalse(success, "Trino should not load the catalog any more: " + sql); + } + } + private static void createMetalake() { GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetaLakes.length); diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java index 8110145f4a7..804f479d1c7 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogConnectorManager.java @@ -21,10 +21,12 @@ import io.trino.spi.TrinoException; import java.util.Arrays; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.lang3.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,7 +123,8 @@ void loadMetalake() { } } - void loadCatalogs(GravitinoMetaLake metalake) { + @VisibleForTesting + public void loadCatalogs(GravitinoMetaLake metalake) { NameIdentifier[] catalogNames; try { catalogNames = metalake.listCatalogs(Namespace.ofCatalog(metalake.name())); @@ -135,6 +138,20 @@ void loadCatalogs(GravitinoMetaLake metalake) { metalake.name(), Arrays.toString(catalogNames)); + // Delete those catalogs that have been deleted in Gravitino server + Set catalogNameStrings = + Arrays.stream(catalogNames).map(NameIdentifier::toString).collect(Collectors.toSet()); + + catalogConnectors.keySet().stream() + .filter(catalogName -> !catalogNameStrings.contains(catalogName)) + .forEach( + (String catalogName) -> { + catalogInjector.removeCatalogConnector(catalogName); + catalogConnectors.remove(catalogName); + LOG.info( + "Remove catalog '{}' in metalake {} successfully.", catalogName, metalake.name()); + }); + Arrays.stream(catalogNames) .forEach( (NameIdentifier nameIdentifier) -> { diff --git a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java index 38064bdafd0..d63844290e8 100644 --- a/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java +++ b/trino-connector/src/main/java/com/datastrato/gravitino/trino/connector/catalog/CatalogInjector.java @@ -36,6 +36,8 @@ public class CatalogInjector { // It is used to inject catalogs to trino private InjectCatalogHandle injectHandle; + // It's used to remove catalogs from trino + private RemoveCatalogHandle removeHandle; // It is used to create internal catalogs. private CreateCatalogHandle createHandle; @@ -171,6 +173,7 @@ public void init(ConnectorContext context) { createInjectHandler( catalogManager, catalogFactory, createCatalogMethod, catalogPropertiesClass); + removeInjectHandle(catalogManager); LOG.info("Bind Trino catalog manager successfully."); } catch (Exception e) { String message = @@ -181,6 +184,30 @@ public void init(ConnectorContext context) { } } + private void removeInjectHandle(Object catalogManager) + throws NoSuchFieldException, IllegalAccessException { + if (isClassObject(catalogManager, "CoordinatorDynamicCatalogManager")) { + ConcurrentHashMap activeCatalogs = + (ConcurrentHashMap) getFiledObject(catalogManager, "activeCatalogs"); + Preconditions.checkNotNull(activeCatalogs, "activeCatalogs should not be null"); + + ConcurrentHashMap allCatalogs = + (ConcurrentHashMap) getFiledObject(catalogManager, "allCatalogs"); + Preconditions.checkNotNull(allCatalogs, "allCatalogs should not be null"); + + removeHandle = + (catalogName) -> { + activeCatalogs.remove(catalogName); + allCatalogs.remove(catalogName); + }; + } else { + // The catalogManager is an instance of StaticCatalogManager + ConcurrentHashMap catalogs = (ConcurrentHashMap) getFiledObject(catalogManager, "catalogs"); + Preconditions.checkNotNull(catalogs, "catalogs should not be null"); + removeHandle = (catalogName) -> catalogs.remove(catalogName); + } + } + private void createInjectHandler( Object catalogManager, Object catalogFactory, @@ -237,6 +264,16 @@ private void createInjectHandler( } } + void removeCatalogConnector(String catalogName) { + try { + removeHandle.invoke(catalogName); + LOG.info("Remove trino catalog {} successfully.", catalogName); + } catch (Exception e) { + LOG.error("Remove trino catalog {} failed.", catalogName, e); + throw new TrinoException(GRAVITINO_CREATE_INNER_CONNECTOR_FAILED, e); + } + } + void injectCatalogConnector(String catalogName) { try { String catalogProperties = createCatalogProperties(catalogName); @@ -288,4 +325,8 @@ interface InjectCatalogHandle { interface CreateCatalogHandle { Object invoke(String name, String properties) throws Exception; } + + interface RemoveCatalogHandle { + void invoke(String catalogName) throws Exception; + } } diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java index b75d667c224..10f846a3acc 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/GravitinoMockServer.java @@ -52,7 +52,7 @@ public class GravitinoMockServer implements AutoCloseable { private final String testCatalogProvider = "memory"; private boolean start = true; - private CatalogConnectorManager catalogConnectorManager; + CatalogConnectorManager catalogConnectorManager; private GeneralDataTypeTransformer dataTypeTransformer = new HiveDataTypeTransformer(); public void setCatalogConnectorManager(CatalogConnectorManager catalogConnectorManager) { @@ -100,6 +100,14 @@ public Catalog answer(InvocationOnMock invocation) throws Throwable { return metaLake; } + void reloadCatalogs() { + GravitinoMetaLake metaLake = mock(GravitinoMetaLake.class); + when(metaLake.name()).thenReturn(testMetalake); + when(metaLake.listCatalogs(any())) + .thenReturn(new NameIdentifier[] {NameIdentifier.ofCatalog(testMetalake, testCatalog)}); + catalogConnectorManager.loadCatalogs(metaLake); + } + private Catalog createGravitinoCatalog(NameIdentifier catalogName) { Catalog catalog = mock(Catalog.class); when(catalog.name()).thenReturn(catalogName.name()); diff --git a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java index 3722e83735d..09559233f2c 100644 --- a/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java +++ b/trino-connector/src/test/java/com/datastrato/gravitino/trino/connector/TestGravitinoConnector.java @@ -23,6 +23,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.shaded.com.google.common.base.Preconditions; +import org.testng.annotations.BeforeMethod; import org.testng.annotations.Parameters; import org.testng.annotations.Test; @@ -33,6 +34,11 @@ public class TestGravitinoConnector extends AbstractTestQueryFramework { GravitinoMockServer server; + @BeforeMethod + public void reloadCatalog() { + server.reloadCatalogs(); + } + @Override protected QueryRunner createQueryRunner() throws Exception { server = closeAfterClass(new GravitinoMockServer()); @@ -57,9 +63,11 @@ protected QueryRunner createQueryRunner() throws Exception { int max_tries = 35; while (GravitinoPlugin.catalogConnectorManager.getCatalogs().isEmpty() && max_tries > 0) { Thread.sleep(1000); - if (max_tries-- == 0) { - throw new Exception("Catalog memory load failed"); - } + max_tries--; + } + + if (max_tries == 0) { + throw new RuntimeException("Failed to create catalog in about 35 seconds..."); } } catch (Exception e) { From 9dddd0c26433157ee5f7847b1963185c9100d56f Mon Sep 17 00:00:00 2001 From: mchades Date: Tue, 19 Dec 2023 23:33:52 +0800 Subject: [PATCH 19/51] [#554] improvement(CI-Hive): improve Hadoop access permission (#1194) ### What changes were proposed in this pull request? - set `hdfs` as HDFS superuser group in container - use `datastrato` as the Hive catalog Integration Test user instead of `root` ### Why are the changes needed? we should not use the `root` user directly to access HDFS Fix: #554 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? existing ITs --- dev/docker/build-docker.sh | 2 +- dev/docker/hive/Dockerfile | 6 ++-- dev/docker/hive/hdfs-site.xml | 5 +++ dev/docker/hive/hive-dependency.sh | 8 +++++ dev/docker/hive/start.sh | 10 ------ docs/docker-image-details.md | 6 +++- integration-test/build.gradle.kts | 5 ++- .../test/catalog/hive/CatalogHiveIT.java | 35 ++++++++++++++++++- .../test/container/ContainerSuite.java | 4 ++- 9 files changed, 60 insertions(+), 21 deletions(-) diff --git a/dev/docker/build-docker.sh b/dev/docker/build-docker.sh index 3ee6206f1ad..c2d99ce612a 100755 --- a/dev/docker/build-docker.sh +++ b/dev/docker/build-docker.sh @@ -64,7 +64,7 @@ fi if [[ "${component_type}" == "hive" ]]; then . ${script_dir}/hive/hive-dependency.sh - build_args="--build-arg HADOOP_PACKAGE_NAME=${HADOOP_PACKAGE_NAME} --build-arg HIVE_PACKAGE_NAME=${HIVE_PACKAGE_NAME}" + build_args="--build-arg HADOOP_PACKAGE_NAME=${HADOOP_PACKAGE_NAME} --build-arg HIVE_PACKAGE_NAME=${HIVE_PACKAGE_NAME} --build-arg JDBC_DIVER_PACKAGE_NAME=${JDBC_DIVER_PACKAGE_NAME}" elif [ "${component_type}" == "trino" ]; then . ${script_dir}/trino/trino-dependency.sh elif [ "${component_type}" == "gravitino" ]; then diff --git a/dev/docker/hive/Dockerfile b/dev/docker/hive/Dockerfile index e1ddfbc3bec..d2eded16d7c 100644 --- a/dev/docker/hive/Dockerfile +++ b/dev/docker/hive/Dockerfile @@ -8,6 +8,7 @@ LABEL maintainer="support@datastrato.com" ARG HADOOP_PACKAGE_NAME ARG HIVE_PACKAGE_NAME +ARG JDBC_DIVER_PACKAGE_NAME WORKDIR / @@ -129,10 +130,7 @@ RUN sed -i "s/.*bind-address.*/bind-address = 0.0.0.0/" /etc/mysql/mysql.conf.d/ ################################################################################ # add mysql jdbc driver -RUN wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-8.0.15.tar.gz -RUN tar -xzf mysql-connector-java-8.0.15.tar.gz -RUN cp mysql-connector-java-8.0.15/mysql-connector-java-8.0.15.jar ${HIVE_HOME}/lib -RUN rm -rf mysql-connector-java-8.0.15 mysql-connector-java-8.0.15.tar.gz +RUN tar -xz -C ${HIVE_HOME}/lib --strip-components 1 -f /tmp/packages/${JDBC_DIVER_PACKAGE_NAME} ################################################################################ # add users and groups diff --git a/dev/docker/hive/hdfs-site.xml b/dev/docker/hive/hdfs-site.xml index 0fcae90b05d..ae15b096f4f 100644 --- a/dev/docker/hive/hdfs-site.xml +++ b/dev/docker/hive/hdfs-site.xml @@ -13,4 +13,9 @@ dfs.datanode.address 0.0.0.0:50010 + + + dfs.permissions.superusergroup + hdfs + diff --git a/dev/docker/hive/hive-dependency.sh b/dev/docker/hive/hive-dependency.sh index 7d1341d6205..7397068642c 100755 --- a/dev/docker/hive/hive-dependency.sh +++ b/dev/docker/hive/hive-dependency.sh @@ -10,6 +10,7 @@ hive_dir="$(cd "${hive_dir}">/dev/null; pwd)" # Environment variables definition HADOOP_VERSION="2.7.3" HIVE_VERSION="2.3.9" +MYSQL_JDBC_DRIVER_VERSION="8.0.15" HADOOP_PACKAGE_NAME="hadoop-${HADOOP_VERSION}.tar.gz" # Must export this variable for Dockerfile HADOOP_DOWNLOAD_URL="http://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_PACKAGE_NAME}" @@ -17,6 +18,9 @@ HADOOP_DOWNLOAD_URL="http://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_ HIVE_PACKAGE_NAME="apache-hive-${HIVE_VERSION}-bin.tar.gz" # Must export this variable for Dockerfile HIVE_DOWNLOAD_URL="https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/${HIVE_PACKAGE_NAME}" +JDBC_DIVER_PACKAGE_NAME="mysql-connector-java-${MYSQL_JDBC_DRIVER_VERSION}.tar.gz" # Must export this variable for Dockerfile +JDBC_DIVER_DOWNLOAD_URL="https://downloads.mysql.com/archives/get/p/3/file/${JDBC_DIVER_PACKAGE_NAME}" + # Prepare download packages if [[ ! -d "${hive_dir}/packages" ]]; then mkdir -p "${hive_dir}/packages" @@ -29,3 +33,7 @@ fi if [ ! -f "${hive_dir}/packages/${HIVE_PACKAGE_NAME}" ]; then curl -s -o "${hive_dir}/packages/${HIVE_PACKAGE_NAME}" ${HIVE_DOWNLOAD_URL} fi + +if [ ! -f "${hive_dir}/packages/${JDBC_DIVER_PACKAGE_NAME}" ]; then + curl -L -s -o "${hive_dir}/packages/${JDBC_DIVER_PACKAGE_NAME}" ${JDBC_DIVER_DOWNLOAD_URL} +fi diff --git a/dev/docker/hive/start.sh b/dev/docker/hive/start.sh index b96874e581f..a80e3a45e05 100644 --- a/dev/docker/hive/start.sh +++ b/dev/docker/hive/start.sh @@ -12,16 +12,6 @@ ssh-keyscan 0.0.0.0 >> /root/.ssh/known_hosts # start hdfs ${HADOOP_HOME}/sbin/start-dfs.sh -${HADOOP_HOME}/bin/hdfs dfs -mkdir /tmp -${HADOOP_HOME}/bin/hdfs dfs -chmod 1777 /tmp -${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/hive/warehouse -${HADOOP_HOME}/bin/hdfs dfs -chown -R hive:hive /user/hive -${HADOOP_HOME}/bin/hdfs dfs -chmod -R 775 /user/hive -${HADOOP_HOME}/bin/hdfs dfs -mkdir -p /user/datastrato -${HADOOP_HOME}/bin/hdfs dfs -chown -R datastrato:hdfs /user/datastrato -${HADOOP_HOME}/bin/hdfs dfs -chmod 755 /user/datastrato -${HADOOP_HOME}/bin/hdfs dfs -chmod -R 777 /user/hive/tmp - # start mysql and create databases/users for hive chown -R mysql:mysql /var/lib/mysql usermod -d /var/lib/mysql/ mysql diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md index c12bdd9b865..35487963f9a 100644 --- a/docs/docker-image-details.md +++ b/docs/docker-image-details.md @@ -68,8 +68,12 @@ You can use this kind of images to test the catalog of Apache Hive. Changelog +- gravitino-ci-hive:0.1.7 + - Download MySQL JDBC driver before building the Docker image + - Set `hdfs` as HDFS superuser group + - gravitino-ci-hive:0.1.6 - - No start YARN when container startup + - No starting YARN when container startup - Removed expose ports: - `22` SSH - `8088` YARN Service diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index e4284fad01f..245e862ad23 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -265,14 +265,13 @@ tasks.test { // Default use MiniGravitino to run integration tests environment("GRAVITINO_ROOT_DIR", rootDir.path) - // TODO: use hive user instead after we fix the permission issue #554 - environment("HADOOP_USER_NAME", "root") + environment("HADOOP_USER_NAME", "datastrato") environment("HADOOP_HOME", "/tmp") environment("PROJECT_VERSION", version) environment("TRINO_CONF_DIR", buildDir.path + "/trino-conf") // Gravitino CI Docker image - environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.6") + environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.7") environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.2") val testMode = project.properties["testMode"] as? String ?: "embedded" diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java index e0041d4edbb..62aad9d1798 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java @@ -62,12 +62,17 @@ import com.datastrato.gravitino.rel.types.Types; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Database; @@ -106,6 +111,7 @@ public class CatalogHiveIT extends AbstractIT { private static GravitinoMetaLake metalake; private static Catalog catalog; private static SparkSession sparkSession; + private static FileSystem hdfs; private static final String SELECT_ALL_TEMPLATE = "SELECT * FROM %s.%s"; private static final String INSERT_WITHOUT_PARTITION_TEMPLATE = "INSERT INTO %s.%s VALUES (%s)"; private static final String INSERT_WITH_PARTITION_TEMPLATE = @@ -155,13 +161,23 @@ public static void startup() throws Exception { .config("mapreduce.input.fileinputformat.input.dir.recursive", "true") .enableHiveSupport() .getOrCreate(); + + Configuration conf = new Configuration(); + conf.set( + "fs.defaultFS", + String.format( + "hdfs://%s:%d", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT)); + hdfs = FileSystem.get(conf); + createMetalake(); createCatalog(); createSchema(); } @AfterAll - public static void stop() { + public static void stop() throws IOException { client.dropMetalake(NameIdentifier.of(metalakeName)); if (hiveClientPool != null) { hiveClientPool.close(); @@ -170,6 +186,10 @@ public static void stop() { if (sparkSession != null) { sparkSession.close(); } + + if (hdfs != null) { + hdfs.close(); + } try { closer.close(); } catch (Exception e) { @@ -286,6 +306,19 @@ private void checkTableReadWrite(org.apache.hadoop.hive.metastore.api.Table tabl } Assertions.assertEquals( count + 1, sparkSession.sql(String.format(SELECT_ALL_TEMPLATE, dbName, tableName)).count()); + // Assert HDFS owner + Path tableDirectory = new Path(table.getSd().getLocation()); + FileStatus[] fileStatuses; + try { + fileStatuses = hdfs.listStatus(tableDirectory); + } catch (IOException e) { + LOG.warn("Failed to list status of table directory", e); + throw new RuntimeException(e); + } + Assertions.assertTrue(fileStatuses.length > 0); + for (FileStatus fileStatus : fileStatuses) { + Assertions.assertEquals("datastrato", fileStatus.getOwner()); + } } private Map createProperties() { diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index d69d146f563..7ee9d2a4069 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -68,7 +68,9 @@ public void startHiveContainer() { HiveContainer.builder() .withHostName("gravitino-ci-hive") .withEnvVars( - ImmutableMap.builder().put("HADOOP_USER_NAME", "root").build()) + ImmutableMap.builder() + .put("HADOOP_USER_NAME", "datastrato") + .build()) .withNetwork(network); hiveContainer = closer.register(hiveBuilder.build()); hiveContainer.start(); From 813c94f061dd50c446add58347d5a861c5cd4fee Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 20 Dec 2023 17:30:39 +0800 Subject: [PATCH 20/51] [#1180] feat(CI): Enable CI to test multiple JDK versions (#1205) ### What changes were proposed in this pull request? This PR proposes to support multiple JDKs in github CI. ### Why are the changes needed? Fix: #1180 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. --- .github/workflows/build.yml | 7 +++-- .github/workflows/integration-test.yml | 13 +++++----- bin/common.sh | 1 + bin/gravitino.sh | 26 ++++++++++++++++++- build.gradle.kts | 2 +- gradle.properties | 2 +- integration-test/build.gradle.kts | 2 ++ .../integration/test/MiniGravitino.java | 6 ++--- 8 files changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7f5f2e38057..5d6874dbf45 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,6 +51,9 @@ jobs: build: # The type of runner that the job will run on runs-on: ubuntu-latest + strategy: + matrix: + java-version: [ 8, 11, 17 ] timeout-minutes: 30 needs: changes if: needs.changes.outputs.source_changes == 'true' @@ -61,11 +64,11 @@ jobs: - uses: actions/setup-java@v3 with: - java-version: '8' + java-version: ${{ matrix.java-version }} distribution: 'temurin' - name: Build with Gradle - run: ./gradlew build -PskipITs + run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} - name: Upload unit tests report uses: actions/upload-artifact@v3 diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ea59697c013..51ad704ecaf 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -55,6 +55,8 @@ jobs: strategy: matrix: architecture: [linux/amd64] + java-version: [ 8, 11, 17 ] + test-mode: [ embedded, deploy ] env: DOCKER_RUN_NAME: hive-amd64 PLATFORM: ${{ matrix.architecture }} @@ -63,7 +65,7 @@ jobs: - uses: actions/setup-java@v3 with: - java-version: '8' + java-version: ${{ matrix.java-version }} distribution: 'temurin' - name: Set up QEMU @@ -71,8 +73,8 @@ jobs: - name: Package Gravitino run: | - ./gradlew build -x test - ./gradlew compileDistribution -x test + ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }} + ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - name: Setup debug Github Action if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} @@ -81,14 +83,13 @@ jobs: - name: Integration Test id: integrationTest run: | - ./gradlew test --rerun-tasks -PskipTests -PtestMode=embedded - ./gradlew test --rerun-tasks -PskipTests -PtestMode=deploy + ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} - name: Upload integrate tests reports uses: actions/upload-artifact@v3 if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} with: - name: integrate test reports + name: integrate test reports path: | build/reports integration-test/build/integration-test.log diff --git a/bin/common.sh b/bin/common.sh index 074a98eb46b..6f9f37eb7c1 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -44,6 +44,7 @@ fi GRAVITINO_CLASSPATH+=":${GRAVITINO_CONF_DIR}" +JVM_VERSION=8 function check_java_version() { if [[ -n "${JAVA_HOME+x}" ]]; then JAVA="$JAVA_HOME/bin/java" diff --git a/bin/gravitino.sh b/bin/gravitino.sh index 651436d454f..802bd589e2e 100755 --- a/bin/gravitino.sh +++ b/bin/gravitino.sh @@ -130,6 +130,30 @@ GRAVITINO_SERVER_NAME=com.datastrato.gravitino.server.GravitinoServer JAVA_OPTS+=" -Dfile.encoding=UTF-8" JAVA_OPTS+=" -Dlog4j2.configurationFile=file://${GRAVITINO_CONF_DIR}/log4j2.properties" JAVA_OPTS+=" -Dgravitino.log.path=${GRAVITINO_LOG_DIR} ${GRAVITINO_MEM}" +if [ "$JVM_VERSION" -eq 17 ]; then + JAVA_OPTS+=" -XX:+IgnoreUnrecognizedVMOptions" + JAVA_OPTS+=" --add-opens java.base/java.io=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.lang.invoke=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.lang.reflect=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.lang=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.math=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.net=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.nio=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.text=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.time=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.util.concurrent=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.util.regex=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/java.util=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/jdk.internal.ref=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.sql/java.sql=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/sun.util.calendar=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/sun.nio.ch=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/sun.nio.cs=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/sun.security.action=ALL-UNNAMED" + JAVA_OPTS+=" --add-opens java.base/sun.util.calendar=ALL-UNNAMED" +fi addJarInDir "${GRAVITINO_HOME}/libs" @@ -149,4 +173,4 @@ case "${1}" in ;; *) echo ${USAGE} -esac \ No newline at end of file +esac diff --git a/build.gradle.kts b/build.gradle.kts index ab9c74558a3..9248002692d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -246,7 +246,7 @@ subprojects { sign(publishing.publications) } - tasks.withType { + tasks.configureEach { testLogging { exceptionFormat = TestExceptionFormat.FULL showExceptions = true diff --git a/gradle.properties b/gradle.properties index 0dbdc6de0a2..a342ae14d79 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,6 +15,6 @@ version = 0.4.0-SNAPSHOT SONATYPE_USER = admin SONATYPE_PASSWORD = password -#jdkVersion is used to specify the version of JDK to build and test against Gravitino, current +#jdkVersion is used to specify the version of JDK to build and test Gravitino, current # supported version is 8, 11, and 17. jdkVersion = 8 diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 245e862ad23..3b6a6e9357c 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -263,6 +263,8 @@ tasks.test { fileMode = 0b111101101 } + jvmArgs(project.property("extraJvmArgs") as List<*>) + // Default use MiniGravitino to run integration tests environment("GRAVITINO_ROOT_DIR", rootDir.path) environment("HADOOP_USER_NAME", "datastrato") diff --git a/integration-test/src/main/java/com/datastrato/gravitino/integration/test/MiniGravitino.java b/integration-test/src/main/java/com/datastrato/gravitino/integration/test/MiniGravitino.java index 26f27035407..3f2eafcdc14 100644 --- a/integration-test/src/main/java/com/datastrato/gravitino/integration/test/MiniGravitino.java +++ b/integration-test/src/main/java/com/datastrato/gravitino/integration/test/MiniGravitino.java @@ -220,14 +220,14 @@ private boolean checkIfServerIsRunning() { Collections.emptyMap(), ErrorHandlers.restErrorHandler()); } catch (RESTException e) { - LOG.warn("checkIfServerIsRunning() fails, GravitinoServer is not running"); + LOG.warn("checkIfServerIsRunning() fails, GravitinoServer is not running", e); + return false; } if (response != null && response.getCode() == 0) { return true; } else { LOG.warn("checkIfServerIsRunning() fails, GravitinoServer is not running"); + return false; } - - return false; } } From fbce0fe98190de2a73df2e8a2065bbd3af77a4c8 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Wed, 20 Dec 2023 07:35:52 -0600 Subject: [PATCH 21/51] [#950] improvment(common): Add a method in ConfigEntry called create to support configurations with no default value (#1141) ### What changes were proposed in this pull request? Some configuration options are necessary but don't have default value. ### Why are the changes needed? Fix: #950 ### Does this PR introduce _any_ user-facing change? Change some configuration options from `` to none ### How was this patch tested? CI passed. --------- Co-authored-by: Heng Qin --- .../catalog/jdbc/config/JdbcConfig.java | 29 +++++++++---------- .../catalog/jdbc/utils/DataSourceUtils.java | 21 +++++++++++--- .../operation/TestJdbcDatabaseOperations.java | 1 + .../operation/TestJdbcTableOperations.java | 1 + .../jdbc/utils/TestDataSourceUtils.java | 1 + .../operation/PostgreSqlSchemaOperations.java | 9 +++--- .../operation/PostgreSqlTableOperations.java | 9 +++--- .../lakehouse/iceberg/IcebergConfig.java | 15 +++++----- .../iceberg/ops/IcebergTableOps.java | 13 +++++++-- .../iceberg/utils/IcebergCatalogUtil.java | 28 +++++++++++------- .../iceberg/utils/TestIcebergCatalogUtil.java | 5 +++- .../gravitino/config/ConfigEntry.java | 22 ++++++++++++++ .../test/catalog/jdbc/TestJdbcAbstractIT.java | 1 + .../catalog/jdbc/mysql/CatalogMysqlIT.java | 1 + .../jdbc/postgresql/CatalogPostgreSqlIT.java | 2 ++ .../TestPostgreSqlSchemaOperations.java | 1 + .../TestPostgreSqlTableOperations.java | 1 + .../iceberg/IcebergRESTJdbcCatalogIT.java | 7 +++++ .../test/trino/TrinoConnectorIT.java | 3 ++ .../gravitino/server/auth/OAuthConfig.java | 6 ++-- .../server/web/JettyServerConfig.java | 10 +++---- 21 files changed, 129 insertions(+), 57 deletions(-) diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java index 71e8cf07783..41dba89aad5 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.config.ConfigBuilder; import com.datastrato.gravitino.config.ConfigEntry; import java.util.Map; -import java.util.Optional; public class JdbcConfig extends Config { @@ -18,35 +17,35 @@ public class JdbcConfig extends Config { .doc("The url of the Jdbc connection") .version("0.3.0") .stringConf() - .createWithDefault(null); + .create(); - public static final ConfigEntry> JDBC_DATABASE = + public static final ConfigEntry JDBC_DATABASE = new ConfigBuilder("jdbc-database") .doc("The database of the jdbc connection") .version("0.3.0") .stringConf() - .createWithOptional(); + .create(); - public static final ConfigEntry> JDBC_DRIVER = + public static final ConfigEntry JDBC_DRIVER = new ConfigBuilder("jdbc-driver") .doc("The driver of the jdbc connection") .version("0.3.0") .stringConf() - .createWithOptional(); + .create(); - public static final ConfigEntry> USERNAME = + public static final ConfigEntry USERNAME = new ConfigBuilder("jdbc-user") .doc("The username of the Jdbc connection") .version("0.3.0") .stringConf() - .createWithOptional(); + .create(); - public static final ConfigEntry> PASSWORD = + public static final ConfigEntry PASSWORD = new ConfigBuilder("jdbc-password") .doc("The password of the Jdbc connection") .version("0.3.0") .stringConf() - .createWithOptional(); + .create(); public static final ConfigEntry POOL_MIN_SIZE = new ConfigBuilder("jdbc.pool.min-size") @@ -66,15 +65,15 @@ public String getJdbcUrl() { return get(JDBC_URL); } - public Optional getJdbcDriverOptional() { + public String getJdbcDriver() { return get(JDBC_DRIVER); } - public Optional getUsernameOptional() { + public String getUsername() { return get(USERNAME); } - public Optional getPasswordOptional() { + public String getPassword() { return get(PASSWORD); } @@ -86,8 +85,8 @@ public int getPoolMaxSize() { return get(POOL_MAX_SIZE); } - public String getJdbcDatabaseOrElseThrow(String errorMessage) { - return get(JDBC_DATABASE).orElseThrow(() -> new IllegalArgumentException(errorMessage)); + public String getJdbcDatabase() { + return get(JDBC_DATABASE); } public JdbcConfig(Map properties) { diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java index fbab7db0058..f63a89d892d 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java @@ -6,12 +6,14 @@ import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; +import com.google.common.base.Preconditions; import java.sql.SQLException; import java.util.Map; import java.util.Properties; import javax.sql.DataSource; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.BasicDataSourceFactory; +import org.apache.commons.lang3.StringUtils; /** * Utility class for creating a {@link DataSource} from a {@link JdbcConfig}. It is mainly @@ -39,10 +41,21 @@ public static DataSource createDataSource(JdbcConfig jdbcConfig) private static DataSource createDBCPDataSource(JdbcConfig jdbcConfig) throws Exception { BasicDataSource basicDataSource = BasicDataSourceFactory.createDataSource(getProperties(jdbcConfig)); - basicDataSource.setUrl(jdbcConfig.getJdbcUrl()); - jdbcConfig.getJdbcDriverOptional().ifPresent(basicDataSource::setDriverClassName); - jdbcConfig.getUsernameOptional().ifPresent(basicDataSource::setUsername); - jdbcConfig.getPasswordOptional().ifPresent(basicDataSource::setPassword); + String jdbcUrl = jdbcConfig.getJdbcUrl(); + Preconditions.checkArgument(StringUtils.isNotBlank(jdbcUrl), "The jdbc url can't be blank."); + basicDataSource.setUrl(jdbcUrl); + String driverClassName = jdbcConfig.getJdbcDriver(); + Preconditions.checkArgument( + StringUtils.isNotBlank(driverClassName), "The jdbc driver can't be blank."); + basicDataSource.setDriverClassName(driverClassName); + String userName = jdbcConfig.getUsername(); + Preconditions.checkArgument( + StringUtils.isNotBlank(userName), "The jdbc user name can't be blank."); + basicDataSource.setUsername(userName); + String password = jdbcConfig.getPassword(); + Preconditions.checkArgument( + StringUtils.isNotBlank(password), "The jdbc password can't be blank."); + basicDataSource.setPassword(password); basicDataSource.setMaxTotal(jdbcConfig.getPoolMaxSize()); basicDataSource.setMinIdle(jdbcConfig.getPoolMinSize()); // Set each time a connection is taken out from the connection pool, a test statement will be diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java index 7005f64d1be..2225548d6a1 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcDatabaseOperations.java @@ -58,6 +58,7 @@ private static void createExceptionMapper() { private static void createDataSource() { HashMap properties = Maps.newHashMap(); + properties.put(JdbcConfig.JDBC_DRIVER.getKey(), "org.sqlite.JDBC"); properties.put(JdbcConfig.JDBC_URL.getKey(), JDBC_URL); properties.put(JdbcConfig.USERNAME.getKey(), "test"); properties.put(JdbcConfig.PASSWORD.getKey(), "test"); diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java index b46ddb1690c..feea5863503 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/operation/TestJdbcTableOperations.java @@ -79,6 +79,7 @@ private static void createExceptionConverter() { private static void createDataSource() { HashMap properties = Maps.newHashMap(); + properties.put(JdbcConfig.JDBC_DRIVER.getKey(), "org.sqlite.JDBC"); properties.put(JdbcConfig.JDBC_URL.getKey(), JDBC_URL); properties.put(JdbcConfig.USERNAME.getKey(), "test"); properties.put(JdbcConfig.PASSWORD.getKey(), "test"); diff --git a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java index 8e91f98c0c3..a1946aff09f 100644 --- a/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java +++ b/catalogs/catalog-jdbc-common/src/test/java/com/datastrato/gravitino/catalog/jdbc/utils/TestDataSourceUtils.java @@ -18,6 +18,7 @@ public class TestDataSourceUtils { @Test public void testCreateDataSource() throws SQLException { HashMap properties = Maps.newHashMap(); + properties.put(JdbcConfig.JDBC_DRIVER.getKey(), "org.sqlite.JDBC"); properties.put(JdbcConfig.JDBC_URL.getKey(), "jdbc:sqlite::memory:"); properties.put(JdbcConfig.USERNAME.getKey(), "test"); properties.put(JdbcConfig.PASSWORD.getKey(), "test"); diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java index cd69536e710..f33c1f12ab1 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java @@ -10,6 +10,7 @@ import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.meta.AuditInfo; +import com.google.common.base.Preconditions; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -31,10 +32,10 @@ public class PostgreSqlSchemaOperations extends JdbcDatabaseOperations { public void initialize( DataSource dataSource, JdbcExceptionConverter exceptionMapper, Map conf) { super.initialize(dataSource, exceptionMapper, conf); - database = - new JdbcConfig(conf) - .getJdbcDatabaseOrElseThrow( - "The `jdbc-database` configuration item is mandatory in PostgreSQL."); + database = new JdbcConfig(conf).getJdbcDatabase(); + Preconditions.checkArgument( + StringUtils.isNotBlank(database), + "The `jdbc-database` configuration item is mandatory in PostgreSQL."); } @Override diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index ce471880a9e..a89b83308c1 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -16,6 +16,7 @@ import com.datastrato.gravitino.meta.AuditInfo; import com.datastrato.gravitino.rel.TableChange; import com.datastrato.gravitino.rel.expressions.transforms.Transform; +import com.google.common.base.Preconditions; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -70,10 +71,10 @@ public void initialize( JdbcTypeConverter jdbcTypeConverter, Map conf) { super.initialize(dataSource, exceptionMapper, jdbcTypeConverter, conf); - database = - new JdbcConfig(conf) - .getJdbcDatabaseOrElseThrow( - "The `jdbc-database` configuration item is mandatory in PostgreSQL."); + database = new JdbcConfig(conf).getJdbcDatabase(); + Preconditions.checkArgument( + StringUtils.isNotBlank(database), + "The `jdbc-database` configuration item is mandatory in PostgreSQL."); } @Override diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java index abba7a63664..86f7c2328a4 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java @@ -17,7 +17,6 @@ import com.datastrato.gravitino.config.ConfigBuilder; import com.datastrato.gravitino.config.ConfigEntry; import java.util.Map; -import java.util.Optional; public class IcebergConfig extends Config { @@ -33,35 +32,35 @@ public class IcebergConfig extends Config { .doc("Warehouse directory of catalog") .version("0.2.0") .stringConf() - .createWithDefault(null); + .create(); public static final ConfigEntry CATALOG_URI = new ConfigBuilder(URI) .doc("The uri config of the Iceberg catalog") .version("0.2.0") .stringConf() - .createWithDefault(null); + .create(); public static final ConfigEntry JDBC_USER = new ConfigBuilder(ICEBERG_JDBC_USER) .doc("The username of the Jdbc connection") .version("0.2.0") .stringConf() - .createWithDefault(null); + .create(); public static final ConfigEntry JDBC_PASSWORD = new ConfigBuilder(ICEBERG_JDBC_PASSWORD) .doc("The password of the Jdbc connection") .version("0.2.0") .stringConf() - .createWithDefault(null); + .create(); - public static final ConfigEntry> JDBC_DRIVER = + public static final ConfigEntry JDBC_DRIVER = new ConfigBuilder(GRAVITINO_JDBC_DRIVER) .doc("The driver of the Jdbc connection") .version("0.3.0") .stringConf() - .createWithOptional(); + .create(); public static final ConfigEntry JDBC_INIT_TABLES = new ConfigBuilder(ICEBERG_JDBC_INITIALIZE) @@ -70,7 +69,7 @@ public class IcebergConfig extends Config { .booleanConf() .createWithDefault(true); - public Optional getJdbcDriverOptional() { + public String getJdbcDriver() { return get(JDBC_DRIVER); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java index 7d015cc574c..f910cb42216 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java @@ -4,6 +4,7 @@ */ package com.datastrato.gravitino.catalog.lakehouse.iceberg.ops; +import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper.IcebergTableChange; import com.datastrato.gravitino.catalog.lakehouse.iceberg.utils.IcebergCatalogUtil; @@ -11,6 +12,7 @@ import java.util.Collections; import java.util.Optional; import javax.ws.rs.NotSupportedException; +import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -40,8 +42,15 @@ public class IcebergTableOps implements AutoCloseable { public IcebergTableOps(IcebergConfig icebergConfig) { String catalogType = icebergConfig.get(IcebergConfig.CATALOG_BACKEND); - catalog = - IcebergCatalogUtil.loadCatalogBackend(catalogType, icebergConfig.getConfigsWithPrefix("")); + if (!IcebergCatalogBackend.MEMORY.name().equalsIgnoreCase(catalogType)) { + Preconditions.checkArgument( + StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)), + "Catalog warehouse can't be blank"); + Preconditions.checkArgument( + StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.CATALOG_URI)), + "Catalog uri can't be blank"); + } + catalog = IcebergCatalogUtil.loadCatalogBackend(catalogType, icebergConfig.getAllConfig()); if (catalog instanceof SupportsNamespaces) { asNamespaceCatalog = (SupportsNamespaces) catalog; } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java index 67adc8828a3..8df85dce349 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java @@ -8,9 +8,11 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; +import com.google.common.base.Preconditions; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; @@ -43,17 +45,21 @@ private static HiveCatalog loadHiveCatalog(Map properties) { private static JdbcCatalog loadJdbcCatalog(Map properties) { IcebergConfig icebergConfig = new IcebergConfig(properties); - icebergConfig - .getJdbcDriverOptional() - .ifPresent( - driverClassName -> { - try { - // Load the jdbc driver - Class.forName(driverClassName); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Couldn't load jdbc driver " + driverClassName); - } - }); + String driverClassName = icebergConfig.getJdbcDriver(); + Preconditions.checkArgument( + StringUtils.isNotBlank(driverClassName), "Jdbc driver can't be blank"); + Preconditions.checkArgument( + StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.JDBC_USER)), + "Jdbc user can't be blank"); + Preconditions.checkArgument( + StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.JDBC_PASSWORD)), + "Jdbc password can't be blank"); + try { + // Load the jdbc driver + Class.forName(driverClassName); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Couldn't load jdbc driver " + driverClassName); + } JdbcCatalog jdbcCatalog = new JdbcCatalog( null, diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java index 601aafed9bc..70bb1c70075 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/TestIcebergCatalogUtil.java @@ -40,7 +40,7 @@ void testLoadCatalog() { Assertions.assertTrue(catalog instanceof HiveCatalog); Assertions.assertThrowsExactly( - NullPointerException.class, + IllegalArgumentException.class, () -> { IcebergCatalogUtil.loadCatalogBackend("jdbc"); }); @@ -48,6 +48,9 @@ void testLoadCatalog() { Map properties = new HashMap<>(); properties.put(CatalogProperties.URI, "jdbc://0.0.0.0:3306"); properties.put(CatalogProperties.WAREHOUSE_LOCATION, "test"); + properties.put(IcebergCatalogPropertiesMetadata.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC"); + properties.put(IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_USER, "test"); + properties.put(IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_PASSWORD, "test"); properties.put(IcebergCatalogPropertiesMetadata.ICEBERG_JDBC_INITIALIZE, "false"); catalog = IcebergCatalogUtil.loadCatalogBackend("jdbc", properties); Assertions.assertTrue(catalog instanceof JdbcCatalog); diff --git a/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java b/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java index ee9e9a5c767..a381117fd19 100644 --- a/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java +++ b/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java @@ -37,6 +37,7 @@ public class ConfigEntry { @Getter private boolean isDeprecated; private boolean isOptional; + private boolean hasNoDefault; /** * Creates a new ConfigEntry instance. @@ -101,6 +102,11 @@ void setOptional() { this.isOptional = true; } + /** Marks this configuration as no default value. */ + void setHasNoDefault() { + this.hasNoDefault = true; + } + /** * Creates a new ConfigEntry instance based on this configuration entry with a default value. * @@ -134,6 +140,20 @@ public ConfigEntry> createWithOptional() { return conf; } + /** + * Creates a new ConfigEntry instance based on this configuration entry with no default value. + * + * @return A new ConfigEntry instance with no default value. + */ + public ConfigEntry create() { + ConfigEntry conf = + new ConfigEntry<>(key, version, doc, alternatives, isPublic, isDeprecated); + conf.setValueConverter(valueConverter); + conf.setStringConverter(stringConverter); + conf.setHasNoDefault(); + return conf; + } + /** * Reads the configuration value. * @@ -155,6 +175,8 @@ public T readFrom(Map properties) throws NoSuchElementException if (value == null) { if (defaultValue != null) { return defaultValue; + } else if (hasNoDefault) { + return null; } else if (!isOptional) { throw new NoSuchElementException("No configuration found for key " + key); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java index 6bf578bf24a..4fb9961364d 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java @@ -39,6 +39,7 @@ public abstract class TestJdbcAbstractIT { public static void startup() { CONTAINER.start(); HashMap properties = Maps.newHashMap(); + properties.put(JdbcConfig.JDBC_DRIVER.getKey(), CONTAINER.getDriverClassName()); properties.put(JdbcConfig.JDBC_URL.getKey(), CONTAINER.getJdbcUrl()); properties.put(JdbcConfig.USERNAME.getKey(), CONTAINER.getUsername()); properties.put(JdbcConfig.PASSWORD.getKey(), CONTAINER.getPassword()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index 597dd12bf7f..a59dc2db8cc 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -144,6 +144,7 @@ private static void createCatalog() { JdbcConfig.JDBC_URL.getKey(), StringUtils.substring( MYSQL_CONTAINER.getJdbcUrl(), 0, MYSQL_CONTAINER.getJdbcUrl().lastIndexOf("/"))); + catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), MYSQL_CONTAINER.getDriverClassName()); catalogProperties.put(JdbcConfig.USERNAME.getKey(), MYSQL_CONTAINER.getUsername()); catalogProperties.put(JdbcConfig.PASSWORD.getKey(), MYSQL_CONTAINER.getPassword()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index e9288e7028f..f3733445957 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -140,6 +140,8 @@ private static void createCatalog() { try { String jdbcUrl = POSTGRESQL_CONTAINER.getJdbcUrl(); String database = new URI(jdbcUrl.substring(jdbcUrl.lastIndexOf("/") + 1)).getPath(); + catalogProperties.put( + JdbcConfig.JDBC_DRIVER.getKey(), POSTGRESQL_CONTAINER.getDriverClassName()); catalogProperties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl); catalogProperties.put(JdbcConfig.JDBC_DATABASE.getKey(), database); catalogProperties.put(JdbcConfig.USERNAME.getKey(), POSTGRESQL_CONTAINER.getUsername()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlSchemaOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlSchemaOperations.java index 79fac4f5b4d..c82f0e5f54f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlSchemaOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlSchemaOperations.java @@ -50,6 +50,7 @@ public void testCreateMultipleSchema() throws SQLException { JdbcConnectorUtils.executeUpdate(connection, "CREATE DATABASE " + testDbName); } HashMap properties = Maps.newHashMap(); + properties.put(JdbcConfig.JDBC_DRIVER.getKey(), CONTAINER.getDriverClassName()); String jdbcUrl = StringUtils.substring(CONTAINER.getJdbcUrl(), 0, CONTAINER.getJdbcUrl().lastIndexOf("/")); properties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl + "/" + testDbName); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java index afa96964797..3451837f75a 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java @@ -310,6 +310,7 @@ public void testCreateMultipleTable() throws SQLException { JdbcConnectorUtils.executeUpdate(connection, "CREATE DATABASE " + testDbName); } HashMap properties = Maps.newHashMap(); + properties.put(JdbcConfig.JDBC_DRIVER.getKey(), CONTAINER.getDriverClassName()); String jdbcUrl = StringUtils.substring(CONTAINER.getJdbcUrl(), 0, CONTAINER.getJdbcUrl().lastIndexOf("/")); properties.put(JdbcConfig.JDBC_URL.getKey(), jdbcUrl + "/" + testDbName); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java index 36088027d5a..1cdfbfc6e6f 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/IcebergRESTJdbcCatalogIT.java @@ -42,6 +42,13 @@ public Map getCatalogConfig() { + IcebergConfig.CATALOG_BACKEND.getKey(), IcebergCatalogBackend.JDBC.toString().toLowerCase()); + configMap.put( + AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + + IcebergRESTService.SERVICE_NAME + + "." + + IcebergConfig.JDBC_DRIVER.getKey(), + "org.sqlite.JDBC"); + configMap.put( AuxiliaryServiceManager.GRAVITINO_AUX_SERVICE_PREFIX + IcebergRESTService.SERVICE_NAME diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java index b8255d61202..1edca4109ed 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/trino/TrinoConnectorIT.java @@ -867,6 +867,7 @@ void testMySQLCatalogCreatedByGravitino() throws InterruptedException { "jdbc-mysql", "comment", ImmutableMap.builder() + .put("jdbc-driver", "com.mysql.cj.jdbc.Driver") .put("jdbc-user", "root") .put("jdbc-password", "ds123") .put("jdbc-url", String.format("jdbc:mysql://%s:3306?useSSL=false", hiveHost)) @@ -909,6 +910,7 @@ void testMySQLTableCreatedByGravitino() throws InterruptedException { "jdbc-mysql", "comment", ImmutableMap.builder() + .put("jdbc-driver", "com.mysql.cj.jdbc.Driver") .put("jdbc-user", "root") .put("jdbc-password", "ds123") .put("jdbc-url", String.format("jdbc:mysql://%s:3306?useSSL=false", hiveHost)) @@ -986,6 +988,7 @@ void testDropCatalogAndCreateAgain() throws InterruptedException { "jdbc-mysql", "comment", ImmutableMap.builder() + .put("jdbc-driver", "com.mysql.cj.jdbc.Driver") .put("jdbc-user", "root") .put("jdbc-password", "ds123") .put("jdbc-url", String.format("jdbc:mysql://%s:3306?useSSL=false", hiveHost)) diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java index 73b9fba1c31..6adfe4b880d 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java @@ -33,7 +33,7 @@ public interface OAuthConfig extends Configs { .doc("The signing key of JWT when Gravitino uses OAuth as the authenticator") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); ConfigEntry SIGNATURE_ALGORITHM_TYPE = new ConfigBuilder(OAUTH_CONFIG_PREFIX + "signAlgorithmType") @@ -47,12 +47,12 @@ public interface OAuthConfig extends Configs { .doc("The uri of the default OAuth server") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); ConfigEntry DEFAULT_TOKEN_PATH = new ConfigBuilder(OAUTH_CONFIG_PREFIX + "tokenPath") .doc("The path for token of the default OAuth server") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java index 90c68b234e1..b27702c6a65 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java @@ -107,21 +107,21 @@ public final class JettyServerConfig { .doc("Path to the key store file") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); public static final ConfigEntry SSL_KEYSTORE_PASSWORD = new ConfigBuilder("keyStorePassword") .doc("Password to the key store") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); public static final ConfigEntry SSL_MANAGER_PASSWORD = new ConfigBuilder("managerPassword") .doc("Manager password to the key store") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); public static final ConfigEntry SSL_KEYSTORE_TYPE = new ConfigBuilder("keyStoreType") @@ -155,14 +155,14 @@ public final class JettyServerConfig { .doc("Path to the trust store file") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); public static final ConfigEntry SSL_TRUST_STORE_PASSWORD = new ConfigBuilder("trustStorePassword") .doc("Password to the trust store") .version("0.3.0") .stringConf() - .createWithDefault(""); + .create(); public static final ConfigEntry SSL_TRUST_STORE_TYPE = new ConfigBuilder("trustStoreType") From 3175f675ebe84e555ddc44abe84be0ad1a3f51d8 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Wed, 20 Dec 2023 08:36:48 -0600 Subject: [PATCH 22/51] [MINOR] fix(docs): Modify the required value for the document `security.md` (#1214) ### What changes were proposed in this pull request? Modify the required value for the document `security.md`. I modify some document mistakes by the way. ### Why are the changes needed? According the review https://github.com/datastrato/gravitino/pull/1191#discussion_r1431374330 , modify the required value. If a config option has a default value, this config option is not required. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No need. --------- Co-authored-by: Heng Qin --- docs/security.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/security.md b/docs/security.md index 69df5475dc8..3c8b095ff0b 100644 --- a/docs/security.md +++ b/docs/security.md @@ -55,11 +55,11 @@ GravitinoClient client = GravitinoClient.builder(uri) | Configuration item | Description | Default value | Required | Since version | |---------------------------------------------------|-----------------------------------------------------------------------------|-------------------|-----------------------------------------|---------------| -| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple` or `oauth`. | `simple` | Yes | 0.3.0 | -| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator` | The authenticator which Gravitino uses, setting as `simple` or `oauth`. | `simple` | No | 0.3.0 | +| `gravitino.authenticator.oauth.serviceAudience` | The audience name when Gravitino uses OAuth as the authenticator. | `GravitinoServer` | No | 0.3.0 | +| `gravitino.authenticator.oauth.allowSkewSecs` | The JWT allows skew seconds when Gravitino uses OAuth as the authenticator. | `0` | No | 0.3.0 | | `gravitino.authenticator.oauth.defaultSignKey` | The signing key of JWT when Gravitino uses OAuth as the authenticator. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | -| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | Yes if use `oauth` as the authenticator | 0.3.0 | +| `gravitino.authenticator.oauth.signAlgorithmType` | The signature algorithm when Gravitino uses OAuth as the authenticator. | `RS256` | No | 0.3.0 | | `gravitino.authenticator.oauth.serverUri` | The URI of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | | `gravitino.authenticator.oauth.tokenPath` | The path for token of the default OAuth server. | (none) | Yes if use `oauth` as the authenticator | 0.3.0 | @@ -152,35 +152,35 @@ Both Gravitino server and Iceberg REST service can configure HTTPS. | Configuration item | Description | Default value | Required | Since version | |-----------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | Yes | 0.3.0 | -| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | +| `gravitino.server.webserver.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | | `gravitino.server.webserver.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.server.webserver.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | +| `gravitino.server.webserver.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | | `gravitino.server.webserver.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | none | No | 0.3.0 | -| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | none | Yes if use HTTPS | 0.3.0 | -| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | none | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | +| `gravitino.server.webserver.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | +| `gravitino.server.webserver.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | +| `gravitino.server.webserver.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.server.webserver.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | ### Iceberg REST service's configuration | Configuration item | Description | Default value | Required | Since version | |------------------------------------------------------------|--------------------------------------------------------------------|---------------|---------------------------------------------------|---------------| -| `gravitino.auxService.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | Yes | 0.3.0 | -| `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.enableHttps` | Enables HTTPS. | `false` | No | 0.3.0 | +| `gravitino.auxService.iceberg-rest.httpsPort` | The HTTPS port number of the Jetty web server. | `8433` | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePath` | Path to the key store file. | (none) | Yes if use HTTPS | 0.3.0 | | `gravitino.auxService.iceberg-rest.keyStorePassword` | Password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.keyStoreType` | The type to the key store. | `JKS` | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.managerPassword` | Manager password to the key store. | (none) | Yes if use HTTPS | 0.3.0 | -| `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.tlsProtocol` | TLS protocol to use. The JVM must support the TLS protocol to use. | (none) | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.enableCipherAlgorithms` | The collection of enabled cipher algorithms. | `` | No | 0.3.0 | -| `gravitino.auxService.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | Yes if use HTTPS | 0.3.0 | +| `gravitino.auxService.iceberg-rest.enableClientAuth` | Enables the authentication of the client. | `false` | No | 0.3.0 | | `gravitino.auxService.iceberg-rest.trustStorePath` | Path to the trust store file. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | | `gravitino.auxService.iceberg-rest.trustStorePassword` | Password to the trust store. | (none) | Yes if use HTTPS and the authentication of client | 0.3.0 | -| `gravitino.auxService.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | Yes if use HTTPS and the authentication of client | 0.3.0 | +| `gravitino.auxService.iceberg-rest.trustStoreType` | The type to the trust store. | `JKS` | No | 0.3.0 | Refer to the "Additional JSSE Standard Names" section of the [Java security guide](https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#jssenames) for the list of protocols related to tlsProtocol. You can find the list of `tlsProtocol` values for Java 8 in this document. From 0cbf4fe368b6a9955d233ea311b6d22c7365658c Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Thu, 21 Dec 2023 14:53:29 +0800 Subject: [PATCH 23/51] [#1181] feat(doc): Improve the doc to support multiple jdk version (#1219) ### What changes were proposed in this pull request? This is a follow-up work to change the docs to support multiple versions of JDK. ### Why are the changes needed? This is a subtask of supporting multiple JDKs for Gravitino. Fix: #1181 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? N/A --- docs/getting-started.md | 19 +++++++++++-------- docs/how-to-build.md | 23 +++++++++++++++++++++-- docs/how-to-install.md | 5 +++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index 37c0d55d93e..d147f8f6191 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -45,10 +45,11 @@ To begin using Gravitino on AWS, follow these steps: You may need to reboot the instance for all changes to take effect. -4. Install the required Java Development Kit for Gravitino: +4. Install the required Java Development Kit for Gravitino, Gravitino supports running on Java 8, + 11 and 17, you can install any of them: ```shell - sudo apt install openjdk-8-jdk-headless + sudo apt install openjdk--jdk-headless ``` Verify the Java version with: @@ -57,7 +58,7 @@ To begin using Gravitino on AWS, follow these steps: java -version ``` - You should see information about OpenJDK 8. + You should see information about OpenJDK version. 5. Install Gravitino on the instance: @@ -90,12 +91,13 @@ To begin using Gravitino on GCP, follow these steps: You may need to reboot the instance for all changes to take effect. -4. Install the required Java Development Kit for Gravitino: +4. Install the required Java Development Kit for Gravitino, Gravitino supports running on Java 8, + 11 and 17, you can install any of them: ```shell wget -O - https://apt.corretto.aws/corretto.key | sudo gpg --dearmor -o /usr/share/keyrings/corretto-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/corretto-keyring.gpg] https://apt.corretto.aws stable main" | sudo tee /etc/apt/sources.list.d/corretto.list sudo apt-get update - sudo apt-get install -y java-1.8.0-amazon-corretto-jdk + sudo apt-get install -y java--amazon-corretto-jdk ``` Verify the Java version with: @@ -104,7 +106,7 @@ To begin using Gravitino on GCP, follow these steps: java -version ``` - You should see information about OpenJDK 8. + You should see information about OpenJDK version. 5. Install Gravitino on the instance: @@ -124,10 +126,11 @@ To begin using Gravitino on GCP, follow these steps: To use Gravitino locally on macOS or Linux, follow similar steps: -1. Install the required Java Development Kit using [sdkman](https://sdkman.io/) for example: +1. Install the required Java Development Kit, Gravitino supports running on Java 8, + 11 and 17, you can install any of them. Using [sdkman](https://sdkman.io/) for example: ```shell - sdk install java 8.0.392-amzn + sdk install java ``` You can also use different package managers to install JDK, for example, diff --git a/docs/how-to-build.md b/docs/how-to-build.md index aa742cc9b45..8b685360092 100644 --- a/docs/how-to-build.md +++ b/docs/how-to-build.md @@ -16,8 +16,9 @@ This software is licensed under the Apache License version 2." + Gravitino requires at least JDK8 and at most JDK17 to run Gradle, so you need to install JDK8 to 17 version to launch the build environment. -+ Gravitino itself uses JDK8 to build, Gravitino Trino connector uses JDK17 to build. You don't - have to preinstall JDK8 or JDK17, Gradle detects the JDK version needed and downloads it automatically. ++ Gravitino itself supports using JDK8, 11, and 17 to build, Gravitino Trino connector uses + JDK17 to build. You don't have to preinstall the specified JDK environment, ++ Gradle detects the JDK version needed and downloads it automatically. + Gravitino uses Gradle Java Toolchain to detect and manage JDK versions, it checks the installed JDK by running `./gradlew javaToolchains` command. For the details of Gradle Java @@ -50,9 +51,27 @@ This software is licensed under the Apache License version 2." ./gradlew build ``` + The default specified JDK version is 8, if you want to use JDK 11 or 17 to build, you can + modify the property `jdkVersion` to 11 or 17 in `gradle.properties` file, or you can specify + with `-P`, like: + + ```shell + ./gradlew build -PjdkVersion=11 + ``` + + Or: + + ```shell + ./gradlew build -PjdkVersion=17 + ``` + :::note The first time you build the project, downloading the dependencies may take a while. You can add `-x test` to skip the tests, by using `./gradlew build -x test`. + +The built Gravitino libraries are Java 8 compatible, and verified under Java 8, 11, and 17 +environment. You can use Java 8, 11, 17 runtime to run the Gravitino server, no matter which +JDK version you use to build the project. ::: 3. Get the Gravitino binary package. diff --git a/docs/how-to-install.md b/docs/how-to-install.md index 6f3c5beec37..77953927613 100644 --- a/docs/how-to-install.md +++ b/docs/how-to-install.md @@ -8,8 +8,9 @@ This software is licensed under the Apache License version 2." ## Install Gravitino from scratch :::note -Gravitino requires JDK8 to run, please make sure you have JDK8 installed and `JAVA_HOME` -configured correctly. To confirm the Java version, you can simply run `${JAVA_HOME}/bin/java -version` command. +Gravitino supports running on Java 8, 11, and 17, please make sure you have Java installed and +`JAVA_HOME` configured correctly. To confirm the Java version, you can simply run +`${JAVA_HOME}/bin/java -version` command. ::: ### Get the Gravitino binary distribution package From 7a34c359ec55d264f41303c135c3bdd5107f17c4 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Thu, 21 Dec 2023 16:13:12 +0800 Subject: [PATCH 24/51] [MINOR] fix(doc): Fix a JSON format error in doc manage-metadata-using-gravitino.md (#1228) ### What changes were proposed in this pull request? Fix a low-level error in the document `manage-metadata-using-gravitino` ### Why are the changes needed? Just fix error. ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- docs/manage-metadata-using-gravitino.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/manage-metadata-using-gravitino.md b/docs/manage-metadata-using-gravitino.md index e807e8a0b80..03d968e1e35 100644 --- a/docs/manage-metadata-using-gravitino.md +++ b/docs/manage-metadata-using-gravitino.md @@ -916,6 +916,7 @@ curl -X PUT -H "Accept: application/vnd.gravitino.v1+json" \ "property": "key3", "value": "value3" } + ] }' http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/schema/tables/table ``` From 972747e772e11a9e889887ae7980b3a79e4919a9 Mon Sep 17 00:00:00 2001 From: Clearvive <143773256+Clearvive@users.noreply.github.com> Date: Thu, 21 Dec 2023 16:33:48 +0800 Subject: [PATCH 25/51] [#1112] fix(jdbc): deleteColumn throws exception for table with ifExisting = true. (#1200) ### What changes were proposed in this pull request? When 'ifExisting' is set to true in the 'deleteColumn' function, it will validate the existence of the specified column. If the column does not exist, it will throw an exception. ### Why are the changes needed? Fix: #1112 ### Does this PR introduce _any_ user-facing change? Users should be aware that when using 'deleteColumn,' it may throw a new exception, 'IllegalArgumentException.' ### How was this patch tested? UT --------- Co-authored-by: Clearvive --- .../jdbc/operation/JdbcTableOperations.java | 9 +++++-- .../mysql/operation/MysqlTableOperations.java | 24 +++++++++++++++-- .../operation/PostgreSqlTableOperations.java | 26 ++++++++++++++++--- .../jdbc/mysql/TestMysqlTableOperations.java | 21 +++++++++------ .../TestPostgreSqlTableOperations.java | 20 ++++++++------ 5 files changed, 77 insertions(+), 23 deletions(-) diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java index f8f78161166..1fcb53fb8a3 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/operation/JdbcTableOperations.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import javax.sql.DataSource; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -157,8 +158,12 @@ public void alterTable(String databaseName, String tableName, TableChange... cha throws NoSuchTableException { LOG.info("Attempting to alter table {} from database {}", tableName, databaseName); try (Connection connection = getConnection(databaseName)) { - JdbcConnectorUtils.executeUpdate( - connection, generateAlterTableSql(databaseName, tableName, changes)); + String sql = generateAlterTableSql(databaseName, tableName, changes); + if (StringUtils.isEmpty(sql)) { + LOG.info("No changes to alter table {} from database {}", tableName, databaseName); + return; + } + JdbcConnectorUtils.executeUpdate(connection, sql); LOG.info("Alter table {} from database {}", tableName, databaseName); } catch (final SQLException se) { throw this.exceptionMapper.toGravitinoException(se); diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java index 48a6f91307c..1b19b2cb788 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -37,6 +37,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; /** Table operations for MySQL. */ @@ -322,7 +323,11 @@ protected String generateAlterTableSql( updateColumnPositionFieldDefinition(updateColumnPosition, lazyLoadCreateTable)); } else if (change instanceof TableChange.DeleteColumn) { TableChange.DeleteColumn deleteColumn = (TableChange.DeleteColumn) change; - alterSql.add(deleteColumnFieldDefinition(deleteColumn)); + lazyLoadCreateTable = getOrCreateTable(databaseName, tableName, lazyLoadCreateTable); + String deleteColSql = deleteColumnFieldDefinition(deleteColumn, lazyLoadCreateTable); + if (StringUtils.isNotEmpty(deleteColSql)) { + alterSql.add(deleteColSql); + } } else if (change instanceof TableChange.UpdateColumnNullability) { lazyLoadCreateTable = getOrCreateTable(databaseName, tableName, lazyLoadCreateTable); alterSql.add( @@ -353,6 +358,9 @@ protected String generateAlterTableSql( alterSql.add("COMMENT '" + newComment + "'"); } + if (CollectionUtils.isEmpty(alterSql)) { + return ""; + } // Return the generated SQL statement String result = "ALTER TABLE " + tableName + "\n" + String.join(",\n", alterSql) + ";"; LOG.info("Generated alter table:{} sql: {}", databaseName + "." + tableName, result); @@ -496,11 +504,23 @@ private String updateColumnPositionFieldDefinition( return columnDefinition.toString(); } - private String deleteColumnFieldDefinition(TableChange.DeleteColumn deleteColumn) { + private String deleteColumnFieldDefinition( + TableChange.DeleteColumn deleteColumn, CreateTable lazyLoadCreateTable) { if (deleteColumn.fieldName().length > 1) { throw new UnsupportedOperationException("Mysql does not support nested column names."); } String col = deleteColumn.fieldName()[0]; + boolean colExists = + lazyLoadCreateTable.getColumnDefinitions().stream() + .map(MysqlTableOperations::getColumnName) + .anyMatch(s -> StringUtils.equals(col, s)); + if (!colExists) { + if (BooleanUtils.isTrue(deleteColumn.getIfExists())) { + return ""; + } else { + throw new IllegalArgumentException("Delete column does not exist: " + col); + } + } return "DROP COLUMN " + col; } diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java index a89b83308c1..cb7c47080f4 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlTableOperations.java @@ -33,6 +33,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; /** Table operations for PostgreSQL. */ @@ -351,8 +352,12 @@ protected String generateAlterTableSql( } else if (change instanceof TableChange.UpdateColumnPosition) { throw new IllegalArgumentException("PostgreSQL does not support column position."); } else if (change instanceof TableChange.DeleteColumn) { + lazyLoadTable = getOrCreateTable(schemaName, tableName, lazyLoadTable); TableChange.DeleteColumn deleteColumn = (TableChange.DeleteColumn) change; - alterSql.add(deleteColumnFieldDefinition(deleteColumn, tableName)); + String deleteColSql = deleteColumnFieldDefinition(deleteColumn, lazyLoadTable); + if (StringUtils.isNotEmpty(deleteColSql)) { + alterSql.add(deleteColSql); + } } else if (change instanceof TableChange.UpdateColumnNullability) { alterSql.add( updateColumnNullabilityDefinition( @@ -363,6 +368,11 @@ protected String generateAlterTableSql( } } + // If there is no change, return directly + if (alterSql.isEmpty()) { + return ""; + } + // Return the generated SQL statement String result = String.join("\n", alterSql); LOG.info("Generated alter table:{}.{} sql: {}", schemaName, tableName, result); @@ -398,11 +408,21 @@ private String updateCommentDefinition( } private String deleteColumnFieldDefinition( - TableChange.DeleteColumn deleteColumn, String tableName) { + TableChange.DeleteColumn deleteColumn, JdbcTable table) { if (deleteColumn.fieldName().length > 1) { throw new UnsupportedOperationException("PostgreSQL does not support nested column names."); } - return "ALTER TABLE " + tableName + " DROP COLUMN " + deleteColumn.fieldName()[0] + ";"; + String col = deleteColumn.fieldName()[0]; + boolean colExists = + Arrays.stream(table.columns()).anyMatch(s -> StringUtils.equals(col, s.name())); + if (!colExists) { + if (BooleanUtils.isTrue(deleteColumn.getIfExists())) { + return ""; + } else { + throw new IllegalArgumentException("Delete column does not exist: " + col); + } + } + return "ALTER TABLE " + table.name() + " DROP COLUMN " + deleteColumn.fieldName()[0] + ";"; } private String updateColumnTypeFieldDefinition( diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java index 206a573b557..8015d1caa6e 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/TestMysqlTableOperations.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.catalog.jdbc.JdbcColumn; import com.datastrato.gravitino.catalog.jdbc.JdbcTable; -import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.rel.TableChange; @@ -138,19 +137,25 @@ public void testOperationTable() { load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); assertionsTableInfo(newName, tableComment, columns, properties, load); - GravitinoRuntimeException gravitinoRuntimeException = + IllegalArgumentException illegalArgumentException = Assertions.assertThrows( - GravitinoRuntimeException.class, + IllegalArgumentException.class, () -> TABLE_OPERATIONS.alterTable( TEST_DB_NAME, newName, - TableChange.deleteColumn(new String[] {newColumn.name()}, true))); + TableChange.deleteColumn(new String[] {newColumn.name()}, false))); + Assertions.assertEquals( + "Delete column does not exist: " + newColumn.name(), illegalArgumentException.getMessage()); + Assertions.assertDoesNotThrow( + () -> + TABLE_OPERATIONS.alterTable( + TEST_DB_NAME, + newName, + TableChange.deleteColumn(new String[] {newColumn.name()}, true))); - Assertions.assertTrue( - gravitinoRuntimeException - .getMessage() - .contains("Can't DROP 'col_5'; check that column/key exists")); + TABLE_OPERATIONS.alterTable( + TEST_DB_NAME, newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true)); Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.purge(TEST_DB_NAME, newName)); Assertions.assertThrows( NoSuchTableException.class, () -> TABLE_OPERATIONS.purge(TEST_DB_NAME, newName)); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java index 3451837f75a..066044c75d5 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/TestPostgreSqlTableOperations.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter; import com.datastrato.gravitino.catalog.postgresql.operation.PostgreSqlSchemaOperations; import com.datastrato.gravitino.catalog.postgresql.operation.PostgreSqlTableOperations; -import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; import com.datastrato.gravitino.exceptions.NoSuchTableException; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.rel.TableChange; @@ -180,23 +179,28 @@ public void testOperationTable() { // delete column TABLE_OPERATIONS.alterTable( TEST_DB_NAME, newName, TableChange.deleteColumn(new String[] {newColumn.name()}, true)); + load = TABLE_OPERATIONS.load(TEST_DB_NAME, newName); alterColumns.remove(newColumn); assertionsTableInfo(newName, tableComment, alterColumns, properties, load); - GravitinoRuntimeException gravitinoRuntimeException = + IllegalArgumentException illegalArgumentException = Assertions.assertThrows( - GravitinoRuntimeException.class, + IllegalArgumentException.class, () -> TABLE_OPERATIONS.alterTable( TEST_DB_NAME, newName, - TableChange.deleteColumn(new String[] {newColumn.name()}, true))); + TableChange.deleteColumn(new String[] {newColumn.name()}, false))); + Assertions.assertEquals( + "Delete column does not exist: " + newColumn.name(), illegalArgumentException.getMessage()); - Assertions.assertTrue( - gravitinoRuntimeException - .getMessage() - .contains("ERROR: column \"col_5\" of relation \"new_table\" does not exist")); + Assertions.assertDoesNotThrow( + () -> + TABLE_OPERATIONS.alterTable( + TEST_DB_NAME, + newName, + TableChange.deleteColumn(new String[] {newColumn.name()}, true))); Assertions.assertDoesNotThrow(() -> TABLE_OPERATIONS.purge(TEST_DB_NAME, newName)); Assertions.assertThrows( NoSuchTableException.class, () -> TABLE_OPERATIONS.purge(TEST_DB_NAME, newName)); From 18feb15d12f36aa3b2f77d60bfc51e3e5b3424ac Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Thu, 21 Dec 2023 20:43:11 +0800 Subject: [PATCH 26/51] [#1190] feat(server-common): Support custom filters (#1191) ### What changes were proposed in this pull request? Add the support of custom filters referring to the Spark. ### Why are the changes needed? Some filters can be configured by users, the needs of users are different. Some filters are implemented by other libs. For example, Jetty has implemented the CORS filter, so we supported custom filters to satisfy the needs of users. Fix: #1190 ### Does this PR introduce _any_ user-facing change? Yes, I have added the document. ### How was this patch tested? test by the browser console I test Jetty's cors filter. Configuration ``` gravitino.server.webserver.customFilters=org.eclipse.jetty.servlets.CrossOriginFilter gravitino.server.webserver.org.eclipse.jetty.servlets.CrossOriginFilter.param.allowedOrigins=* ``` Browser console code ``` var http = new XMLHttpRequest(); var url = 'http://localhost:8090/api/version'; http.onreadystatechange = (e) => { console.log(http.responseText) } http.open("GET", url); http.send(); ``` --------- Co-authored-by: Heng Qin --- .../lakehouse/iceberg/IcebergRESTService.java | 1 + docs/gravitino-server-config.md | 28 +++++++++-------- docs/iceberg-rest-service.md | 5 ++++ .../gravitino/server/web/JettyServer.java | 20 +++++++++++-- .../server/web/JettyServerConfig.java | 30 ++++++++++++++++--- .../server/web/TestJettyServerConfig.java | 18 +++++++++++ .../gravitino/server/GravitinoServer.java | 1 + 7 files changed, 85 insertions(+), 18 deletions(-) diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java index cea2df4440d..bf23f6c0393 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java @@ -61,6 +61,7 @@ protected void configure() { Servlet servlet = new ServletContainer(config); server.addServlet(servlet, "/iceberg/*"); + server.addCustomFilters("/iceberg/*"); server.addFilter(new AuthenticationFilter(), "/iceberg/*"); } diff --git a/docs/gravitino-server-config.md b/docs/gravitino-server-config.md index 8ee97a166cf..ea90870ce19 100644 --- a/docs/gravitino-server-config.md +++ b/docs/gravitino-server-config.md @@ -23,18 +23,22 @@ The `gravitino.conf` file lists the configuration items in the following table. ### Gravitino HTTP Server configuration -| Configuration item | Description | Default value | Required | Since version | -|------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------| -| `gravitino.server.webserver.host` | The host of Gravitino server. | `0.0.0.0` | No | 0.1.0 | -| `gravitino.server.webserver.httpPort` | The port on which the Gravitino server listens for incoming connections. | `8090` | No | 0.1.0 | -| `gravitino.server.webserver.minThreads` | The minimum number of threads in the thread pool used by Jetty webserver. `minThreads` is 8 if the value is less than 8. | `Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100), 8)` | No | 0.2.0 | -| `gravitino.server.webserver.maxThreads` | The maximum number of threads in the thread pool used by Jetty webserver. `maxThreads` is 8 if the value is less than 8, and `maxThreads` must be great or equal to `minThreads`. | `Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)` | No | 0.1.0 | -| `gravitino.server.webserver.threadPoolWorkQueueSize` | The size of the queue in the thread pool used by Jetty webserver. | `100` | No | 0.1.0 | -| `gravitino.server.webserver.stopTimeout` | Time in milliseconds to gracefully shutdown the Jetty webserver, for more, please see `org.eclipse.jetty.server.Server#setStopTimeout`. | `30000` | No | 0.2.0 | -| `gravitino.server.webserver.idleTimeout` | The timeout in milliseconds of idle connections. | `30000` | No | 0.2.0 | -| `gravitino.server.webserver.requestHeaderSize` | Maximum size of HTTP requests. | `131072` | No | 0.1.0 | -| `gravitino.server.webserver.responseHeaderSize` | Maximum size of HTTP responses. | `131072` | No | 0.1.0 | -| `gravitino.server.shutdown.timeout` | Time in milliseconds to gracefully shutdown of the Gravitino webserver. | `3000` | No | 0.2.0 | +| Configuration item | Description | Default value | Required | Since version | +|-------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|----------|---------------| +| `gravitino.server.webserver.host` | The host of Gravitino server. | `0.0.0.0` | No | 0.1.0 | +| `gravitino.server.webserver.httpPort` | The port on which the Gravitino server listens for incoming connections. | `8090` | No | 0.1.0 | +| `gravitino.server.webserver.minThreads` | The minimum number of threads in the thread pool used by Jetty webserver. `minThreads` is 8 if the value is less than 8. | `Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100), 8)` | No | 0.2.0 | +| `gravitino.server.webserver.maxThreads` | The maximum number of threads in the thread pool used by Jetty webserver. `maxThreads` is 8 if the value is less than 8, and `maxThreads` must be great or equal to `minThreads`. | `Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)` | No | 0.1.0 | +| `gravitino.server.webserver.threadPoolWorkQueueSize` | The size of the queue in the thread pool used by Jetty webserver. | `100` | No | 0.1.0 | +| `gravitino.server.webserver.stopTimeout` | Time in milliseconds to gracefully shutdown the Jetty webserver, for more, please see `org.eclipse.jetty.server.Server#setStopTimeout`. | `30000` | No | 0.2.0 | +| `gravitino.server.webserver.idleTimeout` | The timeout in milliseconds of idle connections. | `30000` | No | 0.2.0 | +| `gravitino.server.webserver.requestHeaderSize` | Maximum size of HTTP requests. | `131072` | No | 0.1.0 | +| `gravitino.server.webserver.responseHeaderSize` | Maximum size of HTTP responses. | `131072` | No | 0.1.0 | +| `gravitino.server.shutdown.timeout` | Time in milliseconds to gracefully shutdown of the Gravitino webserver. | `3000` | No | 0.2.0 | +| `gravitino.server.webserver.customFilters` | Comma separated list of filter class names to apply to the APIs. | (none) | No | 0.4.0 | + +The filter in the customFilters should be a standard javax servlet Filter. +Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.server.webserver..param.=` ### Storage configuration diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index 86fd757de73..3d13acef210 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -42,6 +42,11 @@ Deploy the Gravitino server to the `GRAVITINO_HOME` directory. You can find the | `gravitino.auxService.iceberg-rest.idleTimeout` | The timeout in ms of idle connections. | `30000` | No | 0.2.0 | | `gravitino.auxService.iceberg-rest.requestHeaderSize` | The maximum size of an HTTP request. | `131072` | No | 0.2.0 | | `gravitino.auxService.iceberg-rest.responseHeaderSize` | The maximum size of an HTTP response. | `131072` | No | 0.2.0 | +| `gravitino.auxService.iceberg-rest.customFilters` | Comma separated list of filter class names to apply to the APIs. | (none) | No | 0.4.0 | + + +The filter in the customFilters should be a standard javax servlet Filter. +Filter parameters can also be specified in the configuration, by setting config entries of the form `gravitino.auxService.iceberg-rest..param.=` :::caution You must set `gravitino.auxService.iceberg-rest.httpPort` explicitly, like `9001`. diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index f111af3a0a3..cbb0e0ab9c1 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -18,6 +18,7 @@ import java.nio.file.Paths; import java.security.PrivilegedAction; import java.util.EnumSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; @@ -103,7 +104,7 @@ public synchronized void initialize( StringUtils.isNotBlank(serverConfig.getTrustStorePath()), "If enables the authentication of the client, must set trustStorePath"); Preconditions.checkArgument( - StringUtils.isNotBlank(serverConfig.getTrustStorePasword()), + StringUtils.isNotBlank(serverConfig.getTrustStorePassword()), "If enables the authentication of the client, must set trustStorePassword"); } ServerConnector httpsConnector = @@ -122,7 +123,7 @@ public synchronized void initialize( serverConfig.getSupportedAlgorithms(), serverConfig.isEnableClientAuth(), serverConfig.getTrustStorePath(), - serverConfig.getTrustStorePasword(), + serverConfig.getTrustStorePassword(), serverConfig.getTrustStoreType()); server.addConnector(httpsConnector); } else { @@ -433,4 +434,19 @@ public Thread run() { public ThreadPool getThreadPool() { return server.getThreadPool(); } + + public void addCustomFilters(String pathSpec) { + for (String filterName : serverConfig.getCustomFilters()) { + if (StringUtils.isBlank(filterName)) { + continue; + } + FilterHolder filterHolder = new FilterHolder(); + filterHolder.setClassName(filterName); + for (Map.Entry entry : + serverConfig.getAllWithPrefix(String.format("%s.param.", filterName)).entrySet()) { + filterHolder.setInitParameter(entry.getKey(), entry.getValue()); + } + servletContextHandler.addFilter(filterHolder, pathSpec, EnumSet.allOf(DispatcherType.class)); + } + } } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java index b27702c6a65..5b252102c91 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java @@ -171,6 +171,13 @@ public final class JettyServerConfig { .stringConf() .createWithDefault("JKS"); + public static final ConfigEntry> CUSTOM_FILTERS = + new ConfigBuilder("customFilters") + .doc("Comma separated list of filter class names to apply to the APIs") + .version("0.4.0") + .stringConf() + .createWithOptional(); + private final String host; private final int httpPort; @@ -199,7 +206,9 @@ public final class JettyServerConfig { private final Set enableCipherAlgorithms; private final boolean enableClientAuth; private final String trustStorePath; - private final String trustStorePasword; + private final String trustStorePassword; + + private final Set customFilters; private final String trustStoreType; private final Config internalConfig; @@ -245,8 +254,13 @@ private JettyServerConfig(Map configs) { Sets.newHashSet(internalConfig.get(ENABLE_CIPHER_ALGORITHMS).split(SPLITTER))); this.enableClientAuth = internalConfig.get(ENABLE_CLIENT_AUTH); this.trustStorePath = internalConfig.get(SSL_TRUST_STORE_PATH); - this.trustStorePasword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); + this.trustStorePassword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); this.trustStoreType = internalConfig.get(SSL_TRUST_STORE_TYPE); + this.customFilters = + internalConfig + .get(CUSTOM_FILTERS) + .map(filters -> Collections.unmodifiableSet(Sets.newHashSet(filters.split(SPLITTER)))) + .orElse(Collections.emptySet()); } public static JettyServerConfig fromConfig(Config config, String prefix) { @@ -330,14 +344,18 @@ public String getTrustStorePath() { return trustStorePath; } - public String getTrustStorePasword() { - return trustStorePasword; + public String getTrustStorePassword() { + return trustStorePassword; } public String getTrustStoreType() { return trustStoreType; } + public Map getAllWithPrefix(String prefix) { + return internalConfig.getConfigsWithPrefix(prefix); + } + private SSLContext getDefaultSSLContext() { try { return SSLContext.getDefault(); @@ -366,6 +384,10 @@ public Set getSupportedAlgorithms() { return supportedAlgorithms; } + public Set getCustomFilters() { + return customFilters; + } + @VisibleForTesting Set getSupportedCipherSuites() { SSLContext context = diff --git a/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java b/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java index 40b6ec8c943..60c15dd7a34 100644 --- a/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java +++ b/server-common/src/test/java/com/datastrato/gravitino/server/web/TestJettyServerConfig.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.server.web; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.config.ConfigBuilder; import com.google.common.collect.Sets; import java.util.Collections; import java.util.Optional; @@ -44,4 +45,21 @@ public void testCipherAlgorithms() { Assertions.assertIterableEquals( Sets.newHashSet(algorithm), jettyServerConfig.getSupportedAlgorithms()); } + + @Test + public void testCustomFilters() { + Config emptyconfig = new Config() {}; + JettyServerConfig jettyServerConfig = JettyServerConfig.fromConfig(emptyconfig, ""); + Assertions.assertTrue(jettyServerConfig.getCustomFilters().isEmpty()); + + Config somethingConfig = new Config() {}; + somethingConfig.set(JettyServerConfig.CUSTOM_FILTERS, Optional.of("1,2")); + somethingConfig.set(new ConfigBuilder("1.1").stringConf(), "test"); + somethingConfig.set(new ConfigBuilder("1.2").stringConf(), "test"); + jettyServerConfig = JettyServerConfig.fromConfig(somethingConfig, ""); + Assertions.assertIterableEquals( + Sets.newHashSet("1", "2"), jettyServerConfig.getCustomFilters()); + Assertions.assertTrue(jettyServerConfig.getAllWithPrefix("2.").isEmpty()); + Assertions.assertEquals(2, jettyServerConfig.getAllWithPrefix("1.").size()); + } } diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 415078a60da..5e0bd678de1 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -87,6 +87,7 @@ protected void configure() { server.addServlet(servlet, "/api/*"); Servlet configServlet = new ConfigServlet(serverConfig); server.addServlet(configServlet, "/configs"); + server.addCustomFilters("/api/*"); server.addFilter(new VersioningFilter(), "/api/*"); server.addFilter(new AuthenticationFilter(), "/api/*"); } From 38ea933615b7838ad291045328ed3101bb21a492 Mon Sep 17 00:00:00 2001 From: Xun Liu Date: Thu, 21 Dec 2023 20:43:44 +0800 Subject: [PATCH 27/51] [#879][#925] Docker network Pool overlaps with other one on this address space (#1211) ### What changes were proposed in this pull request? 1. Added Docker network overlap detection 2. Reduced IP allocation ranges for lower conflict probability, Modified to `10.20.30.0/28` only allocate IP ranger `10.20.30.1` ~ `10.20.30.14` 3. Docker network only created on MacOS and default Docker server environment ### Why are the changes needed? Fix: #879, #925 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? CI --- dev/docker/tools/docker-connector.conf | 2 +- integration-test/build.gradle.kts | 9 ++ .../test/container/BaseContainer.java | 2 +- .../test/container/ContainerSuite.java | 115 +++++++++++++++++- .../test/container/NetworksConflictTest.java | 24 ++++ 5 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/NetworksConflictTest.java diff --git a/dev/docker/tools/docker-connector.conf b/dev/docker/tools/docker-connector.conf index 378e0699478..d8e859b95cf 100644 --- a/dev/docker/tools/docker-connector.conf +++ b/dev/docker/tools/docker-connector.conf @@ -5,4 +5,4 @@ # Fixed docker container network subnet in Gravitino integration testing module # Generate command: `docker network ls --filter driver=bridge --format "{{.ID}}" | xargs docker network inspect --format "route {{range .IPAM.Config}}{{.Subnet}}{{end}}" > ${bin}/docker-connector.conf` -route 10.0.0.0/22 +route 10.20.30.0/28 diff --git a/integration-test/build.gradle.kts b/integration-test/build.gradle.kts index 3b6a6e9357c..ee7b14860c9 100644 --- a/integration-test/build.gradle.kts +++ b/integration-test/build.gradle.kts @@ -272,6 +272,15 @@ tasks.test { environment("PROJECT_VERSION", version) environment("TRINO_CONF_DIR", buildDir.path + "/trino-conf") + val dockerRunning = project.extra["dockerRunning"] as? Boolean ?: false + val macDockerConnector = project.extra["macDockerConnector"] as? Boolean ?: false + if (OperatingSystem.current().isMacOsX() && + dockerRunning && + macDockerConnector + ) { + environment("NEED_CREATE_DOCKER_NETWORK", "true") + } + // Gravitino CI Docker image environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.7") environment("GRAVITINO_CI_TRINO_DOCKER_IMAGE", "datastrato/gravitino-ci-trino:0.1.2") diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java index d70d3721f61..331efae498e 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/BaseContainer.java @@ -189,7 +189,7 @@ public SELF withEnvVars(Map envVars) { } public SELF withNetwork(Network network) { - this.network = Optional.of(network); + this.network = Optional.ofNullable(network); return self; } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java index 7ee9d2a4069..a42fa815203 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/ContainerSuite.java @@ -6,12 +6,17 @@ import com.datastrato.gravitino.integration.test.util.CloseableGroup; import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.command.RemoveNetworkCmd; import com.github.dockerjava.api.model.Info; +import com.github.dockerjava.api.model.Network.Ipam.Config; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import java.io.Closeable; import java.io.IOException; +import java.net.InetAddress; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.DockerClientFactory; @@ -22,9 +27,11 @@ public class ContainerSuite implements Closeable { private static volatile ContainerSuite instance = null; // The subnet must match the configuration in `dev/docker/tools/mac-docker-connector.conf` - private static final String CONTAINER_NETWORK_SUBNET = "10.0.0.0/22"; - private static final String CONTAINER_NETWORK_GATEWAY = "10.0.0.1"; - private static final String CONTAINER_NETWORK_IPRANGE = "10.0.0.100/22"; + public static final String CONTAINER_NETWORK_SUBNET = "10.20.30.0/28"; + private static final String CONTAINER_NETWORK_GATEWAY = "10.20.30.1"; + private static final String CONTAINER_NETWORK_IPRANGE = "10.20.30.14/28"; + private static final String NETWORK_NAME = "gravitino-ci-network"; + private static Network network = null; private static HiveContainer hiveContainer; private static TrinoContainer trinoContainer; @@ -38,7 +45,9 @@ private ContainerSuite() { Info info = dockerClient.infoCmd().exec(); LOG.info("Docker info: {}", info); - network = createDockerNetwork(); + if ("true".equalsIgnoreCase(System.getenv("NEED_CREATE_DOCKER_NETWORK"))) { + network = createDockerNetwork(); + } } catch (Exception e) { throw new RuntimeException("Failed to initialize ContainerSuite", e); } @@ -126,12 +135,53 @@ public HiveContainer getHiveContainer() { // Let containers assign addresses in a fixed subnet to avoid `mac-docker-connector` needing to // refresh the configuration private static Network createDockerNetwork() { + DockerClient dockerClient = DockerClientFactory.instance().client(); + + // Remove the `gravitino-ci-network` if it exists + boolean networkExists = + dockerClient.listNetworksCmd().withNameFilter(NETWORK_NAME).exec().stream() + .anyMatch(network -> network.getName().equals(NETWORK_NAME)); + if (networkExists) { + RemoveNetworkCmd removeNetworkCmd = dockerClient.removeNetworkCmd(NETWORK_NAME); + removeNetworkCmd.exec(); + } + + // Check if the subnet of the network conflicts with `gravitino-ci-network` + List networks = dockerClient.listNetworksCmd().exec(); + + for (com.github.dockerjava.api.model.Network network : networks) { + List ipamConfigs = network.getIpam().getConfig(); + for (Config ipamConfig : ipamConfigs) { + try { + if (ipRangesOverlap(ipamConfig.getSubnet(), CONTAINER_NETWORK_SUBNET)) { + LOG.error( + "The Docker of the network {} subnet {} conflicts with the `gravitino-ci-network` {}, " + + "You can either remove {} network from Docker, or modify the `ContainerSuite.CONTAINER_NETWORK_SUBNET` variable", + network.getName(), + ipamConfig.getSubnet(), + CONTAINER_NETWORK_SUBNET, + network.getName()); + throw new RuntimeException( + "The Docker of the network " + + network.getName() + + " subnet " + + ipamConfig.getSubnet() + + " conflicts with the `gravitino-ci-network` " + + CONTAINER_NETWORK_SUBNET); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + com.github.dockerjava.api.model.Network.Ipam.Config ipamConfig = new com.github.dockerjava.api.model.Network.Ipam.Config(); ipamConfig .withSubnet(CONTAINER_NETWORK_SUBNET) .withGateway(CONTAINER_NETWORK_GATEWAY) - .withIpRange(CONTAINER_NETWORK_IPRANGE); + .withIpRange(CONTAINER_NETWORK_IPRANGE) + .setNetworkID("gravitino-ci-network"); return closer.register( Network.builder() @@ -142,6 +192,61 @@ private static Network createDockerNetwork() { .build()); } + public static boolean ipRangesOverlap(String cidr1, String cidr2) throws Exception { + long[] net1 = cidrToRange(cidr1); + long[] net2 = cidrToRange(cidr2); + + long startIp1 = net1[0]; + long endIp1 = net1[1]; + long startIp2 = net2[0]; + long endIp2 = net2[1]; + + LOG.info("Subnet1: {} allocate IP ranger [{} ~ {}]", cidr1, long2Ip(startIp1), long2Ip(endIp1)); + LOG.info("Subnet2: {} allocate IP ranger [{} ~ {}]", cidr2, long2Ip(startIp2), long2Ip(endIp2)); + + if (startIp1 > endIp2 || endIp1 < startIp2) { + return false; + } else { + return true; + } + } + + public static String long2Ip(final long ip) { + final StringBuilder result = new StringBuilder(15); + result.append(ip >> 24 & 0xff).append("."); + result.append(ip >> 16 & 0xff).append("."); + result.append(ip >> 8 & 0xff).append("."); + result.append(ip & 0xff); + + return result.toString(); + } + + // Classless Inter-Domain Routing (CIDR) + // https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + private static long[] cidrToRange(String cidr) throws Exception { + String[] parts = cidr.split("/"); + InetAddress inetAddress = InetAddress.getByName(parts[0]); + int prefixLength = Integer.parseInt(parts[1]); + + ByteBuffer buffer = ByteBuffer.wrap(inetAddress.getAddress()); + long ip = + (inetAddress.getAddress().length == 4) ? buffer.getInt() & 0xFFFFFFFFL : buffer.getLong(); + long mask = -(1L << (32 - prefixLength)); + + long startIp = ip & mask; + + long endIp; + if (inetAddress.getAddress().length == 4) { + // IPv4 + endIp = startIp + ((1L << (32 - prefixLength)) - 1); + } else { + // IPv6 + endIp = startIp + ((1L << (128 - prefixLength)) - 1); + } + + return new long[] {startIp, endIp}; + } + @Override public void close() throws IOException { try { diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/NetworksConflictTest.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/NetworksConflictTest.java new file mode 100644 index 00000000000..ad69ccd6529 --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/NetworksConflictTest.java @@ -0,0 +1,24 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ +package com.datastrato.gravitino.integration.test.container; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class NetworksConflictTest { + @Test + public void networksConflictTest1() throws Exception { + final String subnet1 = "10.20.30.0/28"; // allocate IP ranger 10.20.30.1 ~ 10.20.30.14 + final String subnet2 = "10.20.30.0/26"; // allocate IP ranger is 10.20.30.1 ~ 10.20.30.62 + Assertions.assertTrue(ContainerSuite.ipRangesOverlap(subnet1, subnet2)); + } + + @Test + public void networksConflictTest2() throws Exception { + final String subnet1 = "10.20.30.0/28"; // allocate IP ranger is 10.20.30.1 ~ 10.20.30.14 + final String subnet2 = "10.20.31.0/28"; // allocate IP ranger is 10.20.31.1 ~ 10.20.31.14 + Assertions.assertFalse(ContainerSuite.ipRangesOverlap(subnet1, subnet2)); + } +} From 6dbbcee5ffed2e85139c3b20ed5c6de763929970 Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Fri, 22 Dec 2023 16:46:07 +0800 Subject: [PATCH 28/51] [Minor] docs: Add more details about dropping schema and tables (#1236) ### What changes were proposed in this pull request? - Add a description about parameter `cascade` when dropping the schema - Please provide more details on how to drop the table and clarify `dropTable` and `purgeTable` ### Why are the changes needed? Make the document more user-friendly. ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? N/A --- docs/manage-metadata-using-gravitino.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/manage-metadata-using-gravitino.md b/docs/manage-metadata-using-gravitino.md index 03d968e1e35..d968f2d622a 100644 --- a/docs/manage-metadata-using-gravitino.md +++ b/docs/manage-metadata-using-gravitino.md @@ -561,6 +561,9 @@ supportsSchemas.dropSchema(NameIdentifier.of("metalake", "catalog", "schema"), t +If `cascade` is true, Gravitino will drop all tables under the schema. Otherwise, Gravitino will throw an exception if there are tables under the schema. +Some catalogs may not support cascading deletion of a schema, please refer to the related doc for more details. + ### List all schemas under a catalog You can alter all schemas under a catalog by sending a `GET` request to the `/api/metalakes/{metalake_name}/catalogs/{catalog_name}/schemas` endpoint or just use the Gravitino Java client. The following is an example of list all schema @@ -962,9 +965,11 @@ You can remove a table by sending a `DELETE` request to the `/api/metalakes/{met ```bash +## purge can be true or false, if purge is true, Gravitino will remove the data of the table. + curl -X DELETE -H "Accept: application/vnd.gravitino.v1+json" \ -H "Content-Type: application/json" \ -http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/schema/tables/table +http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/schema/tables/table?purge=true ``` @@ -976,13 +981,21 @@ http://localhost:8090/api/metalakes/metalake/catalogs/catalog/schemas/schema/tab Catalog catalog = gravitinoMetaLake.loadCatalog(NameIdentifier.of("metalake", "catalog")); TableCatalog tableCatalog = catalog.asTableCatalog(); + +// Drop a table tableCatalog.dropTable(NameIdentifier.of("metalake", "catalog", "schema", "table")); + +// Purge a table +tableCatalog.purgeTable(NameIdentifier.of("metalake", "catalog", "schema", "table")); // ... ``` +There are two ways to drop a table: `dropTable` and `purgeTable`, the difference between them is that `purgeTable` will remove data of the table, while `dropTable` only removes the metadata of the table. Some engine such as +Apache Hive support both, `dropTable` will only remove the metadata of a table and the data in HDFS can be reused later through the format of external table. + ### List all tables under a schema You can list all tables in a schema by sending a `GET` request to the `/api/metalakes/{metalake_name}/catalogs/{catalog_name}/schemas/{schema_name}/tables` endpoint or just use the Gravitino Java client. The following is an example of list all tables in a schema: From f2a4708ddc1b6ec35a03ee466c96f605ab5d1917 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Fri, 22 Dec 2023 17:58:26 +0800 Subject: [PATCH 29/51] [#1221] improvement(common): Add the method `checkValue` for the ConfigEntry (#1233) ### What changes were proposed in this pull request? Now, ConfigEntry lacks the ability of check config options. So we add the support of this feature referring to Spark. ### Why are the changes needed? Fix: #1221 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add new ut. --------- Co-authored-by: Heng Qin --- .../catalog/jdbc/config/JdbcConfig.java | 9 ++++ .../catalog/jdbc/utils/DataSourceUtils.java | 9 ---- .../operation/PostgreSqlSchemaOperations.java | 4 -- .../lakehouse/iceberg/IcebergConfig.java | 7 +++ .../iceberg/ops/IcebergTableOps.java | 9 +--- .../iceberg/utils/IcebergCatalogUtil.java | 14 ++---- .../gravitino/config/ConfigConstants.java | 14 ++++++ .../gravitino/config/ConfigEntry.java | 45 ++++++++++++++++- .../gravitino/config/TestConfigEntry.java | 29 +++++++++++ .../server/auth/OAuth2TokenAuthenticator.java | 2 - .../gravitino/server/auth/OAuthConfig.java | 5 ++ .../server/web/JettyServerConfig.java | 49 ++++++++++++++++--- .../gravitino/server/ServerConfig.java | 2 + .../gravitino/server/web/ConfigServlet.java | 17 +++++-- 14 files changed, 171 insertions(+), 44 deletions(-) create mode 100644 common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java index 41dba89aad5..a7ae1c55062 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/config/JdbcConfig.java @@ -7,8 +7,10 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import java.util.Map; +import org.apache.commons.lang3.StringUtils; public class JdbcConfig extends Config { @@ -17,6 +19,7 @@ public class JdbcConfig extends Config { .doc("The url of the Jdbc connection") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry JDBC_DATABASE = @@ -24,6 +27,7 @@ public class JdbcConfig extends Config { .doc("The database of the jdbc connection") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry JDBC_DRIVER = @@ -31,6 +35,7 @@ public class JdbcConfig extends Config { .doc("The driver of the jdbc connection") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry USERNAME = @@ -38,6 +43,7 @@ public class JdbcConfig extends Config { .doc("The username of the Jdbc connection") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry PASSWORD = @@ -45,6 +51,7 @@ public class JdbcConfig extends Config { .doc("The password of the Jdbc connection") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry POOL_MIN_SIZE = @@ -52,6 +59,7 @@ public class JdbcConfig extends Config { .doc("The minimum number of connections in the pool") .version("0.3.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(2); public static final ConfigEntry POOL_MAX_SIZE = @@ -59,6 +67,7 @@ public class JdbcConfig extends Config { .doc("The maximum number of connections in the pool") .version("0.3.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(10); public String getJdbcUrl() { diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java index f63a89d892d..4118e687f50 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/utils/DataSourceUtils.java @@ -6,14 +6,12 @@ import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.exceptions.GravitinoRuntimeException; -import com.google.common.base.Preconditions; import java.sql.SQLException; import java.util.Map; import java.util.Properties; import javax.sql.DataSource; import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.BasicDataSourceFactory; -import org.apache.commons.lang3.StringUtils; /** * Utility class for creating a {@link DataSource} from a {@link JdbcConfig}. It is mainly @@ -42,19 +40,12 @@ private static DataSource createDBCPDataSource(JdbcConfig jdbcConfig) throws Exc BasicDataSource basicDataSource = BasicDataSourceFactory.createDataSource(getProperties(jdbcConfig)); String jdbcUrl = jdbcConfig.getJdbcUrl(); - Preconditions.checkArgument(StringUtils.isNotBlank(jdbcUrl), "The jdbc url can't be blank."); basicDataSource.setUrl(jdbcUrl); String driverClassName = jdbcConfig.getJdbcDriver(); - Preconditions.checkArgument( - StringUtils.isNotBlank(driverClassName), "The jdbc driver can't be blank."); basicDataSource.setDriverClassName(driverClassName); String userName = jdbcConfig.getUsername(); - Preconditions.checkArgument( - StringUtils.isNotBlank(userName), "The jdbc user name can't be blank."); basicDataSource.setUsername(userName); String password = jdbcConfig.getPassword(); - Preconditions.checkArgument( - StringUtils.isNotBlank(password), "The jdbc password can't be blank."); basicDataSource.setPassword(password); basicDataSource.setMaxTotal(jdbcConfig.getPoolMaxSize()); basicDataSource.setMinIdle(jdbcConfig.getPoolMinSize()); diff --git a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java index f33c1f12ab1..f7ef970d7d4 100644 --- a/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java +++ b/catalogs/catalog-jdbc-postgresql/src/main/java/com/datastrato/gravitino/catalog/postgresql/operation/PostgreSqlSchemaOperations.java @@ -10,7 +10,6 @@ import com.datastrato.gravitino.catalog.jdbc.operation.JdbcDatabaseOperations; import com.datastrato.gravitino.exceptions.NoSuchSchemaException; import com.datastrato.gravitino.meta.AuditInfo; -import com.google.common.base.Preconditions; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -33,9 +32,6 @@ public void initialize( DataSource dataSource, JdbcExceptionConverter exceptionMapper, Map conf) { super.initialize(dataSource, exceptionMapper, conf); database = new JdbcConfig(conf).getJdbcDatabase(); - Preconditions.checkArgument( - StringUtils.isNotBlank(database), - "The `jdbc-database` configuration item is mandatory in PostgreSQL."); } @Override diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java index 86f7c2328a4..5109052bd49 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergConfig.java @@ -15,8 +15,10 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import java.util.Map; +import org.apache.commons.lang3.StringUtils; public class IcebergConfig extends Config { @@ -32,6 +34,7 @@ public class IcebergConfig extends Config { .doc("Warehouse directory of catalog") .version("0.2.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry CATALOG_URI = @@ -39,6 +42,7 @@ public class IcebergConfig extends Config { .doc("The uri config of the Iceberg catalog") .version("0.2.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry JDBC_USER = @@ -46,6 +50,7 @@ public class IcebergConfig extends Config { .doc("The username of the Jdbc connection") .version("0.2.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry JDBC_PASSWORD = @@ -53,6 +58,7 @@ public class IcebergConfig extends Config { .doc("The password of the Jdbc connection") .version("0.2.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry JDBC_DRIVER = @@ -60,6 +66,7 @@ public class IcebergConfig extends Config { .doc("The driver of the Jdbc connection") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry JDBC_INIT_TABLES = diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java index f910cb42216..76995a83dc8 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/ops/IcebergTableOps.java @@ -12,7 +12,6 @@ import java.util.Collections; import java.util.Optional; import javax.ws.rs.NotSupportedException; -import org.apache.commons.lang3.StringUtils; import org.apache.iceberg.Transaction; import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; @@ -43,12 +42,8 @@ public class IcebergTableOps implements AutoCloseable { public IcebergTableOps(IcebergConfig icebergConfig) { String catalogType = icebergConfig.get(IcebergConfig.CATALOG_BACKEND); if (!IcebergCatalogBackend.MEMORY.name().equalsIgnoreCase(catalogType)) { - Preconditions.checkArgument( - StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE)), - "Catalog warehouse can't be blank"); - Preconditions.checkArgument( - StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.CATALOG_URI)), - "Catalog uri can't be blank"); + icebergConfig.get(IcebergConfig.CATALOG_WAREHOUSE); + icebergConfig.get(IcebergConfig.CATALOG_URI); } catalog = IcebergCatalogUtil.loadCatalogBackend(catalogType, icebergConfig.getAllConfig()); if (catalog instanceof SupportsNamespaces) { diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java index 8df85dce349..c172f73884b 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/utils/IcebergCatalogUtil.java @@ -8,11 +8,9 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; -import com.google.common.base.Preconditions; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.catalog.Catalog; @@ -46,14 +44,10 @@ private static HiveCatalog loadHiveCatalog(Map properties) { private static JdbcCatalog loadJdbcCatalog(Map properties) { IcebergConfig icebergConfig = new IcebergConfig(properties); String driverClassName = icebergConfig.getJdbcDriver(); - Preconditions.checkArgument( - StringUtils.isNotBlank(driverClassName), "Jdbc driver can't be blank"); - Preconditions.checkArgument( - StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.JDBC_USER)), - "Jdbc user can't be blank"); - Preconditions.checkArgument( - StringUtils.isNotBlank(icebergConfig.get(IcebergConfig.JDBC_PASSWORD)), - "Jdbc password can't be blank"); + + icebergConfig.get(IcebergConfig.JDBC_USER); + icebergConfig.get(IcebergConfig.JDBC_PASSWORD); + try { // Load the jdbc driver Class.forName(driverClassName); diff --git a/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java b/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java new file mode 100644 index 00000000000..b2ca294049b --- /dev/null +++ b/common/src/main/java/com/datastrato/gravitino/config/ConfigConstants.java @@ -0,0 +1,14 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.config; + +public interface ConfigConstants { + String NOT_BLANK_ERROR_MSG = "The value can't be blank"; + + String POSITIVE_NUMBER_ERROR_MSG = "The value must be a positive number"; + + String NON_NEGATIVE_NUMBER_ERROR_MSG = "The value must be a non-negative number"; +} diff --git a/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java b/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java index a381117fd19..ee68b726940 100644 --- a/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java +++ b/common/src/main/java/com/datastrato/gravitino/config/ConfigEntry.java @@ -8,7 +8,9 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Stream; import lombok.Getter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +40,7 @@ public class ConfigEntry { private boolean isOptional; private boolean hasNoDefault; + private Consumer validator; /** * Creates a new ConfigEntry instance. @@ -107,6 +110,30 @@ void setHasNoDefault() { this.hasNoDefault = true; } + /** Set the validator value. */ + void setValidator(Consumer validator) { + this.validator = validator; + } + + /** + * Checks if the user-provided value for the config matches the validator. + * + * @param checkValueFunc The validator of the configuration option + * @param errorMsg The thrown error message if the value is invalid + * @return The current ConfigEntry instance + */ + public ConfigEntry checkValue(Function checkValueFunc, String errorMsg) { + setValidator( + value -> { + if (!checkValueFunc.apply(value)) { + throw new IllegalArgumentException( + String.format( + "%s in %s is invalid. %s", stringConverter.apply(value), key, errorMsg)); + } + }); + return this; + } + /** * Creates a new ConfigEntry instance based on this configuration entry with a default value. * @@ -119,6 +146,7 @@ public ConfigEntry createWithDefault(T t) { conf.setValueConverter(valueConverter); conf.setStringConverter(stringConverter); conf.setDefaultValue(t); + conf.setValidator(validator); return conf; } @@ -136,6 +164,13 @@ public ConfigEntry> createWithOptional() { // Unless explicitly set by the user, null values are not expected to occur. conf.setStringConverter(t -> t.map(stringConverter).orElse(null)); conf.setOptional(); + conf.setValidator( + optionValue -> { + if (Stream.of(Optional.ofNullable(validator), optionValue) + .allMatch(Optional::isPresent)) { + validator.accept(optionValue.get()); + } + }); return conf; } @@ -151,6 +186,7 @@ public ConfigEntry create() { conf.setValueConverter(valueConverter); conf.setStringConverter(stringConverter); conf.setHasNoDefault(); + conf.setValidator(validator); return conf; } @@ -176,13 +212,20 @@ public T readFrom(Map properties) throws NoSuchElementException if (defaultValue != null) { return defaultValue; } else if (hasNoDefault) { + if (validator != null) { + validator.accept(null); + } return null; } else if (!isOptional) { throw new NoSuchElementException("No configuration found for key " + key); } } - return valueConverter.apply(value); + T convertedValue = valueConverter.apply(value); + if (validator != null) { + validator.accept(convertedValue); + } + return convertedValue; } /** diff --git a/common/src/test/java/com/datastrato/gravitino/config/TestConfigEntry.java b/common/src/test/java/com/datastrato/gravitino/config/TestConfigEntry.java index 3a360eb3827..4d8eb2ac6bd 100644 --- a/common/src/test/java/com/datastrato/gravitino/config/TestConfigEntry.java +++ b/common/src/test/java/com/datastrato/gravitino/config/TestConfigEntry.java @@ -6,6 +6,7 @@ import com.google.common.collect.Lists; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -125,4 +126,32 @@ public void testSetConf() { testConf1.writeTo(configMap, Optional.empty()); Assertions.assertEquals("11", configMap.get("gravitino.test.int1")); } + + @Test + public void testCheckValue() { + ConfigEntry testConfDefault = + new ConfigBuilder("gravitino.test.default") + .intConf() + .checkValue(value -> value > 2, "error") + .createWithDefault(1); + testConfDefault.writeTo(configMap, -10); + Assertions.assertThrows( + IllegalArgumentException.class, () -> testConfDefault.readFrom(configMap)); + ConfigEntry testConfNoDefault = + new ConfigBuilder("gravitino.test.no.default") + .stringConf() + .checkValue(Objects::nonNull, "error") + .create(); + Assertions.assertThrows( + IllegalArgumentException.class, () -> testConfNoDefault.readFrom(configMap)); + ConfigEntry> testConfOptional = + new ConfigBuilder("gravitino.test.optional") + .stringConf() + .checkValue(value -> !Objects.equals(value, "test"), "error") + .createWithOptional(); + Assertions.assertDoesNotThrow(() -> testConfOptional.readFrom(configMap)); + testConfOptional.writeTo(configMap, Optional.of("test")); + Assertions.assertThrows( + IllegalArgumentException.class, () -> testConfOptional.readFrom(configMap)); + } } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java index 574e9fc2ee4..7ea53aff7c4 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java @@ -100,8 +100,6 @@ public void initialize(Config config) throws RuntimeException { this.serviceAudience = config.get(OAuthConfig.SERVICE_AUDIENCE); this.allowSkewSeconds = config.get(OAuthConfig.ALLOW_SKEW_SECONDS); String configuredSignKey = config.get(OAuthConfig.DEFAULT_SIGN_KEY); - Preconditions.checkArgument( - StringUtils.isNotBlank(configuredSignKey), "Default signing key can't be blank"); Preconditions.checkArgument( StringUtils.isNotBlank(config.get(OAuthConfig.DEFAULT_TOKEN_PATH)), "The path for token of the default OAuth server can't be blank"); diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java index 6adfe4b880d..26581fe0a83 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuthConfig.java @@ -7,8 +7,10 @@ import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import io.jsonwebtoken.SignatureAlgorithm; +import org.apache.commons.lang3.StringUtils; public interface OAuthConfig extends Configs { @@ -33,6 +35,7 @@ public interface OAuthConfig extends Configs { .doc("The signing key of JWT when Gravitino uses OAuth as the authenticator") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); ConfigEntry SIGNATURE_ALGORITHM_TYPE = @@ -47,6 +50,7 @@ public interface OAuthConfig extends Configs { .doc("The uri of the default OAuth server") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); ConfigEntry DEFAULT_TOKEN_PATH = @@ -54,5 +58,6 @@ public interface OAuthConfig extends Configs { .doc("The path for token of the default OAuth server") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java index 5b252102c91..0ca06d0c3ed 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -17,6 +18,7 @@ import java.util.Optional; import java.util.Set; import javax.net.ssl.SSLContext; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,7 @@ public final class JettyServerConfig { .doc("The http port number of the Jetty web server") .version("0.1.0") .intConf() + .checkValue(value -> value >= 0, ConfigConstants.NON_NEGATIVE_NUMBER_ERROR_MSG) .createWithDefault(8090); public static final ConfigEntry WEBSERVER_MIN_THREADS = @@ -43,6 +46,7 @@ public final class JettyServerConfig { .doc("The minimum number of threads in the thread pool used by Jetty webserver") .version("0.2.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault( Math.max(Math.min(Runtime.getRuntime().availableProcessors() * 2, 100), 4)); @@ -51,6 +55,7 @@ public final class JettyServerConfig { .doc("The maximum number of threads in the thread pool used by Jetty webserver") .version("0.1.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(Math.max(Runtime.getRuntime().availableProcessors() * 4, 400)); public static final ConfigEntry WEBSERVER_STOP_TIMEOUT = @@ -58,6 +63,7 @@ public final class JettyServerConfig { .doc("Time in milliseconds to gracefully shutdown the Jetty webserver") .version("0.2.0") .longConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(30 * 1000L); public static final ConfigEntry WEBSERVER_IDLE_TIMEOUT = @@ -65,6 +71,7 @@ public final class JettyServerConfig { .doc("The timeout in milliseconds of idle connections") .version("0.2.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(30 * 1000); public static final ConfigEntry WEBSERVER_REQUEST_HEADER_SIZE = @@ -72,6 +79,7 @@ public final class JettyServerConfig { .doc("Maximum size of HTTP requests") .version("0.1.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(128 * 1024); public static final ConfigEntry WEBSERVER_RESPONSE_HEADER_SIZE = @@ -79,6 +87,7 @@ public final class JettyServerConfig { .doc("Maximum size of HTTP responses") .version("0.1.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(128 * 1024); public static final ConfigEntry WEBSERVER_THREAD_POOL_WORK_QUEUE_SIZE = @@ -86,6 +95,7 @@ public final class JettyServerConfig { .doc("The size of the queue in the thread pool used by Jetty webserver") .version("0.1.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(100); public static final ConfigEntry ENABLE_HTTPS = @@ -100,6 +110,7 @@ public final class JettyServerConfig { .doc("The https port number of the Jetty web server") .version("0.3.0") .intConf() + .checkValue(value -> value >= 0, ConfigConstants.NON_NEGATIVE_NUMBER_ERROR_MSG) .createWithDefault(8433); public static final ConfigEntry SSL_KEYSTORE_PATH = @@ -107,6 +118,7 @@ public final class JettyServerConfig { .doc("Path to the key store file") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry SSL_KEYSTORE_PASSWORD = @@ -114,6 +126,7 @@ public final class JettyServerConfig { .doc("Password to the key store") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry SSL_MANAGER_PASSWORD = @@ -121,6 +134,7 @@ public final class JettyServerConfig { .doc("Manager password to the key store") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry SSL_KEYSTORE_TYPE = @@ -136,6 +150,7 @@ public final class JettyServerConfig { .version("0.3.0") .stringConf() .createWithOptional(); + public static final ConfigEntry ENABLE_CIPHER_ALGORITHMS = new ConfigBuilder("enableCipherAlgorithms") .doc("The collection of the cipher algorithms are enabled ") @@ -155,6 +170,7 @@ public final class JettyServerConfig { .doc("Path to the trust store file") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry SSL_TRUST_STORE_PASSWORD = @@ -162,6 +178,7 @@ public final class JettyServerConfig { .doc("Password to the trust store") .version("0.3.0") .stringConf() + .checkValue(StringUtils::isNotBlank, ConfigConstants.NOT_BLANK_ERROR_MSG) .create(); public static final ConfigEntry SSL_TRUST_STORE_TYPE = @@ -244,23 +261,41 @@ private JettyServerConfig(Map configs) { this.enableHttps = internalConfig.get(ENABLE_HTTPS); this.httpsPort = internalConfig.get(WEBSERVER_HTTPS_PORT); - this.keyStorePath = internalConfig.get(SSL_KEYSTORE_PATH); - this.keyStorePassword = internalConfig.get(SSL_KEYSTORE_PASSWORD); - this.managerPassword = internalConfig.get(SSL_MANAGER_PASSWORD); - this.keyStoreType = internalConfig.get(SSL_KEYSTORE_TYPE); this.tlsProtocol = internalConfig.get(SSL_PROTOCOL); this.enableCipherAlgorithms = Collections.unmodifiableSet( Sets.newHashSet(internalConfig.get(ENABLE_CIPHER_ALGORITHMS).split(SPLITTER))); this.enableClientAuth = internalConfig.get(ENABLE_CLIENT_AUTH); - this.trustStorePath = internalConfig.get(SSL_TRUST_STORE_PATH); - this.trustStorePassword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); - this.trustStoreType = internalConfig.get(SSL_TRUST_STORE_TYPE); + this.customFilters = internalConfig .get(CUSTOM_FILTERS) .map(filters -> Collections.unmodifiableSet(Sets.newHashSet(filters.split(SPLITTER)))) .orElse(Collections.emptySet()); + + this.keyStoreType = internalConfig.get(SSL_KEYSTORE_TYPE); + this.trustStoreType = internalConfig.get(SSL_TRUST_STORE_TYPE); + String keyStorePath = null; + String keyStorePassword = null; + String managerPassword = null; + String trustStorePath = null; + String trustStorePassword = null; + + if (this.enableHttps) { + keyStorePath = internalConfig.get(SSL_KEYSTORE_PATH); + keyStorePassword = internalConfig.get(SSL_KEYSTORE_PASSWORD); + managerPassword = internalConfig.get(SSL_MANAGER_PASSWORD); + if (this.enableClientAuth) { + trustStorePath = internalConfig.get(SSL_TRUST_STORE_PATH); + trustStorePassword = internalConfig.get(SSL_TRUST_STORE_PASSWORD); + } + } + + this.keyStorePath = keyStorePath; + this.keyStorePassword = keyStorePassword; + this.managerPassword = managerPassword; + this.trustStorePassword = trustStorePassword; + this.trustStorePath = trustStorePath; } public static JettyServerConfig fromConfig(Config config, String prefix) { diff --git a/server/src/main/java/com/datastrato/gravitino/server/ServerConfig.java b/server/src/main/java/com/datastrato/gravitino/server/ServerConfig.java index a1f3c732083..1e36e73edd4 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/ServerConfig.java +++ b/server/src/main/java/com/datastrato/gravitino/server/ServerConfig.java @@ -6,6 +6,7 @@ import com.datastrato.gravitino.Config; import com.datastrato.gravitino.config.ConfigBuilder; +import com.datastrato.gravitino.config.ConfigConstants; import com.datastrato.gravitino.config.ConfigEntry; public class ServerConfig extends Config { @@ -15,6 +16,7 @@ public class ServerConfig extends Config { .doc("The stop idle timeout(millis) of the Gravitino Server") .version("0.1.0") .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) .createWithDefault(3 * 1000); public ServerConfig(boolean loadDefaults) { diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/ConfigServlet.java b/server/src/main/java/com/datastrato/gravitino/server/web/ConfigServlet.java index 0a3f51909a5..368396b4595 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/ConfigServlet.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/ConfigServlet.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.server.web; import com.datastrato.gravitino.Configs; +import com.datastrato.gravitino.auth.AuthenticatorType; import com.datastrato.gravitino.config.ConfigEntry; import com.datastrato.gravitino.json.JsonUtils; import com.datastrato.gravitino.server.ServerConfig; @@ -22,17 +23,25 @@ public class ConfigServlet extends HttpServlet { - private static final ImmutableSet> configEntries = - ImmutableSet.of( - Configs.AUTHENTICATOR, OAuthConfig.DEFAULT_SERVER_URI, OAuthConfig.DEFAULT_TOKEN_PATH); + private static final ImmutableSet> oauthConfigEntries = + ImmutableSet.of(OAuthConfig.DEFAULT_SERVER_URI, OAuthConfig.DEFAULT_TOKEN_PATH); + + private static final ImmutableSet> basicConfigEntries = + ImmutableSet.of(Configs.AUTHENTICATOR); private final Map configs = Maps.newHashMap(); public ConfigServlet(ServerConfig serverConfig) { - for (ConfigEntry key : configEntries) { + for (ConfigEntry key : basicConfigEntries) { String config = String.valueOf(serverConfig.get(key)); configs.put(key.getKey(), config); } + if (serverConfig.get(Configs.AUTHENTICATOR).equalsIgnoreCase(AuthenticatorType.OAUTH.name())) { + for (ConfigEntry key : oauthConfigEntries) { + String config = String.valueOf(serverConfig.get(key)); + configs.put(key.getKey(), config); + } + } } protected void doGet(HttpServletRequest req, HttpServletResponse res) From b278df66bf412304c9bb3f4c80686639e94c19e4 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Fri, 22 Dec 2023 19:36:03 +0800 Subject: [PATCH 30/51] [MINOR] refactor: Use `isEmpty()` to replace `size() == 0` (#1242) ### What changes were proposed in this pull request? Some collections have O(n) size method, it's a good way to use `isEmpty`. ### Why are the changes needed? Improve the quality of code. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? CI passed. --------- Co-authored-by: Heng Qin --- .../com/datastrato/gravitino/aux/AuxiliaryServiceManager.java | 2 +- .../java/com/datastrato/gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/integration/test/catalog/hive/CatalogHiveIT.java | 2 +- .../gravitino/integration/test/container/HiveContainer.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java index 05fb03f3c28..ffc3fcdc149 100644 --- a/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java +++ b/core/src/main/java/com/datastrato/gravitino/aux/AuxiliaryServiceManager.java @@ -53,7 +53,7 @@ private Class lookupAuxService( .map(GravitinoAuxiliaryService::getClass) .collect(Collectors.toList()); - if (providers.size() == 0) { + if (providers.isEmpty()) { throw new IllegalArgumentException("No GravitinoAuxiliaryService found for: " + provider); } else if (providers.size() > 1) { throw new IllegalArgumentException( diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 9fb32d2c9b3..1ad192ceacf 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -593,7 +593,7 @@ private Class lookupCatalogProvider(String provider, .map(CatalogProvider::getClass) .collect(Collectors.toList()); - if (providers.size() == 0) { + if (providers.isEmpty()) { throw new IllegalArgumentException("No catalog provider found for: " + provider); } else if (providers.size() > 1) { throw new IllegalArgumentException("Multiple catalog providers found for: " + provider); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java index 62aad9d1798..c0168f73b66 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/hive/CatalogHiveIT.java @@ -144,7 +144,7 @@ public static void startup() throws Exception { // Check if hive client can connect to hive metastore hiveClientPool = new HiveClientPool(1, hiveConf); List dbs = hiveClientPool.run(client -> client.getAllDatabases()); - Assertions.assertTrue(dbs.size() > 0); + Assertions.assertFalse(dbs.isEmpty()); sparkSession = SparkSession.builder() diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java index 227ed9a6cb0..b9543bb5d79 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/container/HiveContainer.java @@ -103,7 +103,7 @@ protected boolean checkContainerStatus(int retryLimit) { while (nRetry++ < retryLimit) { try { List databases = hiveClientPool.run(IMetaStoreClient::getAllDatabases); - if (databases.size() > 0) { + if (!databases.isEmpty()) { isHiveConnectSuccess = true; break; } From fbbb7d6c88d90d87f223e33e77002cbc85350e46 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Fri, 22 Dec 2023 20:47:08 +0800 Subject: [PATCH 31/51] [#42] feat(server-common): Support cors filter (#1238) ### What changes were proposed in this pull request? I wrap the cors filter of Jetty as our system internal filter. Most default values respect the default values of the cors filter of Jetty. I only change the default value of `AllowMethods`.The origin default value is `PUT,GET,POST,HEAD`. Because we usually use `DELETE` method, too. I change it to `PUT,GET,POST,HEAD,DELETE`. ### Why are the changes needed? Fix: #42 ### Does this PR introduce _any_ user-facing change? yes, I have added the document. ### How was this patch tested? UT + manual test ``` var http = new XMLHttpRequest(); var url = 'http://localhost:8090/api/version'; http.onreadystatechange = (e) => { console.log(http.responseText) } http.open("GET", url); http.send(); ``` --------- Co-authored-by: Heng Qin --- .../lakehouse/iceberg/IcebergRESTService.java | 3 +- docs/security.md | 29 +++ gradle/libs.versions.toml | 3 +- .../server/web/CorsFilterHolder.java | 38 ++++ .../gravitino/server/web/JettyServer.java | 9 + .../server/web/JettyServerConfig.java | 168 ++++++++++++++++-- .../server/web/TestCorsFilterHolder.java | 70 ++++++++ .../gravitino/server/GravitinoServer.java | 3 +- 8 files changed, 301 insertions(+), 22 deletions(-) create mode 100644 server-common/src/main/java/com/datastrato/gravitino/server/web/CorsFilterHolder.java create mode 100644 server-common/src/test/java/com/datastrato/gravitino/server/web/TestCorsFilterHolder.java diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java index bf23f6c0393..976adacd7d3 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/com/datastrato/gravitino/catalog/lakehouse/iceberg/IcebergRESTService.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.catalog.lakehouse.iceberg.web.IcebergObjectMapperProvider; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.MetricsSource; -import com.datastrato.gravitino.server.auth.AuthenticationFilter; import com.datastrato.gravitino.server.web.HttpServerMetricsSource; import com.datastrato.gravitino.server.web.JettyServer; import com.datastrato.gravitino.server.web.JettyServerConfig; @@ -62,7 +61,7 @@ protected void configure() { Servlet servlet = new ServletContainer(config); server.addServlet(servlet, "/iceberg/*"); server.addCustomFilters("/iceberg/*"); - server.addFilter(new AuthenticationFilter(), "/iceberg/*"); + server.addSystemFilters("/iceberg/*"); } @Override diff --git a/docs/security.md b/docs/security.md index 3c8b095ff0b..a2758022d9e 100644 --- a/docs/security.md +++ b/docs/security.md @@ -258,3 +258,32 @@ If you want to use the command `curl`, you can follow the commands: openssl x509 -inform der -in $JAVA_HOME/localhost.crt -out certificate.pem curl -v -X GET --cacert ./certificate.pem -H "Accept: application/vnd.gravitino.v1+json" -H "Content-Type: application/json" https://localhost:8433/api/version ``` +## Cross origin resource filter + +### Server configuration + +| Configuration item | Description | Default value | Required | Since version | +|----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|----------|---------------| +| `gravitino.server.webserver.enableCorsFilter` | Enable cross origin resource share filter. | false | No | 0.4.0 | +| `gravitino.server.webserver.allowedOrigins` | A comma separated list of origins that are allowed to access the resources. Default value is *, means all origins. | `*` | No | 0.4.0 | +| `gravitino.server.webserver.allowedTimingOrigins` | A comma separated list of origins that are allowed to time the resource. Default value is the empty string, means no origins. | `` | No | 0.4.0 | +| `gravitino.server.webserver.allowedMethods` | A comma separated list of HTTP methods that are allowed to be used when accessing the resources. Default value is GET,POST,HEAD,DELETE. | `GET,POST,HEAD,DELETE` | No | 0.4.0 | +| `gravitino.server.webserver.allowedHeaders` | A comma separated list of HTTP headers that are allowed to be specified when accessing the resources. Default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *, this means that any headers will be accepted. | `X-Requested-With,Content-Type,Accept,Origin` | No | 0.4.0 | +| `gravitino.server.webserver.preflightMaxAgeInSecs` | The number of seconds that preflight requests can be cached by the client. Default value is 1800 seconds, or 30 minutes. | `1800` | No | 0.4.0 | +| `gravitino.server.webserver.allowCredentials` | A boolean indicating if the resource allows requests with credentials. Default value is true. | `true` | No | 0.4.0 | +| `gravitino.server.webserver.exposedHeaders` | A comma separated list of HTTP headers that are allowed to be exposed on the client. Default value is the empty list. | `` | No | 0.4.0 | +| `gravitino.server.webserver.chainPreflight` | If true preflight requests are chained to their target resource for normal handling (as an OPTION request). Otherwise the filter will response to the preflight. Default is true. | `true` | No | 0.4.0 | + +### Iceberg REST service's configuration + +| Configuration item | Description | Default value | Required | Since version | +|-----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------|----------|---------------| +| `gravitino.auxService.iceberg-rest.enableCorsFilter` | Enable cross origin resource share filter. | false | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.allowedOrigins` | A comma separated list of origins that are allowed to access the resources. Default value is *, means all origins. | `*` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.allowedTimingOrigins` | A comma separated list of origins that are allowed to time the resource. Default value is the empty string, means no origins. | `` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.allowedMethods` | A comma separated list of HTTP methods that are allowed to be used when accessing the resources. Default value is GET,POST,HEAD,DELETE. | `GET,POST,HEAD,DELETE` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.allowedHeaders` | A comma separated list of HTTP headers that are allowed to be specified when accessing the resources. Default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *, this means that any headers will be accepted. | `X-Requested-With,Content-Type,Accept,Origin` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.preflightMaxAgeInSecs` | The number of seconds that preflight requests can be cached by the client. Default value is 1800 seconds, or 30 minutes. | `1800` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.allowCredentials` | A boolean indicating if the resource allows requests with credentials. Default value is true. | `true` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.exposedHeaders` | A comma separated list of HTTP headers that are allowed to be exposed on the client. Default value is the empty list. | `` | No | 0.4.0 | +| `gravitino.auxService.iceberg-rest.chainPreflight` | If true preflight requests are chained to their target resource for normal handling (as an OPTION request). Otherwise the filter will response to the preflight. Default is true. | `true` | No | 0.4.0 | diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index afa90a46c4c..87bd41a5dae 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -70,6 +70,7 @@ log4j-core = { group = "org.apache.logging.log4j", name = "log4j-core", version. log4j-12-api = { group = "org.apache.logging.log4j", name = "log4j-1.2-api", version.ref = "log4j" } jetty-server = { group = "org.eclipse.jetty", name = "jetty-server", version.ref = "jetty" } jetty-servlet = { group = "org.eclipse.jetty", name = "jetty-servlet", version.ref = "jetty" } +jetty-servlets = { group = "org.eclipse.jetty", name = "jetty-servlets", version.ref = "jetty" } jetty-webapp = { group = "org.eclipse.jetty", name = "jetty-webapp", version.ref = "jetty" } jersey-server = { group = "org.glassfish.jersey.core", name = "jersey-server", version.ref = "jersey" } jersey-container-servlet-core = { group = "org.glassfish.jersey.containers", name = "jersey-container-servlet-core", version.ref = "jersey" } @@ -135,7 +136,7 @@ postgresql-driver = { group = "org.postgresql", name = "postgresql", version.ref [bundles] log4j = ["slf4j-api", "log4j-slf4j2-impl", "log4j-api", "log4j-core", "log4j-12-api"] -jetty = ["jetty-server", "jetty-servlet", "jetty-webapp"] +jetty = ["jetty-server", "jetty-servlet", "jetty-webapp", "jetty-servlets"] jersey = ["jersey-server", "jersey-container-servlet-core", "jersey-container-jetty-http", "jersey-media-json-jackson", "jersey-hk2"] iceberg = ["iceberg-core", "iceberg-api"] jwt = ["jwt-api", "jwt-impl", "jwt-gson"] diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/CorsFilterHolder.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/CorsFilterHolder.java new file mode 100644 index 00000000000..38983a4e041 --- /dev/null +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/CorsFilterHolder.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.server.web; + +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlets.CrossOriginFilter; + +class CorsFilterHolder { + + static final String PREFLIGHT_MAX_AGE = "preflightMaxAge"; + + private CorsFilterHolder() {} + + public static FilterHolder create(JettyServerConfig config) { + FilterHolder filterHolder = new FilterHolder(); + filterHolder.setClassName(CrossOriginFilter.class.getName()); + filterHolder.setInitParameter( + JettyServerConfig.ALLOWED_ORIGINS.getKey(), config.getAllowedOrigins()); + filterHolder.setInitParameter( + JettyServerConfig.ALLOWED_TIMING_ORIGINS.getKey(), config.getAllowedTimingOrigins()); + filterHolder.setInitParameter( + JettyServerConfig.CHAIN_PREFLIGHT.getKey(), String.valueOf(config.isChainPreflight())); + filterHolder.setInitParameter( + PREFLIGHT_MAX_AGE, String.valueOf(config.getPreflightMaxAgeInSecs())); + filterHolder.setInitParameter( + JettyServerConfig.ALLOW_CREDENTIALS.getKey(), String.valueOf(config.isAllowCredentials())); + filterHolder.setInitParameter( + JettyServerConfig.ALLOWED_METHODS.getKey(), config.getAllowedMethods()); + filterHolder.setInitParameter( + JettyServerConfig.ALLOWED_HEADERS.getKey(), config.getAllowedHeaders()); + filterHolder.setInitParameter( + JettyServerConfig.EXPOSED_HEADERS.getKey(), config.getExposedHeaders()); + return filterHolder; + } +} diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java index cbb0e0ab9c1..9cc81b221c5 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServer.java @@ -8,6 +8,7 @@ import com.codahale.metrics.servlets.MetricsServlet; import com.datastrato.gravitino.GravitinoEnv; import com.datastrato.gravitino.metrics.MetricsSystem; +import com.datastrato.gravitino.server.auth.AuthenticationFilter; import com.google.common.base.Preconditions; import java.io.File; import java.io.IOException; @@ -449,4 +450,12 @@ public void addCustomFilters(String pathSpec) { servletContextHandler.addFilter(filterHolder, pathSpec, EnumSet.allOf(DispatcherType.class)); } } + + public void addSystemFilters(String pathSpec) { + if (serverConfig.isEnableCorsFilter()) { + servletContextHandler.addFilter( + CorsFilterHolder.create(serverConfig), pathSpec, EnumSet.allOf(DispatcherType.class)); + } + addFilter(new AuthenticationFilter(), pathSpec); + } } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java index 0ca06d0c3ed..aa69411218a 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/web/JettyServerConfig.java @@ -100,7 +100,7 @@ public final class JettyServerConfig { public static final ConfigEntry ENABLE_HTTPS = new ConfigBuilder("enableHttps") - .doc("Enables https") + .doc("Enable https") .version("0.3.0") .booleanConf() .createWithDefault(false); @@ -160,7 +160,7 @@ public final class JettyServerConfig { public static final ConfigEntry ENABLE_CLIENT_AUTH = new ConfigBuilder("enableClientAuth") - .doc("Enables the authentication of the client") + .doc("Enable the authentication of the client") .version("0.3.0") .booleanConf() .createWithDefault(false); @@ -194,6 +194,85 @@ public final class JettyServerConfig { .version("0.4.0") .stringConf() .createWithOptional(); + public static final ConfigEntry ENABLE_CORS_FILTER = + new ConfigBuilder("enableCorsFilter") + .doc("Enable cross origin resource share filter") + .version("0.4.0") + .booleanConf() + .createWithDefault(false); + + public static final ConfigEntry ALLOWED_ORIGINS = + new ConfigBuilder("allowedOrigins") + .doc( + "A comma separated list of origins that are allowed to access the resources." + + " Default value is *, means all origins") + .version("0.4.0") + .stringConf() + .createWithDefault("*"); + + public static final ConfigEntry ALLOWED_TIMING_ORIGINS = + new ConfigBuilder("allowedTimingOrigins") + .doc( + "A comma separated list of origins that are allowed to time the resource." + + " Default value is the empty string, means no origins.") + .version("0.4.0") + .stringConf() + .createWithDefault(""); + + public static final ConfigEntry ALLOWED_METHODS = + new ConfigBuilder("allowedMethods") + .doc( + "A comma separated list of HTTP methods that are allowed to be used when accessing the resources." + + " Default value is GET,POST,HEAD,DELETE") + .version("0.4.0") + .stringConf() + .createWithDefault("GET,POST,HEAD,DELETE"); + + public static final ConfigEntry ALLOWED_HEADERS = + new ConfigBuilder("allowedHeaders") + .doc( + "A comma separated list of HTTP headers that are allowed to be specified when accessing the resources." + + " Default value is X-Requested-With,Content-Type,Accept,Origin. If the value is a single *," + + " this means that any headers will be accepted.") + .version("0.4.0") + .stringConf() + .createWithDefault("X-Requested-With,Content-Type,Accept,Origin"); + + public static final ConfigEntry PREFLIGHT_MAX_AGE_IN_SECS = + new ConfigBuilder("preflightMaxAgeInSecs") + .doc( + "The number of seconds that preflight requests can be cached by the client." + + " Default value is 1800 seconds, or 30 minutes") + .version("0.4.0") + .intConf() + .checkValue(value -> value > 0, ConfigConstants.POSITIVE_NUMBER_ERROR_MSG) + .createWithDefault(1800); + + public static final ConfigEntry ALLOW_CREDENTIALS = + new ConfigBuilder("allowCredentials") + .doc( + "A boolean indicating if the resource allows requests with credentials. Default value is true") + .version("0.4.0") + .booleanConf() + .createWithDefault(true); + + public static final ConfigEntry EXPOSED_HEADERS = + new ConfigBuilder("exposedHeaders") + .doc( + "A comma separated list of HTTP headers that are allowed to be exposed on the client." + + " Default value is the empty list") + .version("0.4.0") + .stringConf() + .createWithDefault(""); + + public static final ConfigEntry CHAIN_PREFLIGHT = + new ConfigBuilder("chainPreflight") + .doc( + "If true preflight requests are chained to their target resource for normal handling " + + "(as an OPTION request). Otherwise the filter will response to the preflight. Default is true.") + .version("0.4.0") + .booleanConf() + .createWithDefault(true); private final String host; @@ -224,9 +303,18 @@ public final class JettyServerConfig { private final boolean enableClientAuth; private final String trustStorePath; private final String trustStorePassword; - private final Set customFilters; private final String trustStoreType; + private final boolean enableCorsFilter; + private final String allowedOrigins; + private final String allowedTimingOrigins; + private final int preflightMaxAgeInSecs; + private final String allowedMethods; + private final String allowedHeaders; + private final boolean allowCredentials; + private final String exposedHeaders; + private final boolean chainPreflight; + private final Config internalConfig; private JettyServerConfig(Map configs) { @@ -296,6 +384,16 @@ private JettyServerConfig(Map configs) { this.managerPassword = managerPassword; this.trustStorePassword = trustStorePassword; this.trustStorePath = trustStorePath; + + this.enableCorsFilter = internalConfig.get(ENABLE_CORS_FILTER); + this.allowedOrigins = internalConfig.get(ALLOWED_ORIGINS); + this.allowedTimingOrigins = internalConfig.get(ALLOWED_TIMING_ORIGINS); + this.preflightMaxAgeInSecs = internalConfig.get(PREFLIGHT_MAX_AGE_IN_SECS); + this.allowedMethods = internalConfig.get(ALLOWED_METHODS); + this.allowedHeaders = internalConfig.get(ALLOWED_HEADERS); + this.allowCredentials = internalConfig.get(ALLOW_CREDENTIALS); + this.exposedHeaders = internalConfig.get(EXPOSED_HEADERS); + this.chainPreflight = internalConfig.get(CHAIN_PREFLIGHT); } public static JettyServerConfig fromConfig(Config config, String prefix) { @@ -387,10 +485,60 @@ public String getTrustStoreType() { return trustStoreType; } + public Set getSupportedAlgorithms() { + if (enableCipherAlgorithms.isEmpty()) { + return Collections.emptySet(); + } + + Set supportedAlgorithms = Sets.newHashSet(enableCipherAlgorithms); + supportedAlgorithms.retainAll(getSupportedCipherSuites()); + return supportedAlgorithms; + } + public Map getAllWithPrefix(String prefix) { return internalConfig.getConfigsWithPrefix(prefix); } + public Set getCustomFilters() { + return customFilters; + } + + public boolean isEnableCorsFilter() { + return enableCorsFilter; + } + + public String getAllowedOrigins() { + return allowedOrigins; + } + + public String getAllowedTimingOrigins() { + return allowedTimingOrigins; + } + + public int getPreflightMaxAgeInSecs() { + return preflightMaxAgeInSecs; + } + + public String getAllowedMethods() { + return allowedMethods; + } + + public boolean isAllowCredentials() { + return allowCredentials; + } + + public String getExposedHeaders() { + return exposedHeaders; + } + + public boolean isChainPreflight() { + return chainPreflight; + } + + public String getAllowedHeaders() { + return allowedHeaders; + } + private SSLContext getDefaultSSLContext() { try { return SSLContext.getDefault(); @@ -409,20 +557,6 @@ private SSLContext getSSLContextInstance(String protocol) { } } - public Set getSupportedAlgorithms() { - if (enableCipherAlgorithms.isEmpty()) { - return Collections.emptySet(); - } - - Set supportedAlgorithms = Sets.newHashSet(enableCipherAlgorithms); - supportedAlgorithms.retainAll(getSupportedCipherSuites()); - return supportedAlgorithms; - } - - public Set getCustomFilters() { - return customFilters; - } - @VisibleForTesting Set getSupportedCipherSuites() { SSLContext context = diff --git a/server-common/src/test/java/com/datastrato/gravitino/server/web/TestCorsFilterHolder.java b/server-common/src/test/java/com/datastrato/gravitino/server/web/TestCorsFilterHolder.java new file mode 100644 index 00000000000..54537bc8a82 --- /dev/null +++ b/server-common/src/test/java/com/datastrato/gravitino/server/web/TestCorsFilterHolder.java @@ -0,0 +1,70 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.server.web; + +import com.datastrato.gravitino.Config; +import java.util.Map; +import org.eclipse.jetty.servlet.FilterHolder; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestCorsFilterHolder { + + @Test + public void testCreateCorsFilterHolder() { + Config config = new Config() {}; + JettyServerConfig jettyServerConfig = JettyServerConfig.fromConfig(config, ""); + FilterHolder filterHolder = CorsFilterHolder.create(jettyServerConfig); + Map parameters = filterHolder.getInitParameters(); + Assertions.assertEquals( + JettyServerConfig.ALLOWED_ORIGINS.getDefaultValue(), + parameters.get(JettyServerConfig.ALLOWED_ORIGINS.getKey())); + Assertions.assertEquals( + JettyServerConfig.ALLOWED_TIMING_ORIGINS.getDefaultValue(), + parameters.get(JettyServerConfig.ALLOWED_TIMING_ORIGINS.getKey())); + Assertions.assertEquals( + String.valueOf(JettyServerConfig.ALLOW_CREDENTIALS.getDefaultValue()), + parameters.get(JettyServerConfig.ALLOW_CREDENTIALS.getKey())); + Assertions.assertEquals( + JettyServerConfig.ALLOWED_HEADERS.getDefaultValue(), + parameters.get(JettyServerConfig.ALLOWED_HEADERS.getKey())); + Assertions.assertEquals( + String.valueOf(JettyServerConfig.CHAIN_PREFLIGHT.getDefaultValue()), + parameters.get(JettyServerConfig.CHAIN_PREFLIGHT.getKey())); + Assertions.assertEquals( + JettyServerConfig.EXPOSED_HEADERS.getDefaultValue(), + parameters.get(JettyServerConfig.EXPOSED_HEADERS.getKey())); + Assertions.assertEquals( + JettyServerConfig.ALLOWED_METHODS.getDefaultValue(), + parameters.get(JettyServerConfig.ALLOWED_METHODS.getKey())); + Assertions.assertEquals( + String.valueOf(JettyServerConfig.PREFLIGHT_MAX_AGE_IN_SECS.getDefaultValue()), + parameters.get(CorsFilterHolder.PREFLIGHT_MAX_AGE)); + Assertions.assertEquals( + "org.eclipse.jetty.servlets.CrossOriginFilter", filterHolder.getClassName()); + config.set(JettyServerConfig.ALLOWED_ORIGINS, "a"); + config.set(JettyServerConfig.ALLOWED_TIMING_ORIGINS, "b"); + config.set(JettyServerConfig.ALLOWED_HEADERS, "c"); + config.set(JettyServerConfig.ALLOWED_METHODS, "d"); + config.set(JettyServerConfig.EXPOSED_HEADERS, "e"); + config.set(JettyServerConfig.ALLOW_CREDENTIALS, false); + config.set(JettyServerConfig.CHAIN_PREFLIGHT, false); + config.set(JettyServerConfig.PREFLIGHT_MAX_AGE_IN_SECS, 10); + jettyServerConfig = JettyServerConfig.fromConfig(config, ""); + filterHolder = CorsFilterHolder.create(jettyServerConfig); + parameters = filterHolder.getInitParameters(); + Assertions.assertEquals("a", parameters.get(JettyServerConfig.ALLOWED_ORIGINS.getKey())); + Assertions.assertEquals("b", parameters.get(JettyServerConfig.ALLOWED_TIMING_ORIGINS.getKey())); + Assertions.assertEquals("false", parameters.get(JettyServerConfig.ALLOW_CREDENTIALS.getKey())); + Assertions.assertEquals("c", parameters.get(JettyServerConfig.ALLOWED_HEADERS.getKey())); + Assertions.assertEquals("false", parameters.get(JettyServerConfig.CHAIN_PREFLIGHT.getKey())); + Assertions.assertEquals("e", parameters.get(JettyServerConfig.EXPOSED_HEADERS.getKey())); + Assertions.assertEquals("d", parameters.get(JettyServerConfig.ALLOWED_METHODS.getKey())); + Assertions.assertNull(parameters.get(JettyServerConfig.PREFLIGHT_MAX_AGE_IN_SECS.getKey())); + + Assertions.assertEquals("10", parameters.get(CorsFilterHolder.PREFLIGHT_MAX_AGE)); + } +} diff --git a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java index 5e0bd678de1..460db9add03 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/com/datastrato/gravitino/server/GravitinoServer.java @@ -10,7 +10,6 @@ import com.datastrato.gravitino.meta.MetalakeManager; import com.datastrato.gravitino.metrics.MetricsSystem; import com.datastrato.gravitino.metrics.source.MetricsSource; -import com.datastrato.gravitino.server.auth.AuthenticationFilter; import com.datastrato.gravitino.server.auth.ServerAuthenticator; import com.datastrato.gravitino.server.web.ConfigServlet; import com.datastrato.gravitino.server.web.HttpServerMetricsSource; @@ -89,7 +88,7 @@ protected void configure() { server.addServlet(configServlet, "/configs"); server.addCustomFilters("/api/*"); server.addFilter(new VersioningFilter(), "/api/*"); - server.addFilter(new AuthenticationFilter(), "/api/*"); + server.addSystemFilters("/api/*"); } public void start() throws Exception { From fbc690f05a11a594c3cc3e31b8b3f75402512e76 Mon Sep 17 00:00:00 2001 From: qqqttt123 <148952220+qqqttt123@users.noreply.github.com> Date: Mon, 25 Dec 2023 18:10:45 +0800 Subject: [PATCH 32/51] [MINOR] fix(docs): Modify the document according to the README of playground (#1243) ### What changes were proposed in this pull request? Modify the document according to the pr https://github.com/datastrato/gravitino-playground/pull/8 I will submit another pr to modify the document of website. ### Why are the changes needed? We should keep consistent. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? No. Co-authored-by: Heng Qin --- docs/how-to-use-the-playground.md | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/how-to-use-the-playground.md b/docs/how-to-use-the-playground.md index c7f92016d4b..a254719affc 100644 --- a/docs/how-to-use-the-playground.md +++ b/docs/how-to-use-the-playground.md @@ -49,12 +49,12 @@ Use simple queries to test in the Trino CLI. ```SQL SHOW CATALOGS; -CREATE SCHEMA "metalake_demo.catalog_demo".db1 +CREATE SCHEMA "metalake_demo.catalog_hive".db1 WITH (location = 'hdfs://hive:9000/user/hive/warehouse/db1.db'); -SHOW CREATE SCHEMA "metalake_demo.catalog_demo".db1; +SHOW CREATE SCHEMA "metalake_demo.catalog_hive".db1; -CREATE TABLE "metalake_demo.catalog_demo".db1.table_001 +CREATE TABLE "metalake_demo.catalog_hive".db1.table_001 ( name varchar, salary varchar @@ -63,15 +63,15 @@ WITH ( format = 'TEXTFILE' ); -INSERT INTO "metalake_demo.catalog_demo".db1.table_001 (name, salary) VALUES ('sam', '11'); +INSERT INTO "metalake_demo.catalog_hive".db1.table_001 (name, salary) VALUES ('sam', '11'); -SELECT * FROM "metalake_demo.catalog_demo".db1.table_001; +SELECT * FROM "metalake_demo.catalog_hive".db1.table_001; -SHOW SCHEMAS from "metalake_demo.catalog_demo"; +SHOW SCHEMAS from "metalake_demo.catalog_hive"; -DESCRIBE "metalake_demo.catalog_demo".db1.table_001; +DESCRIBE "metalake_demo.catalog_hive".db1.table_001; -SHOW TABLES from "metalake_demo.catalog_demo".db1; +SHOW TABLES from "metalake_demo.catalog_hive".db1; ``` ### Cross-catalog queries @@ -90,7 +90,7 @@ WITH totalsales AS ( SELECT employee_id, SUM(total_amount) AS sales_amount - FROM "metalake_demo.catalog_demo".sales.sales + FROM "metalake_demo.catalog_hive".sales.sales GROUP BY employee_id ), rankedemployees AS ( @@ -107,7 +107,7 @@ SELECT job_title, sales_amount FROM rankedemployees AS r -JOIN "metalake_demo.catalog_pg1".hr.employees AS e +JOIN "metalake_demo.catalog_postgres".hr.employees AS e ON r.employee_id = e.employee_id WHERE sales_rank = 1; @@ -119,18 +119,18 @@ You run the SQL. ```SQL WITH customersales AS ( SELECT - "metalake_demo.catalog_demo".sales.customers.customer_id, + "metalake_demo.catalog_hive".sales.customers.customer_id, customer_name, customer_email, location AS state, SUM(total_amount) AS total_spent - FROM "metalake_demo.catalog_demo".sales.sales - JOIN "metalake_demo.catalog_demo".sales.customers - ON "metalake_demo.catalog_demo".sales.sales.customer_id = "metalake_demo.catalog_demo".sales.customers.customer_id - JOIN "metalake_demo.catalog_demo".sales.stores - ON "metalake_demo.catalog_demo".sales.sales.store_id = "metalake_demo.catalog_demo".sales.stores.store_id + FROM "metalake_demo.catalog_hive".sales.sales + JOIN "metalake_demo.catalog_hive".sales.customers + ON "metalake_demo.catalog_hive".sales.sales.customer_id = "metalake_demo.catalog_hive".sales.customers.customer_id + JOIN "metalake_demo.catalog_hive".sales.stores + ON "metalake_demo.catalog_hive".sales.sales.store_id = "metalake_demo.catalog_hive".sales.stores.store_id GROUP BY - "metalake_demo.catalog_demo".sales.customers.customer_id, + "metalake_demo.catalog_hive".sales.customers.customer_id, customer_name, customer_email, location @@ -167,14 +167,14 @@ WITH employeeperformance AS ( SELECT employee_id, AVG(rating) AS average_rating - FROM "metalake_demo.catalog_pg1".hr.employee_performance + FROM "metalake_demo.catalog_postgres".hr.employee_performance GROUP BY employee_id ), employeesales AS ( SELECT employee_id, SUM(total_amount) AS total_sales - FROM "metalake_demo.catalog_demo".sales.sales + FROM "metalake_demo.catalog_hive".sales.sales GROUP BY employee_id ) From 3ece62754647eb1b54ac7a93e6844c31df363d8f Mon Sep 17 00:00:00 2001 From: Qi Yu Date: Mon, 25 Dec 2023 19:01:40 +0800 Subject: [PATCH 33/51] [#1231] Fix bugs in drop MySQL schemas (#1235) ### What changes were proposed in this pull request? - Allow dropping a database when `cascase` is `true`. - If the database is not empty and `cascade` is `false`, disable the drop operation. ### Why are the changes needed? Align the logic with Gravitino EntityStore. Fix: #1231 ### Does this PR introduce _any_ user-facing change? N/A ### How was this patch tested? Add UT `testDropMySQLDatabase` --- .../operation/MysqlDatabaseOperations.java | 24 ++++++++++-- .../test/catalog/jdbc/TestJdbcAbstractIT.java | 12 +----- .../catalog/jdbc/mysql/CatalogMysqlIT.java | 38 ++++++++++++++++++- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java index 33fd2c6f862..b2c68133e19 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/com/datastrato/gravitino/catalog/mysql/operation/MysqlDatabaseOperations.java @@ -12,6 +12,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.HashMap; import java.util.Map; import org.apache.commons.collections4.MapUtils; @@ -47,11 +48,28 @@ public String generateCreateDatabaseSql( @Override public String generateDropDatabaseSql(String databaseName, boolean cascade) { + final String dropDatabaseSql = "DROP DATABASE `" + databaseName + "`"; if (cascade) { - throw new UnsupportedOperationException( - "MySQL does not support CASCADE option for DROP DATABASE."); + return dropDatabaseSql; } - return "DROP DATABASE `" + databaseName + "`"; + + try (final Connection connection = this.dataSource.getConnection()) { + String query = "SHOW TABLES IN " + databaseName; + try (Statement statement = connection.createStatement()) { + // Execute the query and check if there exists any tables in the database + try (ResultSet resultSet = statement.executeQuery(query)) { + if (resultSet.next()) { + throw new IllegalStateException( + String.format( + "Database %s is not empty, the value of cascade should be true.", + databaseName)); + } + } + } + } catch (SQLException sqlException) { + throw this.exceptionMapper.toGravitinoException(sqlException); + } + return dropDatabaseSql; } @Override diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java index 4fb9961364d..9d5c9548e58 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/TestJdbcAbstractIT.java @@ -62,17 +62,7 @@ protected void testBaseOperation( protected static void testDropDatabase(String databaseName) { List databases; - // delete database. - UnsupportedOperationException unsupportedOperationException = - Assertions.assertThrows( - UnsupportedOperationException.class, - () -> DATABASE_OPERATIONS.delete(databaseName, true)); - Assertions.assertTrue( - unsupportedOperationException - .getMessage() - .contains("does not support CASCADE option for DROP DATABASE.")); - - Assertions.assertDoesNotThrow(() -> DATABASE_OPERATIONS.delete(databaseName, false)); + DATABASE_OPERATIONS.delete(databaseName, true); Assertions.assertThrows( NoSuchSchemaException.class, () -> DATABASE_OPERATIONS.load(databaseName)); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index a59dc2db8cc..0677ee84754 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -120,7 +120,7 @@ private static void clearTableAndSchema() { NameIdentifier[] nameIdentifiers = catalog.asTableCatalog().listTables(Namespace.of(metalakeName, catalogName, schemaName)); for (NameIdentifier nameIdentifier : nameIdentifiers) { - catalog.asTableCatalog().dropTable(nameIdentifier); + catalog.asTableCatalog().purgeTable(nameIdentifier); } catalog.asSchemas().dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), false); } @@ -456,4 +456,40 @@ void testAlterAndDropMysqlTable() { catalog.asTableCatalog().dropTable(tableIdentifier); }); } + + @Test + void testDropMySQLDatabase() { + String schemaName = GravitinoITUtils.genRandomName("mysql_schema").toLowerCase(); + String tableName = GravitinoITUtils.genRandomName("mysql_table").toLowerCase(); + + catalog + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), + "Created by gravitino client", + ImmutableMap.builder().build()); + + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + createColumns(), + "Created by gravitino client", + ImmutableMap.builder().build()); + + // Try to drop a database, and cascade equals to false, it should not be allowed. + catalog.asSchemas().dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), false); + // Check the database still exists + catalog.asSchemas().loadSchema(NameIdentifier.of(metalakeName, catalogName, schemaName)); + + // Try to drop a database, and cascade equals to true, it should be allowed. + catalog.asSchemas().dropSchema(NameIdentifier.of(metalakeName, catalogName, schemaName), true); + // Check database has been dropped + Assertions.assertThrows( + NoSuchSchemaException.class, + () -> + catalog + .asSchemas() + .loadSchema(NameIdentifier.of(metalakeName, catalogName, schemaName))); + } } From 89dd66a8fbff8f0978e3cf6e6f3dae4876d53e9a Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Fri, 22 Dec 2023 16:45:54 +0800 Subject: [PATCH 34/51] [ISSUE-1206] improvement: Use the real user for audit information --- .../gravitino/auth/AuthConstants.java | 2 + .../gravitino/PrincipalContext.java | 38 ++++ .../gravitino/catalog/CatalogManager.java | 6 +- .../catalog/CatalogOperationDispatcher.java | 14 +- .../gravitino/meta/MetalakeManager.java | 3 +- .../test/catalog/hive/CatalogHiveIT.java | 42 ++-- .../jdbc/mysql/AuditCatalogMysqlIT.java | 183 ++++++++++++++++++ .../jdbc/postgresql/CatalogPostgreSqlIT.java | 22 ++- .../lakehouse/iceberg/CatalogIcebergIT.java | 39 ++-- .../integration/test/client/AuditIT.java | 53 +++++ .../integration/test/util/AbstractIT.java | 5 + .../server/auth/AuthenticationFilter.java | 4 +- .../gravitino/server/web/Utils.java | 10 + .../server/web/rest/CatalogOperations.java | 5 +- .../server/web/rest/MetalakeOperations.java | 5 +- .../server/web/rest/SchemaOperations.java | 5 +- .../server/web/rest/TableOperations.java | 5 +- 17 files changed, 382 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/com/datastrato/gravitino/PrincipalContext.java create mode 100644 integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java create mode 100644 integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java diff --git a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java index 34eb81bc188..29a1ee27242 100644 --- a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java +++ b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java @@ -13,4 +13,6 @@ public interface AuthConstants { String AUTHORIZATION_BASIC_HEADER = "Basic "; String ANONYMOUS_USER = "anonymous"; + + String AuthenticatedPrincipalAttributeName = AuthConstants.class.getName() + "-principal"; } diff --git a/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java b/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java new file mode 100644 index 00000000000..0616bac11fb --- /dev/null +++ b/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java @@ -0,0 +1,38 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino; + +import org.apache.commons.lang3.StringUtils; + +public class PrincipalContext implements AutoCloseable { + + private String currentUser; + + private static final ThreadLocal context = new ThreadLocal<>(); + + public static PrincipalContext get() { + return context.get(); + } + + public String getCurrentUser() { + if (StringUtils.isBlank(currentUser)) { + return "gravitino"; + } + return currentUser; + } + + public static PrincipalContext createPrincipalContext(String currentUser) { + PrincipalContext principalContext = new PrincipalContext(); + principalContext.currentUser = currentUser; + context.set(principalContext); + return principalContext; + } + + @Override + public void close() throws Exception { + context.remove(); + } +} diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 9fb32d2c9b3..e9ce3a8bd98 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -18,6 +18,7 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.SupportsCatalogs; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; @@ -261,7 +262,7 @@ public Catalog createCatalog( .withProperties(StringIdentifier.addToProperties(stringId, mergedConfig)) .withAuditInfo( new AuditInfo.Builder() - .withCreator("gravitino") /* TODO. Should change to real user */ + .withCreator(PrincipalContext.get().getCurrentUser()) .withCreateTime(Instant.now()) .build()) .build(); @@ -366,8 +367,7 @@ public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) new AuditInfo.Builder() .withCreator(catalog.auditInfo().creator()) .withCreateTime(catalog.auditInfo().createTime()) - .withLastModifier( - catalog.auditInfo().creator()) /* TODO. We should use real user */ + .withLastModifier(PrincipalContext.get().getCurrentUser()) .withLastModifiedTime(Instant.now()) .build(); newCatalogBuilder.withAuditInfo(newInfo); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java index 2409cefd331..e1dddd7016d 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java @@ -12,6 +12,7 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.catalog.rel.EntityCombinedSchema; import com.datastrato.gravitino.catalog.rel.EntityCombinedTable; @@ -149,8 +150,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map client.getTable(schemaName, ALTER_TABLE_NAME)); @@ -897,12 +902,19 @@ public void testAlterSchema() throws TException, InterruptedException { GravitinoMetaLake metalake = client.loadMetalake(NameIdentifier.of(metalakeName)); Catalog catalog = metalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName)); - catalog - .asSchemas() - .alterSchema( - ident, - SchemaChange.removeProperty("key1"), - SchemaChange.setProperty("key2", "val2-alter")); + Schema schema = catalog.asSchemas().loadSchema(ident); + Assertions.assertNull(schema.auditInfo().lastModifier()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().creator()); + schema = + catalog + .asSchemas() + .alterSchema( + ident, + SchemaChange.removeProperty("key1"), + SchemaChange.setProperty("key2", "val2-alter")); + + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().lastModifier()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().creator()); Map properties2 = catalog.asSchemas().loadSchema(ident).properties(); Assertions.assertFalse(properties2.containsKey("key1")); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java new file mode 100644 index 00000000000..9f9a5fdfcff --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java @@ -0,0 +1,183 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.integration.test.catalog.jdbc.mysql; + +import com.datastrato.gravitino.Catalog; +import com.datastrato.gravitino.CatalogChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.auth.AuthenticatorType; +import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; +import com.datastrato.gravitino.client.GravitinoMetaLake; +import com.datastrato.gravitino.dto.rel.ColumnDTO; +import com.datastrato.gravitino.integration.test.catalog.jdbc.mysql.service.MysqlService; +import com.datastrato.gravitino.integration.test.catalog.jdbc.utils.JdbcDriverDownloader; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.integration.test.util.ITUtils; +import com.datastrato.gravitino.rel.Schema; +import com.datastrato.gravitino.rel.Table; +import com.datastrato.gravitino.rel.TableChange; +import com.datastrato.gravitino.rel.types.Types; +import com.datastrato.gravitino.server.auth.OAuthConfig; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Map; +import org.apache.commons.lang.math.RandomUtils; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MySQLContainer; + +@Tag("gravitino-docker-it") +public class AuditCatalogMysqlIT extends AbstractIT { + + public static final String metalakeName = GravitinoITUtils.genRandomName("audit_mysql_metalake"); + private static final String expectUser = System.getProperty("user.name"); + public static final String DOWNLOAD_JDBC_DRIVER_URL = + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/mysql-connector-java-8.0.27.jar"; + public static final String mysqlImageName = "mysql:8.0"; + protected static final String TEST_DB_NAME = RandomUtils.nextInt(10000) + "_test_db"; + private static final String provider = "jdbc-mysql"; + + private static MysqlService mysqlService; + private static MySQLContainer MYSQL_CONTAINER; + private static GravitinoMetaLake metalake; + + @BeforeAll + public static void startIntegrationTest() throws Exception { + Map configs = Maps.newHashMap(); + configs.put(OAuthConfig.AUTHENTICATOR.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase()); + registerCustomConfigs(configs); + AbstractIT.startIntegrationTest(); + + if (!ITUtils.EMBEDDED_TEST_MODE.equals(testMode)) { + String gravitinoHome = System.getenv("GRAVITINO_HOME"); + Path tmpPath = Paths.get(gravitinoHome, "/catalogs/jdbc-mysql/libs"); + JdbcDriverDownloader.downloadJdbcDriver(DOWNLOAD_JDBC_DRIVER_URL, tmpPath.toString()); + } + + MYSQL_CONTAINER = + new MySQLContainer<>(mysqlImageName) + .withDatabaseName(TEST_DB_NAME) + .withUsername("root") + .withPassword("root"); + MYSQL_CONTAINER.start(); + mysqlService = new MysqlService(MYSQL_CONTAINER); + createMetalake(); + } + + @AfterAll + public static void stopIntegrationTest() throws IOException, InterruptedException { + AbstractIT.stopIntegrationTest(); + client.dropMetalake(NameIdentifier.of(metalakeName)); + mysqlService.close(); + MYSQL_CONTAINER.stop(); + } + + @Test + public void testAuditCatalog() throws Exception { + String catalogName = GravitinoITUtils.genRandomName("audit_mysql_catalog"); + Catalog catalog = createCatalog(catalogName); + Assertions.assertEquals(expectUser, catalog.auditInfo().creator()); + Assertions.assertNull(catalog.auditInfo().lastModifier()); + catalog = + metalake.alterCatalog( + NameIdentifier.of(metalakeName, catalogName), + CatalogChange.setProperty("key1", "value1")); + Assertions.assertEquals(expectUser, catalog.auditInfo().creator()); + Assertions.assertEquals(expectUser, catalog.auditInfo().lastModifier()); + } + + @Test + public void testAuditSchema() throws Exception { + String catalogName = GravitinoITUtils.genRandomName("audit_mysql_schema_catalog"); + String schemaName = GravitinoITUtils.genRandomName("audit_mysql_schema"); + Catalog catalog = createCatalog(catalogName); + NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName); + Map prop = Maps.newHashMap(); + Schema schema = catalog.asSchemas().createSchema(ident, "comment", prop); + Assertions.assertEquals(expectUser, schema.auditInfo().creator()); + Assertions.assertNull(schema.auditInfo().lastModifier()); + } + + @Test + public void testAuditTable() throws Exception { + String catalogName = GravitinoITUtils.genRandomName("audit_mysql_table_catalog"); + String schemaName = GravitinoITUtils.genRandomName("audit_mysql_table_schma"); + String tableName = GravitinoITUtils.genRandomName("audit_mysql_table"); + Catalog catalog = createCatalog(catalogName); + Map properties = Maps.newHashMap(); + + ColumnDTO col1 = + new ColumnDTO.Builder() + .withName("col_1") + .withDataType(Types.IntegerType.get()) + .withComment("col_1_comment") + .build(); + + catalog + .asSchemas() + .createSchema( + NameIdentifier.of(metalakeName, catalogName, schemaName), "comment", properties); + Table table = + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + new ColumnDTO[] {col1}, + "comment", + properties); + Assertions.assertEquals(expectUser, table.auditInfo().creator()); + Assertions.assertNull(table.auditInfo().lastModifier()); + table = + catalog + .asTableCatalog() + .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); + catalog + .asTableCatalog() + .alterTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get())); + Assertions.assertEquals(expectUser, table.auditInfo().creator()); + Assertions.assertEquals(expectUser, table.auditInfo().lastModifier()); + } + + private static Catalog createCatalog(String catalogName) { + Map catalogProperties = Maps.newHashMap(); + + catalogProperties.put( + JdbcConfig.JDBC_URL.getKey(), + StringUtils.substring( + MYSQL_CONTAINER.getJdbcUrl(), 0, MYSQL_CONTAINER.getJdbcUrl().lastIndexOf("/"))); + catalogProperties.put(JdbcConfig.JDBC_DRIVER.getKey(), MYSQL_CONTAINER.getDriverClassName()); + catalogProperties.put(JdbcConfig.USERNAME.getKey(), MYSQL_CONTAINER.getUsername()); + catalogProperties.put(JdbcConfig.PASSWORD.getKey(), MYSQL_CONTAINER.getPassword()); + + return metalake.createCatalog( + NameIdentifier.of(metalakeName, catalogName), + Catalog.Type.RELATIONAL, + provider, + "comment", + catalogProperties); + } + + private static void createMetalake() { + GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes(); + Assertions.assertEquals(0, gravitinoMetaLakes.length); + + GravitinoMetaLake createdMetalake = + client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap()); + GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName)); + Assertions.assertEquals(createdMetalake, loadMetalake); + metalake = loadMetalake; + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index f3733445957..c829393849c 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.dto.rel.ColumnDTO; @@ -324,13 +325,16 @@ void testCreateAndLoadPostgreSqlTable() { @Test void testAlterAndDropPostgreSqlTable() { ColumnDTO[] columns = createColumns(); - catalog - .asTableCatalog() - .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - columns, - table_comment, - createProperties()); + Table table = + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + columns, + table_comment, + createProperties()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); + Assertions.assertNull(table.auditInfo().lastModifier()); Assertions.assertThrows( IllegalArgumentException.class, () -> { @@ -348,6 +352,8 @@ void testAlterAndDropPostgreSqlTable() { .alterTable( NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), TableChange.rename(alertTableName)); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().lastModifier()); // update table catalog @@ -360,7 +366,7 @@ void testAlterAndDropPostgreSqlTable() { TableChange.updateColumnType( new String[] {POSTGRESQL_COL_NAME1}, Types.IntegerType.get())); - Table table = + table = catalog .asTableCatalog() .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName)); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java index b679eecf75d..1f9ad9058e7 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/lakehouse/iceberg/CatalogIcebergIT.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.Catalog; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig; import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata; @@ -487,14 +488,17 @@ void testListAndDropIcebergTable() { @Test public void testAlterIcebergTable() { ColumnDTO[] columns = createColumns(); - catalog - .asTableCatalog() - .createTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - columns, - table_comment, - createProperties(), - new Partitioning[] {IdentityPartitioningDTO.of(columns[0].name())}); + Table table = + catalog + .asTableCatalog() + .createTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + columns, + table_comment, + createProperties(), + new Partitioning[] {IdentityPartitioningDTO.of(columns[0].name())}); + Assertions.assertNull(table.auditInfo().lastModifier()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); Assertions.assertThrows( IllegalArgumentException.class, () -> { @@ -506,11 +510,14 @@ public void testAlterIcebergTable() { TableChange.updateComment(table_comment + "_new")); }); - catalog - .asTableCatalog() - .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - TableChange.rename(alertTableName)); + table = + catalog + .asTableCatalog() + .alterTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + TableChange.rename(alertTableName)); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().lastModifier()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); catalog .asTableCatalog() @@ -525,7 +532,7 @@ public void testAlterIcebergTable() { TableChange.updateColumnType( new String[] {ICEBERG_COL_NAME1}, Types.IntegerType.get())); - Table table = + table = catalog .asTableCatalog() .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, alertTableName)); @@ -787,6 +794,8 @@ public void testOperatorSchemeProperties() { prop.remove(IcebergSchemaPropertiesMetadata.COMMENT); catalog.asSchemas().createSchema(ident, schema_comment, prop); Schema loadSchema = catalog.asSchemas().loadSchema(ident); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, loadSchema.auditInfo().creator()); + Assertions.assertNull(loadSchema.auditInfo().lastModifier()); Assertions.assertFalse( loadSchema.properties().containsKey(IcebergSchemaPropertiesMetadata.COMMENT)); prop.forEach((key, value) -> Assertions.assertEquals(loadSchema.properties().get(key), value)); @@ -800,6 +809,8 @@ public void testOperatorSchemeProperties() { () -> catalog.asSchemas().alterSchema(ident, SchemaChange.setProperty("comment-test", "v1"))); Schema schema = catalog.asSchemas().loadSchema(ident); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().creator()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, schema.auditInfo().lastModifier()); Assertions.assertEquals("v1", schema.properties().get("comment-test")); // drop diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java new file mode 100644 index 00000000000..95cf56b26fe --- /dev/null +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + +package com.datastrato.gravitino.integration.test.client; + +import com.datastrato.gravitino.MetalakeChange; +import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.auth.AuthenticatorType; +import com.datastrato.gravitino.client.GravitinoMetaLake; +import com.datastrato.gravitino.integration.test.util.AbstractIT; +import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; +import com.datastrato.gravitino.server.auth.OAuthConfig; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class AuditIT extends AbstractIT { + + private static final String expectUser = System.getProperty("user.name"); + + @BeforeAll + public static void startIntegrationTest() throws Exception { + Map configs = Maps.newHashMap(); + configs.put(OAuthConfig.AUTHENTICATOR.getKey(), AuthenticatorType.SIMPLE.name().toLowerCase()); + registerCustomConfigs(configs); + AbstractIT.startIntegrationTest(); + } + + @Test + public void testAuditMetalake() throws Exception { + String metalakeAuditName = GravitinoITUtils.genRandomName("metalakeAudit"); + String newName = GravitinoITUtils.genRandomName("newmetaname"); + + GravitinoMetaLake metaLake = + client.createMetalake( + NameIdentifier.parse(metalakeAuditName), "metalake A comment", Collections.emptyMap()); + Assertions.assertEquals(expectUser, metaLake.auditInfo().creator()); + Assertions.assertNull(metaLake.auditInfo().lastModifier()); + MetalakeChange[] changes = + new MetalakeChange[] { + MetalakeChange.rename(newName), MetalakeChange.updateComment("new metalake comment") + }; + metaLake = client.alterMetalake(NameIdentifier.of(metalakeAuditName), changes); + Assertions.assertEquals(expectUser, metaLake.auditInfo().creator()); + Assertions.assertEquals(expectUser, metaLake.auditInfo().lastModifier()); + client.dropMetalake(NameIdentifier.parse(newName)); + } +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java index 2cbf71c96b6..b9651be9ce1 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/util/AbstractIT.java @@ -125,6 +125,11 @@ public static void startIntegrationTest() throws Exception { .toLowerCase() .equals(customConfigs.get(OAuthConfig.AUTHENTICATOR.getKey()))) { client = GravitinoClient.builder(uri).withOAuth(mockDataProvider).build(); + } else if (AuthenticatorType.SIMPLE + .name() + .toLowerCase() + .equals(customConfigs.get(OAuthConfig.AUTHENTICATOR.getKey()))) { + client = GravitinoClient.builder(uri).withSimpleAuth().build(); } else { client = GravitinoClient.builder(uri).build(); } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java index 609d09279a2..94318403796 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java @@ -9,6 +9,7 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.security.Principal; import java.util.Enumeration; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -52,7 +53,8 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha authData = headerData.nextElement().getBytes(StandardCharsets.UTF_8); } if (authenticator.isDataFromToken()) { - authenticator.authenticateToken(authData); + Principal principal = authenticator.authenticateToken(authData); + request.setAttribute(AuthConstants.AuthenticatedPrincipalAttributeName, principal); } chain.doFilter(request, response); } catch (UnauthorizedException ue) { diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java index 8921e511e6a..4ba7660e160 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java @@ -4,7 +4,10 @@ */ package com.datastrato.gravitino.server.web; +import com.datastrato.gravitino.PrincipalContext; +import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.dto.responses.ErrorResponse; +import com.datastrato.gravitino.server.auth.UserPrincipal; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.MediaType; @@ -94,4 +97,11 @@ public static Response nonEmpty(String type, String message, Throwable throwable .type(MediaType.APPLICATION_JSON) .build(); } + + public static PrincipalContext createUserPrincipalContext(HttpServletRequest httpRequest) { + return PrincipalContext.createPrincipalContext( + ((UserPrincipal) + httpRequest.getAttribute(AuthConstants.AuthenticatedPrincipalAttributeName)) + .getName()); + } } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java index 32d9df230ab..c7aa695dfcb 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.CatalogChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.dto.requests.CatalogCreateRequest; import com.datastrato.gravitino.dto.requests.CatalogUpdateRequest; @@ -66,7 +67,7 @@ public Response listCatalogs(@PathParam("metalake") String metalake) { @Produces("application/vnd.gravitino.v1+json") public Response createCatalog( @PathParam("metalake") String metalake, CatalogCreateRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofCatalog(metalake, request.getName()); Catalog catalog = @@ -107,7 +108,7 @@ public Response alterCatalog( @PathParam("metalake") String metalakeName, @PathParam("catalog") String catalogName, CatalogUpdatesRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofCatalog(metalakeName, catalogName); CatalogChange[] changes = diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java index cb15d683d78..e790d16df08 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java @@ -9,6 +9,7 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.dto.MetalakeDTO; import com.datastrato.gravitino.dto.requests.MetalakeCreateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdateRequest; @@ -76,7 +77,7 @@ public Response listMetalakes() { @Timed(name = "create-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-metalake", absolute = true) public Response createMetalake(MetalakeCreateRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofMetalake(request.getName()); BaseMetalake metalake = @@ -111,7 +112,7 @@ public Response loadMetalake(@PathParam("name") String metalakeName) { @ResponseMetered(name = "alter-metalake", absolute = true) public Response alterMetalake( @PathParam("name") String metalakeName, MetalakeUpdatesRequest updatesRequest) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { updatesRequest.validate(); NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); MetalakeChange[] changes = diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java index 530d55517fa..2a6af235aa7 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java @@ -8,6 +8,7 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogOperationDispatcher; import com.datastrato.gravitino.dto.requests.SchemaCreateRequest; import com.datastrato.gravitino.dto.requests.SchemaUpdateRequest; @@ -78,7 +79,7 @@ public Response createSchema( @PathParam("metalake") String metalake, @PathParam("catalog") String catalog, SchemaCreateRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, request.getName()); Schema schema = dispatcher.createSchema(ident, request.getComment(), request.getProperties()); @@ -119,7 +120,7 @@ public Response alterSchema( @PathParam("catalog") String catalog, @PathParam("schema") String schema, SchemaUpdatesRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); SchemaChange[] changes = diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java index 1388613b6dc..e8fcfe9a4e0 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java @@ -11,6 +11,7 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogOperationDispatcher; import com.datastrato.gravitino.dto.requests.TableCreateRequest; import com.datastrato.gravitino.dto.requests.TableUpdateRequest; @@ -80,7 +81,7 @@ public Response createTable( @PathParam("catalog") String catalog, @PathParam("schema") String schema, TableCreateRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, request.getName()); @@ -132,7 +133,7 @@ public Response alterTable( @PathParam("schema") String schema, @PathParam("table") String table, TableUpdatesRequest request) { - try { + try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { request.validate(); NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); TableChange[] changes = From 06f8d47c5aa2e0d0c765a54ddb8a9978b68a31c3 Mon Sep 17 00:00:00 2001 From: Clearvive <143773256+Clearvive@users.noreply.github.com> Date: Tue, 26 Dec 2023 19:26:30 +0800 Subject: [PATCH 35/51] [#1253] bugfix(jdbc): fix jdbc catalog store empty audit info issue (#1257) ### What changes were proposed in this pull request? Mysql&Postgres catalog store empty audit info fix. ### Why are the changes needed? Fix: #1253 ### Does this PR introduce _any_ user-facing change? Users will receive audit information ### How was this patch tested? IT --------- Co-authored-by: Clearvive --- .../gravitino/catalog/jdbc/JdbcCatalogOperations.java | 4 ++-- .../java/com/datastrato/gravitino/StringIdentifier.java | 9 +++++---- .../com/datastrato/gravitino/catalog/CatalogManager.java | 2 +- .../gravitino/catalog/CatalogOperationDispatcher.java | 6 ++++-- .../java/com/datastrato/gravitino/meta/BaseMetalake.java | 2 +- .../com/datastrato/gravitino/meta/MetalakeManager.java | 2 +- .../com/datastrato/gravitino/TestStringIdentifier.java | 8 ++++---- .../test/catalog/jdbc/mysql/CatalogMysqlIT.java | 5 +++++ .../catalog/jdbc/postgresql/CatalogPostgreSqlIT.java | 5 +++++ 9 files changed, 28 insertions(+), 15 deletions(-) diff --git a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java index 404e980f8ab..61d6a520c88 100644 --- a/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java +++ b/catalogs/catalog-jdbc-common/src/main/java/com/datastrato/gravitino/catalog/jdbc/JdbcCatalogOperations.java @@ -203,7 +203,7 @@ public JdbcSchema loadSchema(NameIdentifier ident) throws NoSuchSchemaException } Map properties = load.properties() == null ? Maps.newHashMap() : Maps.newHashMap(load.properties()); - StringIdentifier.addToProperties(id, properties); + StringIdentifier.newPropertiesWithId(id, properties); return new JdbcSchema.Builder() .withAuditInfo(load.auditInfo()) .withName(load.name()) @@ -276,7 +276,7 @@ public Table loadTable(NameIdentifier tableIdent) throws NoSuchTableException { } Map properties = load.properties() == null ? Maps.newHashMap() : Maps.newHashMap(load.properties()); - StringIdentifier.addToProperties(id, properties); + properties = StringIdentifier.newPropertiesWithId(id, properties); return new JdbcTable.Builder() .withAuditInfo(load.auditInfo()) .withName(tableName) diff --git a/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java b/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java index 942ce27ab53..6ece84a80dd 100644 --- a/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java +++ b/core/src/main/java/com/datastrato/gravitino/StringIdentifier.java @@ -8,6 +8,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import java.util.Collections; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -88,7 +89,7 @@ public String toString() { * @param properties the properties to add the string identifier to * @return the properties with the string identifier added */ - public static Map addToProperties( + public static Map newPropertiesWithId( StringIdentifier stringId, Map properties) { if (properties == null) { return ImmutableMap.of(ID_KEY, stringId.toString()); @@ -100,7 +101,7 @@ public static Map addToProperties( + "ignore adding the identifier to the properties", ID_KEY, properties.get(ID_KEY)); - return properties; + return Collections.unmodifiableMap(properties); } return ImmutableMap.builder() @@ -115,13 +116,13 @@ public static Map addToProperties( * @param properties the properties to remove the string identifier from. * @return the properties with the string identifier removed. */ - public static Map removeIdFromProperties(Map properties) { + public static Map newPropertiesWithoutId(Map properties) { if (properties == null) { return null; } if (!properties.containsKey(ID_KEY)) { - return properties; + return Collections.unmodifiableMap(properties); } Map copy = Maps.newHashMap(properties); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 1ad192ceacf..62b6b5dcfcd 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -258,7 +258,7 @@ public Catalog createCatalog( .withType(type) .withProvider(provider) .withComment(comment) - .withProperties(StringIdentifier.addToProperties(stringId, mergedConfig)) + .withProperties(StringIdentifier.newPropertiesWithId(stringId, mergedConfig)) .withAuditInfo( new AuditInfo.Builder() .withCreator("gravitino") /* TODO. Should change to real user */ diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java index 2409cefd331..6fb28d9f548 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java @@ -127,7 +127,8 @@ public Schema createSchema(NameIdentifier ident, String comment, Map updatedProperties = StringIdentifier.addToProperties(stringId, properties); + Map updatedProperties = + StringIdentifier.newPropertiesWithId(stringId, properties); doWithCatalog( catalogIdent, @@ -421,7 +422,8 @@ public Table createTable( // StringIdentifier to make sure only when the operation is successful, the related // TableEntity will be visible. StringIdentifier stringId = StringIdentifier.fromId(uid); - Map updatedProperties = StringIdentifier.addToProperties(stringId, properties); + Map updatedProperties = + StringIdentifier.newPropertiesWithId(stringId, properties); doWithCatalog( catalogIdent, diff --git a/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java b/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java index f6de4bf2a54..bbe9e118ef3 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/BaseMetalake.java @@ -127,7 +127,7 @@ public EntityType type() { */ @Override public Map properties() { - return StringIdentifier.removeIdFromProperties(properties); + return StringIdentifier.newPropertiesWithoutId(properties); } /** Builder class for creating instances of {@link BaseMetalake}. */ diff --git a/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java b/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java index 5ca502f367a..2ff1ab8c7d9 100644 --- a/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java +++ b/core/src/main/java/com/datastrato/gravitino/meta/MetalakeManager.java @@ -103,7 +103,7 @@ public BaseMetalake createMetalake( .withId(uid) .withName(ident.name()) .withComment(comment) - .withProperties(StringIdentifier.addToProperties(stringId, properties)) + .withProperties(StringIdentifier.newPropertiesWithId(stringId, properties)) .withVersion(SchemaVersion.V_0_1) .withAuditInfo( new AuditInfo.Builder() diff --git a/core/src/test/java/com/datastrato/gravitino/TestStringIdentifier.java b/core/src/test/java/com/datastrato/gravitino/TestStringIdentifier.java index d480c7d24e9..a4e8d2130d2 100644 --- a/core/src/test/java/com/datastrato/gravitino/TestStringIdentifier.java +++ b/core/src/test/java/com/datastrato/gravitino/TestStringIdentifier.java @@ -74,14 +74,14 @@ public void testAddStringIdToProperties() { StringIdentifier stringId = StringIdentifier.fromId(uid); Map prop = ImmutableMap.of("key1", "value1", "key2", "value2"); - Map propWithId = StringIdentifier.addToProperties(stringId, prop); + Map propWithId = StringIdentifier.newPropertiesWithId(stringId, prop); Assertions.assertTrue(propWithId.containsKey(StringIdentifier.ID_KEY)); Assertions.assertEquals(stringId.toString(), propWithId.get(StringIdentifier.ID_KEY)); Assertions.assertEquals("value1", propWithId.get("key1")); Assertions.assertEquals("value2", propWithId.get("key2")); // Test if the input properties is null - Map propWithId1 = StringIdentifier.addToProperties(stringId, null); + Map propWithId1 = StringIdentifier.newPropertiesWithId(stringId, null); Assertions.assertTrue(propWithId1.containsKey(StringIdentifier.ID_KEY)); Assertions.assertEquals(stringId.toString(), propWithId1.get(StringIdentifier.ID_KEY)); Assertions.assertEquals(1, propWithId1.size()); @@ -90,7 +90,7 @@ public void testAddStringIdToProperties() { Map prop1 = ImmutableMap.of("k1", "v1", StringIdentifier.ID_KEY, stringId.toString()); StringIdentifier newStringId = StringIdentifier.fromId(12341234L); - Map propWithId2 = StringIdentifier.addToProperties(newStringId, prop1); + Map propWithId2 = StringIdentifier.newPropertiesWithId(newStringId, prop1); Assertions.assertEquals(stringId.toString(), propWithId2.get(StringIdentifier.ID_KEY)); } @@ -100,7 +100,7 @@ public void testGetStringIdFromProperties() { StringIdentifier stringId = StringIdentifier.fromId(uid); Map prop = ImmutableMap.of("key1", "value1", "key2", "value2"); - Map propWithId = StringIdentifier.addToProperties(stringId, prop); + Map propWithId = StringIdentifier.newPropertiesWithId(stringId, prop); StringIdentifier stringIdFromProp = StringIdentifier.fromProperties(propWithId); Assertions.assertEquals(stringId.id(), stringIdFromProp.id()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java index 0677ee84754..1f0c2261989 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/CatalogMysqlIT.java @@ -375,6 +375,11 @@ void testAlterAndDropMysqlTable() { Assertions.assertEquals("col_4", table.columns()[3].name()); Assertions.assertEquals(Types.StringType.get(), table.columns()[3].dataType()); Assertions.assertNull(table.columns()[3].comment()); + Assertions.assertNotNull(table.auditInfo()); + Assertions.assertNotNull(table.auditInfo().createTime()); + Assertions.assertNotNull(table.auditInfo().creator()); + Assertions.assertNotNull(table.auditInfo().lastModifiedTime()); + Assertions.assertNotNull(table.auditInfo().lastModifier()); ColumnDTO col1 = new ColumnDTO.Builder() diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index f3733445957..fe6c6416445 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -384,6 +384,11 @@ void testAlterAndDropPostgreSqlTable() { Assertions.assertEquals(Types.StringType.get(), table.columns()[3].dataType()); Assertions.assertNull(table.columns()[3].comment()); Assertions.assertTrue(table.columns()[3].nullable()); + Assertions.assertNotNull(table.auditInfo()); + Assertions.assertNotNull(table.auditInfo().createTime()); + Assertions.assertNotNull(table.auditInfo().creator()); + Assertions.assertNotNull(table.auditInfo().lastModifiedTime()); + Assertions.assertNotNull(table.auditInfo().lastModifier()); ColumnDTO col1 = new ColumnDTO.Builder() From b20c7fdda33ca88f3e103452a58e0458526cba8b Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Tue, 26 Dec 2023 19:59:44 +0800 Subject: [PATCH 36/51] fix --- .../gravitino/catalog/TestCatalogManager.java | 7 +++++++ .../catalog/TestCatalogOperationDispatcher.java | 9 ++++++++- .../gravitino/meta/TestMetalakeManager.java | 10 ++++++++-- .../test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java | 9 +++------ .../catalog/jdbc/postgresql/CatalogPostgreSqlIT.java | 11 ++++++----- .../gravitino/integration/test/client/MetalakeIT.java | 5 +++-- .../server/web/rest/TestTableOperations.java | 6 ++++-- 7 files changed, 39 insertions(+), 18 deletions(-) diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java index cfa6a24be1e..fed19c99e31 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java @@ -12,6 +12,7 @@ import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.TestEntityStore; import com.datastrato.gravitino.TestEntityStore.InMemoryEntityStore; @@ -49,6 +50,8 @@ public class TestCatalogManager { private static String provider = "test"; + private static PrincipalContext context; + private static BaseMetalake metalakeEntity = new BaseMetalake.Builder() .withId(1L) @@ -71,6 +74,7 @@ public static void setUp() throws IOException { catalogManager = new CatalogManager(config, entityStore, new RandomIdGenerator()); catalogManager = Mockito.spy(catalogManager); + context = PrincipalContext.createPrincipalContext("test"); } @BeforeEach @@ -91,6 +95,9 @@ public static void tearDown() throws Exception { catalogManager.close(); catalogManager = null; } + if (context != null) { + context.close(); + } } @Test diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java index 4320ded4447..1c5a7350815 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java @@ -23,6 +23,7 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.TestColumn; import com.datastrato.gravitino.TestEntityStore; @@ -72,6 +73,8 @@ public class TestCatalogOperationDispatcher { private static CatalogOperationDispatcher dispatcher; + private static PrincipalContext context; + @BeforeAll public static void setUp() throws IOException { config = new Config(false) {}; @@ -96,11 +99,12 @@ public static void setUp() throws IOException { NameIdentifier ident = NameIdentifier.of(metalake, catalog); Map props = ImmutableMap.of(); + context = PrincipalContext.createPrincipalContext("gravitino"); catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, "test", "comment", props); } @AfterAll - public static void tearDown() throws IOException { + public static void tearDown() throws Exception { if (entityStore != null) { entityStore.close(); entityStore = null; @@ -110,6 +114,9 @@ public static void tearDown() throws IOException { catalogManager.close(); catalogManager = null; } + if (context != null) { + context.close(); + } } @BeforeEach diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java index eede4fb2c69..d91bc394523 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java @@ -8,6 +8,7 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.TestEntityStore; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; @@ -15,7 +16,6 @@ import com.datastrato.gravitino.storage.RandomIdGenerator; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import java.io.IOException; import java.util.Map; import java.util.Set; import org.junit.jupiter.api.AfterAll; @@ -31,6 +31,8 @@ public class TestMetalakeManager { private static Config config; + private static PrincipalContext context; + @BeforeAll public static void setUp() { config = new Config(false) {}; @@ -40,14 +42,18 @@ public static void setUp() { entityStore.setSerDe(null); metalakeManager = new MetalakeManager(entityStore, new RandomIdGenerator()); + context = PrincipalContext.createPrincipalContext("test"); } @AfterAll - public static void tearDown() throws IOException { + public static void tearDown() throws Exception { if (entityStore != null) { entityStore.close(); entityStore = null; } + if (context != null) { + context.close(); + } } @Test diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java index 9f9a5fdfcff..30587e82295 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/mysql/AuditCatalogMysqlIT.java @@ -141,12 +141,9 @@ public void testAuditTable() throws Exception { table = catalog .asTableCatalog() - .loadTable(NameIdentifier.of(metalakeName, catalogName, schemaName, tableName)); - catalog - .asTableCatalog() - .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get())); + .alterTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + TableChange.addColumn(new String[] {"col_4"}, Types.StringType.get())); Assertions.assertEquals(expectUser, table.auditInfo().creator()); Assertions.assertEquals(expectUser, table.auditInfo().lastModifier()); } diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java index 821d9f8f000..7459882da13 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/catalog/jdbc/postgresql/CatalogPostgreSqlIT.java @@ -347,11 +347,12 @@ void testAlterAndDropPostgreSqlTable() { }); // rename table - catalog - .asTableCatalog() - .alterTable( - NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), - TableChange.rename(alertTableName)); + table = + catalog + .asTableCatalog() + .alterTable( + NameIdentifier.of(metalakeName, catalogName, schemaName, tableName), + TableChange.rename(alertTableName)); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().creator()); Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, table.auditInfo().lastModifier()); diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java index e221b3725dc..839496b472b 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/MetalakeIT.java @@ -11,6 +11,7 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.exceptions.IllegalNameIdentifierException; import com.datastrato.gravitino.exceptions.IllegalNamespaceException; @@ -104,7 +105,7 @@ public void testAlterMetalake() { GravitinoMetaLake metaLake = client.alterMetalake(NameIdentifier.of(metalakeNameA), changes); assertEquals(newName, metaLake.name()); assertEquals("new metalake comment", metaLake.comment()); - assertEquals("gravitino", metaLake.auditInfo().creator()); + assertEquals(AuthConstants.ANONYMOUS_USER, metaLake.auditInfo().creator()); // Reload metadata via new name to check if the changes are applied GravitinoMetaLake newMetalake = client.loadMetalake(NameIdentifier.of(newName)); @@ -142,7 +143,7 @@ public void testCreateMetalake() { GravitinoMetaLake metalake = client.loadMetalake(NameIdentifier.of(metalakeNameA)); assertEquals(metalakeNameA, metalake.name()); assertEquals("metalake A comment", metalake.comment()); - assertEquals("gravitino", metalake.auditInfo().creator()); + assertEquals(AuthConstants.ANONYMOUS_USER, metalake.auditInfo().creator()); // Test metalake name already exists assertThrows( diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java index c8a9d9bff72..c06ed707d86 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java @@ -12,6 +12,7 @@ import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogOperationDispatcher; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.DistributionDTO; @@ -62,8 +63,7 @@ import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.test.JerseyTest; import org.glassfish.jersey.test.TestProperties; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; public class TestTableOperations extends JerseyTest { @@ -84,6 +84,8 @@ public HttpServletRequest get() { private final String schema = "schema1"; + + @Override protected Application configure() { try { From 391042a1a9c87e774db9b588d2cf2a76c683d94a Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Tue, 26 Dec 2023 20:19:20 +0800 Subject: [PATCH 37/51] fix --- .../com/datastrato/gravitino/PrincipalContext.java | 3 ++- .../com/datastrato/gravitino/server/web/Utils.java | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java b/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java index 0616bac11fb..1f4a0feefb6 100644 --- a/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java +++ b/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino; +import com.datastrato.gravitino.auth.AuthConstants; import org.apache.commons.lang3.StringUtils; public class PrincipalContext implements AutoCloseable { @@ -19,7 +20,7 @@ public static PrincipalContext get() { public String getCurrentUser() { if (StringUtils.isBlank(currentUser)) { - return "gravitino"; + return AuthConstants.ANONYMOUS_USER; } return currentUser; } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java index 4ba7660e160..3c9ef8d5a8b 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java @@ -99,9 +99,11 @@ public static Response nonEmpty(String type, String message, Throwable throwable } public static PrincipalContext createUserPrincipalContext(HttpServletRequest httpRequest) { - return PrincipalContext.createPrincipalContext( - ((UserPrincipal) - httpRequest.getAttribute(AuthConstants.AuthenticatedPrincipalAttributeName)) - .getName()); + UserPrincipal principal = + (UserPrincipal) httpRequest.getAttribute(AuthConstants.AuthenticatedPrincipalAttributeName); + if (principal == null) { + principal = new UserPrincipal(AuthConstants.ANONYMOUS_USER); + } + return PrincipalContext.createPrincipalContext(principal.getName()); } } From ab524ab16395c5ea7e2c1bd15aa099c0fe9794f5 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Tue, 26 Dec 2023 20:21:08 +0800 Subject: [PATCH 38/51] fix --- .../gravitino/server/web/rest/TestTableOperations.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java index c06ed707d86..c8a9d9bff72 100644 --- a/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java +++ b/server/src/test/java/com/datastrato/gravitino/server/web/rest/TestTableOperations.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.Audit; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogOperationDispatcher; import com.datastrato.gravitino.dto.rel.ColumnDTO; import com.datastrato.gravitino.dto.rel.DistributionDTO; @@ -63,7 +62,8 @@ import org.glassfish.jersey.server.ResourceConfig; import org.glassfish.jersey.test.JerseyTest; import org.glassfish.jersey.test.TestProperties; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class TestTableOperations extends JerseyTest { @@ -84,8 +84,6 @@ public HttpServletRequest get() { private final String schema = "schema1"; - - @Override protected Application configure() { try { From aca126b09c76e47e4bb2c917916fdf0d19d6feec Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 10:35:20 +0800 Subject: [PATCH 39/51] Use another method --- .../datastrato/gravitino}/UserPrincipal.java | 2 +- .../gravitino/catalog/CatalogManager.java | 6 +- .../catalog/CatalogOperationDispatcher.java | 12 ++-- .../gravitino/meta/MetalakeManager.java | 4 +- .../gravitino/utils/PrincipalUtils.java | 49 ++++++++++++++++ .../gravitino/catalog/TestCatalogManager.java | 4 -- .../TestCatalogOperationDispatcher.java | 1 - .../gravitino/meta/TestMetalakeManager.java | 4 -- .../integration/test/client/AuditIT.java | 32 +++++++++++ .../server/auth/OAuth2TokenAuthenticator.java | 1 + .../server/auth/SimpleAuthenticator.java | 1 + .../server/auth/TestAuthenticationFilter.java | 1 + .../gravitino/server/web/Utils.java | 10 ++-- .../server/web/rest/CatalogOperations.java | 51 +++++++++-------- .../server/web/rest/MetalakeOperations.java | 41 ++++++++------ .../server/web/rest/SchemaOperations.java | 38 ++++++++----- .../server/web/rest/TableOperations.java | 56 +++++++++++-------- 17 files changed, 210 insertions(+), 103 deletions(-) rename {server-common/src/main/java/com/datastrato/gravitino/server/auth => core/src/main/java/com/datastrato/gravitino}/UserPrincipal.java (95%) create mode 100644 core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/UserPrincipal.java b/core/src/main/java/com/datastrato/gravitino/UserPrincipal.java similarity index 95% rename from server-common/src/main/java/com/datastrato/gravitino/server/auth/UserPrincipal.java rename to core/src/main/java/com/datastrato/gravitino/UserPrincipal.java index db7fbe321f1..a869c1de369 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/UserPrincipal.java +++ b/core/src/main/java/com/datastrato/gravitino/UserPrincipal.java @@ -3,7 +3,7 @@ * This software is licensed under the Apache License version 2. */ -package com.datastrato.gravitino.server.auth; +package com.datastrato.gravitino; import com.google.common.base.Preconditions; import java.security.Principal; diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java index 260a4436ce9..5c75d57aa56 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogManager.java @@ -18,7 +18,6 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.SupportsCatalogs; import com.datastrato.gravitino.exceptions.CatalogAlreadyExistsException; @@ -31,6 +30,7 @@ import com.datastrato.gravitino.rel.TableCatalog; import com.datastrato.gravitino.storage.IdGenerator; import com.datastrato.gravitino.utils.IsolatedClassLoader; +import com.datastrato.gravitino.utils.PrincipalUtils; import com.datastrato.gravitino.utils.ThrowableFunction; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -262,7 +262,7 @@ public Catalog createCatalog( .withProperties(StringIdentifier.newPropertiesWithId(stringId, mergedConfig)) .withAuditInfo( new AuditInfo.Builder() - .withCreator(PrincipalContext.get().getCurrentUser()) + .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) .withCreateTime(Instant.now()) .build()) .build(); @@ -367,7 +367,7 @@ public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) new AuditInfo.Builder() .withCreator(catalog.auditInfo().creator()) .withCreateTime(catalog.auditInfo().createTime()) - .withLastModifier(PrincipalContext.get().getCurrentUser()) + .withLastModifier(PrincipalUtils.getCurrentPrincipal().getName()) .withLastModifiedTime(Instant.now()) .build(); newCatalogBuilder.withAuditInfo(newInfo); diff --git a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java index 5229b50ca84..76ed6ff6203 100644 --- a/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java +++ b/core/src/main/java/com/datastrato/gravitino/catalog/CatalogOperationDispatcher.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.HasIdentifier; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.catalog.rel.EntityCombinedSchema; import com.datastrato.gravitino.catalog.rel.EntityCombinedTable; @@ -41,6 +40,7 @@ import com.datastrato.gravitino.rel.expressions.sorts.SortOrder; import com.datastrato.gravitino.rel.expressions.transforms.Transform; import com.datastrato.gravitino.storage.IdGenerator; +import com.datastrato.gravitino.utils.PrincipalUtils; import com.datastrato.gravitino.utils.ThrowableFunction; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; @@ -151,7 +151,7 @@ public Schema createSchema(NameIdentifier ident, String comment, Map T doAs(Principal principal, PrivilegedExceptionAction action) + throws Exception { + try { + Subject subject = new Subject(); + subject.getPrincipals().add(principal); + return Subject.doAs(subject, action); + } catch (PrivilegedActionException pae) { + Throwable cause = pae.getCause(); + if (!(cause instanceof Exception)) { + throw new RuntimeException( + "PrivilegedActionException with no " + + "underlying cause. Principal " + + principal.getName() + + " : " + + pae, + pae); + } + throw (Exception) cause; + } + } + + public static Principal getCurrentPrincipal() { + AccessControlContext context = AccessController.getContext(); + Subject subject = Subject.getSubject(context); + if (subject == null || subject.getPrincipals(UserPrincipal.class).isEmpty()) { + return new UserPrincipal(AuthConstants.ANONYMOUS_USER); + } + return subject.getPrincipals(UserPrincipal.class).iterator().next(); + } +} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java index fed19c99e31..5b64f61d43f 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java @@ -12,7 +12,6 @@ import com.datastrato.gravitino.Configs; import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.TestEntityStore; import com.datastrato.gravitino.TestEntityStore.InMemoryEntityStore; @@ -50,8 +49,6 @@ public class TestCatalogManager { private static String provider = "test"; - private static PrincipalContext context; - private static BaseMetalake metalakeEntity = new BaseMetalake.Builder() .withId(1L) @@ -74,7 +71,6 @@ public static void setUp() throws IOException { catalogManager = new CatalogManager(config, entityStore, new RandomIdGenerator()); catalogManager = Mockito.spy(catalogManager); - context = PrincipalContext.createPrincipalContext("test"); } @BeforeEach diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java index 1c5a7350815..b9c2eb0fc17 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java @@ -99,7 +99,6 @@ public static void setUp() throws IOException { NameIdentifier ident = NameIdentifier.of(metalake, catalog); Map props = ImmutableMap.of(); - context = PrincipalContext.createPrincipalContext("gravitino"); catalogManager.createCatalog(ident, Catalog.Type.RELATIONAL, "test", "comment", props); } diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java index d91bc394523..e336c9c7931 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java @@ -8,7 +8,6 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.TestEntityStore; import com.datastrato.gravitino.exceptions.MetalakeAlreadyExistsException; @@ -31,8 +30,6 @@ public class TestMetalakeManager { private static Config config; - private static PrincipalContext context; - @BeforeAll public static void setUp() { config = new Config(false) {}; @@ -42,7 +39,6 @@ public static void setUp() { entityStore.setSerDe(null); metalakeManager = new MetalakeManager(entityStore, new RandomIdGenerator()); - context = PrincipalContext.createPrincipalContext("test"); } @AfterAll diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java index 95cf56b26fe..6d2db391ef6 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java @@ -7,14 +7,19 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; +import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthenticatorType; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.server.auth.OAuthConfig; import com.google.common.collect.Maps; +import java.security.AccessControlContext; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.Collections; import java.util.Map; +import javax.security.auth.Subject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -50,4 +55,31 @@ public void testAuditMetalake() throws Exception { Assertions.assertEquals(expectUser, metaLake.auditInfo().lastModifier()); client.dropMetalake(NameIdentifier.parse(newName)); } + + @Test + public void testSubject() { + Subject subject = new Subject(); + subject.getPrincipals().add(new UserPrincipal("test")); + Subject.doAs( + subject, + (PrivilegedAction) + () -> { + Thread thread = + new Thread( + () -> { + AccessControlContext context = AccessController.getContext(); + Subject subject1 = Subject.getSubject(context); + UserPrincipal principal = + subject1.getPrincipals(UserPrincipal.class).iterator().next(); + System.out.println("principal name: " + principal.getName()); + }); + thread.start(); + try { + thread.join(); + } catch (Exception e) { + // e.printStackTrace(); + } + return null; + }); + } } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java index 7ea53aff7c4..f2dfb0288d2 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/OAuth2TokenAuthenticator.java @@ -5,6 +5,7 @@ package com.datastrato.gravitino.server.auth; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.auth.SignatureAlgorithmFamilyType; import com.datastrato.gravitino.exceptions.UnauthorizedException; diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/SimpleAuthenticator.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/SimpleAuthenticator.java index 15be05bd1d6..6de1d2b8610 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/SimpleAuthenticator.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/SimpleAuthenticator.java @@ -6,6 +6,7 @@ package com.datastrato.gravitino.server.auth; import com.datastrato.gravitino.Config; +import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthConstants; import java.security.Principal; import java.util.Base64; diff --git a/server-common/src/test/java/com/datastrato/gravitino/server/auth/TestAuthenticationFilter.java b/server-common/src/test/java/com/datastrato/gravitino/server/auth/TestAuthenticationFilter.java index 8b398b9c2ba..60f02e37c7f 100644 --- a/server-common/src/test/java/com/datastrato/gravitino/server/auth/TestAuthenticationFilter.java +++ b/server-common/src/test/java/com/datastrato/gravitino/server/auth/TestAuthenticationFilter.java @@ -13,6 +13,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.exceptions.UnauthorizedException; import java.io.IOException; diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java index 3c9ef8d5a8b..ccebee100b4 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java @@ -4,10 +4,11 @@ */ package com.datastrato.gravitino.server.web; -import com.datastrato.gravitino.PrincipalContext; +import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.dto.responses.ErrorResponse; -import com.datastrato.gravitino.server.auth.UserPrincipal; +import com.datastrato.gravitino.utils.PrincipalUtils; +import java.security.PrivilegedExceptionAction; import java.util.Optional; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.MediaType; @@ -98,12 +99,13 @@ public static Response nonEmpty(String type, String message, Throwable throwable .build(); } - public static PrincipalContext createUserPrincipalContext(HttpServletRequest httpRequest) { + public static Response doAs( + HttpServletRequest httpRequest, PrivilegedExceptionAction action) throws Exception { UserPrincipal principal = (UserPrincipal) httpRequest.getAttribute(AuthConstants.AuthenticatedPrincipalAttributeName); if (principal == null) { principal = new UserPrincipal(AuthConstants.ANONYMOUS_USER); } - return PrincipalContext.createPrincipalContext(principal.getName()); + return PrincipalUtils.doAs(principal, action); } } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java index c7aa695dfcb..1b8ad520330 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java @@ -8,7 +8,6 @@ import com.datastrato.gravitino.CatalogChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogManager; import com.datastrato.gravitino.dto.requests.CatalogCreateRequest; import com.datastrato.gravitino.dto.requests.CatalogUpdateRequest; @@ -67,17 +66,21 @@ public Response listCatalogs(@PathParam("metalake") String metalake) { @Produces("application/vnd.gravitino.v1+json") public Response createCatalog( @PathParam("metalake") String metalake, CatalogCreateRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofCatalog(metalake, request.getName()); - Catalog catalog = - manager.createCatalog( - ident, - request.getType(), - request.getProvider(), - request.getComment(), - request.getProperties()); - return Utils.ok(new CatalogResponse(DTOConverters.toDTO(catalog))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = NameIdentifier.ofCatalog(metalake, request.getName()); + Catalog catalog = + manager.createCatalog( + ident, + request.getType(), + request.getProvider(), + request.getComment(), + request.getProperties()); + return Utils.ok(new CatalogResponse(DTOConverters.toDTO(catalog))); + }); } catch (Exception e) { return ExceptionHandlers.handleCatalogException( @@ -108,16 +111,20 @@ public Response alterCatalog( @PathParam("metalake") String metalakeName, @PathParam("catalog") String catalogName, CatalogUpdatesRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofCatalog(metalakeName, catalogName); - CatalogChange[] changes = - request.getUpdates().stream() - .map(CatalogUpdateRequest::catalogChange) - .toArray(CatalogChange[]::new); - - Catalog catalog = manager.alterCatalog(ident, changes); - return Utils.ok(new CatalogResponse(DTOConverters.toDTO(catalog))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = NameIdentifier.ofCatalog(metalakeName, catalogName); + CatalogChange[] changes = + request.getUpdates().stream() + .map(CatalogUpdateRequest::catalogChange) + .toArray(CatalogChange[]::new); + + Catalog catalog = manager.alterCatalog(ident, changes); + return Utils.ok(new CatalogResponse(DTOConverters.toDTO(catalog))); + }); } catch (Exception e) { return ExceptionHandlers.handleCatalogException( diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java index e790d16df08..2c501d0683c 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java @@ -9,7 +9,6 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.dto.MetalakeDTO; import com.datastrato.gravitino.dto.requests.MetalakeCreateRequest; import com.datastrato.gravitino.dto.requests.MetalakeUpdateRequest; @@ -77,12 +76,16 @@ public Response listMetalakes() { @Timed(name = "create-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "create-metalake", absolute = true) public Response createMetalake(MetalakeCreateRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofMetalake(request.getName()); - BaseMetalake metalake = - manager.createMetalake(ident, request.getComment(), request.getProperties()); - return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(metalake))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = NameIdentifier.ofMetalake(request.getName()); + BaseMetalake metalake = + manager.createMetalake(ident, request.getComment(), request.getProperties()); + return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(metalake))); + }); } catch (Exception e) { return ExceptionHandlers.handleMetalakeException(OperationType.CREATE, request.getName(), e); @@ -112,16 +115,20 @@ public Response loadMetalake(@PathParam("name") String metalakeName) { @ResponseMetered(name = "alter-metalake", absolute = true) public Response alterMetalake( @PathParam("name") String metalakeName, MetalakeUpdatesRequest updatesRequest) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - updatesRequest.validate(); - NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); - MetalakeChange[] changes = - updatesRequest.getUpdates().stream() - .map(MetalakeUpdateRequest::metalakeChange) - .toArray(MetalakeChange[]::new); - - BaseMetalake updatedMetalake = manager.alterMetalake(identifier, changes); - return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(updatedMetalake))); + try { + return Utils.doAs( + httpRequest, + () -> { + updatesRequest.validate(); + NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); + MetalakeChange[] changes = + updatesRequest.getUpdates().stream() + .map(MetalakeUpdateRequest::metalakeChange) + .toArray(MetalakeChange[]::new); + + BaseMetalake updatedMetalake = manager.alterMetalake(identifier, changes); + return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(updatedMetalake))); + }); } catch (Exception e) { return ExceptionHandlers.handleMetalakeException(OperationType.ALTER, metalakeName, e); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java index 2a6af235aa7..59d13c44577 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java @@ -8,7 +8,6 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogOperationDispatcher; import com.datastrato.gravitino.dto.requests.SchemaCreateRequest; import com.datastrato.gravitino.dto.requests.SchemaUpdateRequest; @@ -79,11 +78,16 @@ public Response createSchema( @PathParam("metalake") String metalake, @PathParam("catalog") String catalog, SchemaCreateRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, request.getName()); - Schema schema = dispatcher.createSchema(ident, request.getComment(), request.getProperties()); - return Utils.ok(new SchemaResponse(DTOConverters.toDTO(schema))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, request.getName()); + Schema schema = + dispatcher.createSchema(ident, request.getComment(), request.getProperties()); + return Utils.ok(new SchemaResponse(DTOConverters.toDTO(schema))); + }); } catch (Exception e) { return ExceptionHandlers.handleSchemaException( @@ -120,15 +124,19 @@ public Response alterSchema( @PathParam("catalog") String catalog, @PathParam("schema") String schema, SchemaUpdatesRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); - SchemaChange[] changes = - request.getUpdates().stream() - .map(SchemaUpdateRequest::schemaChange) - .toArray(SchemaChange[]::new); - Schema s = dispatcher.alterSchema(ident, changes); - return Utils.ok(new SchemaResponse(DTOConverters.toDTO(s))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); + SchemaChange[] changes = + request.getUpdates().stream() + .map(SchemaUpdateRequest::schemaChange) + .toArray(SchemaChange[]::new); + Schema s = dispatcher.alterSchema(ident, changes); + return Utils.ok(new SchemaResponse(DTOConverters.toDTO(s))); + }); } catch (Exception e) { return ExceptionHandlers.handleSchemaException(OperationType.ALTER, schema, catalog, e); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java index e8fcfe9a4e0..9844d979ec4 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java @@ -11,7 +11,6 @@ import com.codahale.metrics.annotation.Timed; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.catalog.CatalogOperationDispatcher; import com.datastrato.gravitino.dto.requests.TableCreateRequest; import com.datastrato.gravitino.dto.requests.TableUpdateRequest; @@ -81,20 +80,25 @@ public Response createTable( @PathParam("catalog") String catalog, @PathParam("schema") String schema, TableCreateRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, request.getName()); - - Table table = - dispatcher.createTable( - ident, - request.getColumns(), - request.getComment(), - request.getProperties(), - fromDTOs(request.getPartitioning()), - fromDTO(request.getDistribution()), - fromDTOs(request.getSortOrders())); - return Utils.ok(new TableResponse(DTOConverters.toDTO(table))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = + NameIdentifier.ofTable(metalake, catalog, schema, request.getName()); + + Table table = + dispatcher.createTable( + ident, + request.getColumns(), + request.getComment(), + request.getProperties(), + fromDTOs(request.getPartitioning()), + fromDTO(request.getDistribution()), + fromDTOs(request.getSortOrders())); + return Utils.ok(new TableResponse(DTOConverters.toDTO(table))); + }); } catch (Exception e) { return ExceptionHandlers.handleTableException( @@ -133,15 +137,19 @@ public Response alterTable( @PathParam("schema") String schema, @PathParam("table") String table, TableUpdatesRequest request) { - try (PrincipalContext context = Utils.createUserPrincipalContext(httpRequest)) { - request.validate(); - NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); - TableChange[] changes = - request.getUpdates().stream() - .map(TableUpdateRequest::tableChange) - .toArray(TableChange[]::new); - Table t = dispatcher.alterTable(ident, changes); - return Utils.ok(new TableResponse(DTOConverters.toDTO(t))); + try { + return Utils.doAs( + httpRequest, + () -> { + request.validate(); + NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); + TableChange[] changes = + request.getUpdates().stream() + .map(TableUpdateRequest::tableChange) + .toArray(TableChange[]::new); + Table t = dispatcher.alterTable(ident, changes); + return Utils.ok(new TableResponse(DTOConverters.toDTO(t))); + }); } catch (Exception e) { return ExceptionHandlers.handleTableException(OperationType.ALTER, table, schema, e); From 854669b8c604554ee4f8d03580e5f11df453641b Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 10:46:37 +0800 Subject: [PATCH 40/51] fix --- .../gravitino/PrincipalContext.java | 39 ------------------- .../gravitino/catalog/TestCatalogManager.java | 3 -- .../TestCatalogOperationDispatcher.java | 23 +++++------ .../gravitino/meta/TestMetalakeManager.java | 3 -- 4 files changed, 9 insertions(+), 59 deletions(-) delete mode 100644 core/src/main/java/com/datastrato/gravitino/PrincipalContext.java diff --git a/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java b/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java deleted file mode 100644 index 1f4a0feefb6..00000000000 --- a/core/src/main/java/com/datastrato/gravitino/PrincipalContext.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 Datastrato Pvt Ltd. - * This software is licensed under the Apache License version 2. - */ - -package com.datastrato.gravitino; - -import com.datastrato.gravitino.auth.AuthConstants; -import org.apache.commons.lang3.StringUtils; - -public class PrincipalContext implements AutoCloseable { - - private String currentUser; - - private static final ThreadLocal context = new ThreadLocal<>(); - - public static PrincipalContext get() { - return context.get(); - } - - public String getCurrentUser() { - if (StringUtils.isBlank(currentUser)) { - return AuthConstants.ANONYMOUS_USER; - } - return currentUser; - } - - public static PrincipalContext createPrincipalContext(String currentUser) { - PrincipalContext principalContext = new PrincipalContext(); - principalContext.currentUser = currentUser; - context.set(principalContext); - return principalContext; - } - - @Override - public void close() throws Exception { - context.remove(); - } -} diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java index 5b64f61d43f..cfa6a24be1e 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogManager.java @@ -91,9 +91,6 @@ public static void tearDown() throws Exception { catalogManager.close(); catalogManager = null; } - if (context != null) { - context.close(); - } } @Test diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java index b9c2eb0fc17..1389ec8f523 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java @@ -23,10 +23,10 @@ import com.datastrato.gravitino.EntityStore; import com.datastrato.gravitino.NameIdentifier; import com.datastrato.gravitino.Namespace; -import com.datastrato.gravitino.PrincipalContext; import com.datastrato.gravitino.StringIdentifier; import com.datastrato.gravitino.TestColumn; import com.datastrato.gravitino.TestEntityStore; +import com.datastrato.gravitino.auth.AuthConstants; import com.datastrato.gravitino.exceptions.IllegalNamespaceException; import com.datastrato.gravitino.exceptions.NoSuchEntityException; import com.datastrato.gravitino.meta.AuditInfo; @@ -73,8 +73,6 @@ public class TestCatalogOperationDispatcher { private static CatalogOperationDispatcher dispatcher; - private static PrincipalContext context; - @BeforeAll public static void setUp() throws IOException { config = new Config(false) {}; @@ -113,9 +111,6 @@ public static void tearDown() throws Exception { catalogManager.close(); catalogManager = null; } - if (context != null) { - context.close(); - } } @BeforeEach @@ -200,7 +195,7 @@ public void testCreateAndLoadSchema() throws IOException { Assertions.assertEquals(schema.comment(), loadedSchema.comment()); testProperties(schema.properties(), loadedSchema.properties()); // Audit info is gotten from entity store - Assertions.assertEquals("gravitino", loadedSchema.auditInfo().creator()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, loadedSchema.auditInfo().creator()); // Case 2: Test if the schema is not found in entity store doThrow(new NoSuchEntityException("mock error")).when(entityStore).get(any(), any(), any()); @@ -224,7 +219,7 @@ public void testCreateAndLoadSchema() throws IOException { .withNamespace(Namespace.of(metalake, catalog)) .withAuditInfo( new AuditInfo.Builder() - .withCreator("gravitino") + .withCreator(AuthConstants.ANONYMOUS_USER) .withCreateTime(Instant.now()) .build()) .build(); @@ -258,8 +253,8 @@ public void testCreateAndAlterSchema() throws IOException { Map expectedProps = ImmutableMap.of("k2", "v2", "k3", "v3"); testProperties(expectedProps, alteredSchema.properties()); // Audit info is gotten from gravitino entity store. - Assertions.assertEquals("gravitino", alteredSchema.auditInfo().creator()); - Assertions.assertEquals("gravitino", alteredSchema.auditInfo().lastModifier()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, alteredSchema.auditInfo().creator()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, alteredSchema.auditInfo().lastModifier()); // Case 2: Test if the schema is not found in entity store doThrow(new NoSuchEntityException("mock error")) @@ -293,7 +288,7 @@ public void testCreateAndAlterSchema() throws IOException { .withNamespace(Namespace.of(metalake, catalog)) .withAuditInfo( new AuditInfo.Builder() - .withCreator("gravitino") + .withCreator(AuthConstants.ANONYMOUS_USER) .withCreateTime(Instant.now()) .build()) .build(); @@ -420,7 +415,7 @@ public void testCreateAndLoadTable() throws IOException { Assertions.assertEquals(0, loadedTable1.partitioning().length); Assertions.assertArrayEquals(table1.columns(), loadedTable1.columns()); // Audit info is gotten from the entity store - Assertions.assertEquals("gravitino", loadedTable1.auditInfo().creator()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, loadedTable1.auditInfo().creator()); // Case 2: Test if the table entity is not found in the entity store reset(entityStore); @@ -486,8 +481,8 @@ public void testCreateAndAlterTable() throws IOException { Map expectedProps = ImmutableMap.of("k2", "v2", "k3", "v3"); testProperties(expectedProps, alteredTable.properties()); // Audit info is gotten from gravitino entity store - Assertions.assertEquals("gravitino", alteredTable.auditInfo().creator()); - Assertions.assertEquals("gravitino", alteredTable.auditInfo().lastModifier()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, alteredTable.auditInfo().creator()); + Assertions.assertEquals(AuthConstants.ANONYMOUS_USER, alteredTable.auditInfo().lastModifier()); // Case 2: Test if the table entity is not found in the entity store reset(entityStore); diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java index e336c9c7931..e357e5abee9 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java @@ -47,9 +47,6 @@ public static void tearDown() throws Exception { entityStore.close(); entityStore = null; } - if (context != null) { - context.close(); - } } @Test From 2167435aa6a6dc4f2ed21181219766718886ee67 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 11:26:20 +0800 Subject: [PATCH 41/51] fix --- .../gravitino/auth/AuthConstants.java | 2 +- .../gravitino/utils/TestPrincipalUtils.java | 39 +++++++++++++++++++ .../integration/test/client/AuditIT.java | 32 --------------- .../server/auth/AuthenticationFilter.java | 2 +- .../gravitino/server/web/Utils.java | 3 +- .../server/web/rest/CatalogOperations.java | 28 +++++++------ .../server/web/rest/MetalakeOperations.java | 38 +++++++++++------- .../server/web/rest/SchemaOperations.java | 38 +++++++++++------- .../server/web/rest/TableOperations.java | 37 +++++++++++------- 9 files changed, 133 insertions(+), 86 deletions(-) create mode 100644 core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java diff --git a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java index 29a1ee27242..02975749974 100644 --- a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java +++ b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java @@ -14,5 +14,5 @@ public interface AuthConstants { String ANONYMOUS_USER = "anonymous"; - String AuthenticatedPrincipalAttributeName = AuthConstants.class.getName() + "-principal"; + String AUTHENTICATED_PRINCIPAL_ATTRIBUTE_NAME = AuthConstants.class.getName() + "-principal"; } diff --git a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java new file mode 100644 index 00000000000..3d635e4824a --- /dev/null +++ b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java @@ -0,0 +1,39 @@ +package com.datastrato.gravitino.utils; + +import com.datastrato.gravitino.UserPrincipal; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestPrincipalUtils { + + @Test + public void testNormal() throws Exception { + UserPrincipal principal = new UserPrincipal("testNormal"); + PrincipalUtils.doAs( + principal, + () -> { + Assertions.assertEquals("testNormal", PrincipalUtils.getCurrentPrincipal().getName()); + return null; + }); + } + + @Test + public void testThread() throws Exception { + UserPrincipal principal = new UserPrincipal("testThread"); + PrincipalUtils.doAs( + principal, + () -> { + Thread thread = + new Thread( + () -> + Assertions.assertEquals( + "testThread", PrincipalUtils.getCurrentPrincipal().getName())); + thread.start(); + thread.join(); + return null; + }); + } + + @Test + public void testThreadPool() {} +} diff --git a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java index 6d2db391ef6..95cf56b26fe 100644 --- a/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java +++ b/integration-test/src/test/java/com/datastrato/gravitino/integration/test/client/AuditIT.java @@ -7,19 +7,14 @@ import com.datastrato.gravitino.MetalakeChange; import com.datastrato.gravitino.NameIdentifier; -import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthenticatorType; import com.datastrato.gravitino.client.GravitinoMetaLake; import com.datastrato.gravitino.integration.test.util.AbstractIT; import com.datastrato.gravitino.integration.test.util.GravitinoITUtils; import com.datastrato.gravitino.server.auth.OAuthConfig; import com.google.common.collect.Maps; -import java.security.AccessControlContext; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.Collections; import java.util.Map; -import javax.security.auth.Subject; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -55,31 +50,4 @@ public void testAuditMetalake() throws Exception { Assertions.assertEquals(expectUser, metaLake.auditInfo().lastModifier()); client.dropMetalake(NameIdentifier.parse(newName)); } - - @Test - public void testSubject() { - Subject subject = new Subject(); - subject.getPrincipals().add(new UserPrincipal("test")); - Subject.doAs( - subject, - (PrivilegedAction) - () -> { - Thread thread = - new Thread( - () -> { - AccessControlContext context = AccessController.getContext(); - Subject subject1 = Subject.getSubject(context); - UserPrincipal principal = - subject1.getPrincipals(UserPrincipal.class).iterator().next(); - System.out.println("principal name: " + principal.getName()); - }); - thread.start(); - try { - thread.join(); - } catch (Exception e) { - // e.printStackTrace(); - } - return null; - }); - } } diff --git a/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java b/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java index 94318403796..c2498fed7a7 100644 --- a/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java +++ b/server-common/src/main/java/com/datastrato/gravitino/server/auth/AuthenticationFilter.java @@ -54,7 +54,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha } if (authenticator.isDataFromToken()) { Principal principal = authenticator.authenticateToken(authData); - request.setAttribute(AuthConstants.AuthenticatedPrincipalAttributeName, principal); + request.setAttribute(AuthConstants.AUTHENTICATED_PRINCIPAL_ATTRIBUTE_NAME, principal); } chain.doFilter(request, response); } catch (UnauthorizedException ue) { diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java index ccebee100b4..91f0854178f 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/Utils.java @@ -102,7 +102,8 @@ public static Response nonEmpty(String type, String message, Throwable throwable public static Response doAs( HttpServletRequest httpRequest, PrivilegedExceptionAction action) throws Exception { UserPrincipal principal = - (UserPrincipal) httpRequest.getAttribute(AuthConstants.AuthenticatedPrincipalAttributeName); + (UserPrincipal) + httpRequest.getAttribute(AuthConstants.AUTHENTICATED_PRINCIPAL_ATTRIBUTE_NAME); if (principal == null) { principal = new UserPrincipal(AuthConstants.ANONYMOUS_USER); } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java index 1b8ad520330..696a93b6e2c 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/CatalogOperations.java @@ -53,10 +53,13 @@ public CatalogOperations(CatalogManager manager) { @Produces("application/vnd.gravitino.v1+json") public Response listCatalogs(@PathParam("metalake") String metalake) { try { - Namespace catalogNS = Namespace.ofCatalog(metalake); - NameIdentifier[] idents = manager.listCatalogs(catalogNS); - return Utils.ok(new EntityListResponse(idents)); - + return Utils.doAs( + httpRequest, + () -> { + Namespace catalogNS = Namespace.ofCatalog(metalake); + NameIdentifier[] idents = manager.listCatalogs(catalogNS); + return Utils.ok(new EntityListResponse(idents)); + }); } catch (Exception e) { return ExceptionHandlers.handleCatalogException(OperationType.LIST, "", metalake, e); } @@ -138,14 +141,17 @@ public Response alterCatalog( public Response dropCatalog( @PathParam("metalake") String metalakeName, @PathParam("catalog") String catalogName) { try { - NameIdentifier ident = NameIdentifier.ofCatalog(metalakeName, catalogName); - boolean dropped = manager.dropCatalog(ident); - if (!dropped) { - LOG.warn("Failed to drop catalog {} under metalake {}", catalogName, metalakeName); - } - - return Utils.ok(new DropResponse(dropped)); + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifier.ofCatalog(metalakeName, catalogName); + boolean dropped = manager.dropCatalog(ident); + if (!dropped) { + LOG.warn("Failed to drop catalog {} under metalake {}", catalogName, metalakeName); + } + return Utils.ok(new DropResponse(dropped)); + }); } catch (Exception e) { return ExceptionHandlers.handleCatalogException( OperationType.DROP, catalogName, metalakeName, e); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java index 2c501d0683c..db3dfd1cd66 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/MetalakeOperations.java @@ -60,10 +60,14 @@ public MetalakeOperations(MetalakeManager manager) { @ResponseMetered(name = "list-metalake", absolute = true) public Response listMetalakes() { try { - BaseMetalake[] metalakes = manager.listMetalakes(); - MetalakeDTO[] metalakeDTOS = - Arrays.stream(metalakes).map(DTOConverters::toDTO).toArray(MetalakeDTO[]::new); - return Utils.ok(new MetalakeListResponse(metalakeDTOS)); + return Utils.doAs( + httpRequest, + () -> { + BaseMetalake[] metalakes = manager.listMetalakes(); + MetalakeDTO[] metalakeDTOS = + Arrays.stream(metalakes).map(DTOConverters::toDTO).toArray(MetalakeDTO[]::new); + return Utils.ok(new MetalakeListResponse(metalakeDTOS)); + }); } catch (Exception e) { return ExceptionHandlers.handleMetalakeException( @@ -99,9 +103,13 @@ public Response createMetalake(MetalakeCreateRequest request) { @ResponseMetered(name = "load-metalake", absolute = true) public Response loadMetalake(@PathParam("name") String metalakeName) { try { - NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); - BaseMetalake metalake = manager.loadMetalake(identifier); - return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(metalake))); + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); + BaseMetalake metalake = manager.loadMetalake(identifier); + return Utils.ok(new MetalakeResponse(DTOConverters.toDTO(metalake))); + }); } catch (Exception e) { return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); @@ -142,13 +150,17 @@ public Response alterMetalake( @ResponseMetered(name = "drop-metalake", absolute = true) public Response dropMetalake(@PathParam("name") String metalakeName) { try { - NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); - boolean dropped = manager.dropMetalake(identifier); - if (!dropped) { - LOG.warn("Failed to drop metalake by name {}", metalakeName); - } + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifier.ofMetalake(metalakeName); + boolean dropped = manager.dropMetalake(identifier); + if (!dropped) { + LOG.warn("Failed to drop metalake by name {}", metalakeName); + } - return Utils.ok(new DropResponse(dropped)); + return Utils.ok(new DropResponse(dropped)); + }); } catch (Exception e) { return ExceptionHandlers.handleMetalakeException(OperationType.DROP, metalakeName, e); diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java index 59d13c44577..0342668d7e7 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/SchemaOperations.java @@ -61,10 +61,13 @@ public SchemaOperations(CatalogOperationDispatcher dispatcher) { public Response listSchemas( @PathParam("metalake") String metalake, @PathParam("catalog") String catalog) { try { - Namespace schemaNS = Namespace.ofSchema(metalake, catalog); - NameIdentifier[] idents = dispatcher.listSchemas(schemaNS); - return Utils.ok(new EntityListResponse(idents)); - + return Utils.doAs( + httpRequest, + () -> { + Namespace schemaNS = Namespace.ofSchema(metalake, catalog); + NameIdentifier[] idents = dispatcher.listSchemas(schemaNS); + return Utils.ok(new EntityListResponse(idents)); + }); } catch (Exception e) { return ExceptionHandlers.handleSchemaException(OperationType.LIST, "", catalog, e); } @@ -105,9 +108,13 @@ public Response loadSchema( @PathParam("catalog") String catalog, @PathParam("schema") String schema) { try { - NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); - Schema s = dispatcher.loadSchema(ident); - return Utils.ok(new SchemaResponse(DTOConverters.toDTO(s))); + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); + Schema s = dispatcher.loadSchema(ident); + return Utils.ok(new SchemaResponse(DTOConverters.toDTO(s))); + }); } catch (Exception e) { return ExceptionHandlers.handleSchemaException(OperationType.LOAD, schema, catalog, e); @@ -154,14 +161,17 @@ public Response dropSchema( @PathParam("schema") String schema, @DefaultValue("false") @QueryParam("cascade") boolean cascade) { try { - NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); - boolean dropped = dispatcher.dropSchema(ident, cascade); - if (!dropped) { - LOG.warn("Fail to drop schema {} under namespace {}", schema, ident.namespace()); - } - - return Utils.ok(new DropResponse(dropped)); + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifier.ofSchema(metalake, catalog, schema); + boolean dropped = dispatcher.dropSchema(ident, cascade); + if (!dropped) { + LOG.warn("Fail to drop schema {} under namespace {}", schema, ident.namespace()); + } + return Utils.ok(new DropResponse(dropped)); + }); } catch (Exception e) { return ExceptionHandlers.handleSchemaException(OperationType.DROP, schema, catalog, e); } diff --git a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java index 9844d979ec4..9d9114ee58f 100644 --- a/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java +++ b/server/src/main/java/com/datastrato/gravitino/server/web/rest/TableOperations.java @@ -62,9 +62,13 @@ public Response listTables( @PathParam("catalog") String catalog, @PathParam("schema") String schema) { try { - Namespace tableNS = Namespace.ofTable(metalake, catalog, schema); - NameIdentifier[] idents = dispatcher.listTables(tableNS); - return Utils.ok(new EntityListResponse(idents)); + return Utils.doAs( + httpRequest, + () -> { + Namespace tableNS = Namespace.ofTable(metalake, catalog, schema); + NameIdentifier[] idents = dispatcher.listTables(tableNS); + return Utils.ok(new EntityListResponse(idents)); + }); } catch (Exception e) { return ExceptionHandlers.handleTableException(OperationType.LIST, "", schema, e); @@ -117,10 +121,13 @@ public Response loadTable( @PathParam("schema") String schema, @PathParam("table") String table) { try { - NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); - Table t = dispatcher.loadTable(ident); - return Utils.ok(new TableResponse(DTOConverters.toDTO(t))); - + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); + Table t = dispatcher.loadTable(ident); + return Utils.ok(new TableResponse(DTOConverters.toDTO(t))); + }); } catch (Exception e) { return ExceptionHandlers.handleTableException(OperationType.LOAD, table, schema, e); } @@ -168,13 +175,17 @@ public Response dropTable( @PathParam("table") String table, @QueryParam("purge") @DefaultValue("false") boolean purge) { try { - NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); - boolean dropped = purge ? dispatcher.purgeTable(ident) : dispatcher.dropTable(ident); - if (!dropped) { - LOG.warn("Failed to drop table {} under schema {}", table, schema); - } + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier ident = NameIdentifier.ofTable(metalake, catalog, schema, table); + boolean dropped = purge ? dispatcher.purgeTable(ident) : dispatcher.dropTable(ident); + if (!dropped) { + LOG.warn("Failed to drop table {} under schema {}", table, schema); + } - return Utils.ok(new DropResponse(dropped)); + return Utils.ok(new DropResponse(dropped)); + }); } catch (Exception e) { return ExceptionHandlers.handleTableException(OperationType.DROP, table, schema, e); From 320ca46316f275b9eb0e30e82d921a52d132165e Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 11:34:15 +0800 Subject: [PATCH 42/51] fix --- .../gravitino/utils/TestPrincipalUtils.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java index 3d635e4824a..eef8388a129 100644 --- a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java +++ b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java @@ -1,6 +1,14 @@ +/* + * Copyright 2023 Datastrato Pvt Ltd. + * This software is licensed under the Apache License version 2. + */ + package com.datastrato.gravitino.utils; import com.datastrato.gravitino.UserPrincipal; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -35,5 +43,20 @@ public void testThread() throws Exception { } @Test - public void testThreadPool() {} + public void testThreadPool() throws Exception { + UserPrincipal principal = new UserPrincipal("testThreadPool"); + ExecutorService executorService = Executors.newCachedThreadPool(); + PrincipalUtils.doAs( + principal, + () -> { + Future future = + executorService.submit( + () -> + Assertions.assertEquals( + "testThreadPool", PrincipalUtils.getCurrentPrincipal().getName())); + future.get(); + return null; + }); + executorService.shutdownNow(); + } } From 12f5c9329bb84203709964338855bd62502f6df9 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 11:48:32 +0800 Subject: [PATCH 43/51] fix --- .../main/java/com/datastrato/gravitino/utils/PrincipalUtils.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java index 8cfcf41e373..458966d2115 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java @@ -14,6 +14,7 @@ import java.security.PrivilegedExceptionAction; import javax.security.auth.Subject; +@SuppressWarnings("deprecation") public class PrincipalUtils { private PrincipalUtils() {} From 989d850161763e3fa75b958946c64622fd76395d Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 11:49:19 +0800 Subject: [PATCH 44/51] fix --- .../java/com/datastrato/gravitino/utils/PrincipalUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java index 458966d2115..937abda02a0 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java @@ -14,7 +14,7 @@ import java.security.PrivilegedExceptionAction; import javax.security.auth.Subject; -@SuppressWarnings("deprecation") +@SuppressWarnings("removal") public class PrincipalUtils { private PrincipalUtils() {} From 50127a177ce7ed703db001a13416a8d97aeaf899 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 11:55:40 +0800 Subject: [PATCH 45/51] fix --- .../java/com/datastrato/gravitino/utils/PrincipalUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java index 937abda02a0..a9275b22daa 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java @@ -7,8 +7,6 @@ import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthConstants; -import java.security.AccessControlContext; -import java.security.AccessController; import java.security.Principal; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -40,7 +38,7 @@ public static T doAs(Principal principal, PrivilegedExceptionAction actio } public static Principal getCurrentPrincipal() { - AccessControlContext context = AccessController.getContext(); + java.security.AccessControlContext context = java.security.AccessController.getContext(); Subject subject = Subject.getSubject(context); if (subject == null || subject.getPrincipals(UserPrincipal.class).isEmpty()) { return new UserPrincipal(AuthConstants.ANONYMOUS_USER); From fc8d3fd90956abf64138e2fac45e81cb60fceb30 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 14:49:51 +0800 Subject: [PATCH 46/51] fix --- .../gravitino/catalog/TestCatalogOperationDispatcher.java | 2 +- .../com/datastrato/gravitino/meta/TestMetalakeManager.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java index 1389ec8f523..6cd9e7fec1d 100644 --- a/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java +++ b/core/src/test/java/com/datastrato/gravitino/catalog/TestCatalogOperationDispatcher.java @@ -101,7 +101,7 @@ public static void setUp() throws IOException { } @AfterAll - public static void tearDown() throws Exception { + public static void tearDown() throws IOException { if (entityStore != null) { entityStore.close(); entityStore = null; diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java index e357e5abee9..5c4702a5e7f 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java @@ -15,6 +15,8 @@ import com.datastrato.gravitino.storage.RandomIdGenerator; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; + +import java.io.IOException; import java.util.Map; import java.util.Set; import org.junit.jupiter.api.AfterAll; @@ -42,7 +44,7 @@ public static void setUp() { } @AfterAll - public static void tearDown() throws Exception { + public static void tearDown() throws IOException { if (entityStore != null) { entityStore.close(); entityStore = null; From 0b1b3a9be787b9a202c9f29f873e50034d317a68 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 14:54:04 +0800 Subject: [PATCH 47/51] fix --- .../java/com/datastrato/gravitino/meta/TestMetalakeManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java index 5c4702a5e7f..eede4fb2c69 100644 --- a/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java +++ b/core/src/test/java/com/datastrato/gravitino/meta/TestMetalakeManager.java @@ -15,7 +15,6 @@ import com.datastrato.gravitino.storage.RandomIdGenerator; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; - import java.io.IOException; import java.util.Map; import java.util.Set; From 9896175918a236ef2972d7ec79ebd749cedc802e Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 19:21:14 +0800 Subject: [PATCH 48/51] fix --- .../main/java/com/datastrato/gravitino/auth/AuthConstants.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java index 02975749974..bda20c47aa6 100644 --- a/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java +++ b/common/src/main/java/com/datastrato/gravitino/auth/AuthConstants.java @@ -14,5 +14,6 @@ public interface AuthConstants { String ANONYMOUS_USER = "anonymous"; + // Refer to the style of `AuthenticationFilter#AuthenticatedRoleAttributeName` of Apache Pulsar String AUTHENTICATED_PRINCIPAL_ATTRIBUTE_NAME = AuthConstants.class.getName() + "-principal"; } From ba2384cfd06c949e7f31a35eef2ff439258322ed Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 19:42:11 +0800 Subject: [PATCH 49/51] fix --- .../datastrato/gravitino/utils/PrincipalUtils.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java index a9275b22daa..cbaba064ec6 100644 --- a/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java +++ b/core/src/main/java/com/datastrato/gravitino/utils/PrincipalUtils.java @@ -7,6 +7,7 @@ import com.datastrato.gravitino.UserPrincipal; import com.datastrato.gravitino.auth.AuthConstants; +import com.google.common.base.Throwables; import java.security.Principal; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; @@ -24,16 +25,8 @@ public static T doAs(Principal principal, PrivilegedExceptionAction actio return Subject.doAs(subject, action); } catch (PrivilegedActionException pae) { Throwable cause = pae.getCause(); - if (!(cause instanceof Exception)) { - throw new RuntimeException( - "PrivilegedActionException with no " - + "underlying cause. Principal " - + principal.getName() - + " : " - + pae, - pae); - } - throw (Exception) cause; + Throwables.propagateIfPossible(cause, Exception.class); + throw new RuntimeException("doAs method occurs an unexpected exception", pae); } } From 9911e191afc2892ed76823ba28a07f4cfcaee4d8 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 20:03:44 +0800 Subject: [PATCH 50/51] remove bad case --- .../gravitino/utils/TestPrincipalUtils.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java index eef8388a129..0d6f53bcbc6 100644 --- a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java +++ b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java @@ -41,22 +41,4 @@ public void testThread() throws Exception { return null; }); } - - @Test - public void testThreadPool() throws Exception { - UserPrincipal principal = new UserPrincipal("testThreadPool"); - ExecutorService executorService = Executors.newCachedThreadPool(); - PrincipalUtils.doAs( - principal, - () -> { - Future future = - executorService.submit( - () -> - Assertions.assertEquals( - "testThreadPool", PrincipalUtils.getCurrentPrincipal().getName())); - future.get(); - return null; - }); - executorService.shutdownNow(); - } } From 4a4bafdd7c4525c177d33dc5a3bda202f45c2bc0 Mon Sep 17 00:00:00 2001 From: Heng Qin Date: Thu, 28 Dec 2023 20:03:55 +0800 Subject: [PATCH 51/51] fix --- .../com/datastrato/gravitino/utils/TestPrincipalUtils.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java index 0d6f53bcbc6..0394ea9465f 100644 --- a/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java +++ b/core/src/test/java/com/datastrato/gravitino/utils/TestPrincipalUtils.java @@ -6,9 +6,6 @@ package com.datastrato.gravitino.utils; import com.datastrato.gravitino.UserPrincipal; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test;