Skip to content

Commit

Permalink
[apache#2478] Improvement(client-java): Separate Java clients for adm…
Browse files Browse the repository at this point in the history
…in and normal user (apache#2483)

### What changes were proposed in this pull request?

Separate the current Java client into two, one is "GravitinoAdminClient"
which similar to as-is version; the other is "GravitinoClient" which has
a "metalake_name" as input parametor, and then user mainly works on
catalogs.

### Why are the changes needed?

Today in the Java client, the "GravitinoClient" has all the operations
for metalakes, including list, create, alter and drop, etc. That is
mostly for the system administrator's operation, not friendly for a
normal user. Normal user mainly works on catalogs.

Fix: apache#2478

### Does this PR introduce _any_ user-facing change?

Yes, the Java client "GravitinoClient.java" has user interface change;
If the user want to operate Metalake, he can switch to use the
"GravitinoAdminClient.java".

### How was this patch tested?

The UT for the GravitinoClient can ensure the function is not broken.
  • Loading branch information
shaofengshi authored Mar 17, 2024
1 parent f30e566 commit bc43ec8
Show file tree
Hide file tree
Showing 33 changed files with 622 additions and 397 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import com.datastrato.gravitino.Catalog;
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.Namespace;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.exceptions.FilesetAlreadyExistsException;
import com.datastrato.gravitino.exceptions.NoSuchFilesetException;
import com.datastrato.gravitino.file.Fileset;
Expand Down Expand Up @@ -47,7 +47,7 @@ public class HadoopCatalogIT extends AbstractIT {
public static final String SCHEMA_PREFIX = "CatalogFilesetIT_schema";
public static final String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX);
private static final String provider = "hadoop";
private static GravitinoMetaLake metalake;
private static GravitinoMetalake metalake;
private static Catalog catalog;
private static FileSystem hdfs;
private static String defaultBaseLocation;
Expand Down Expand Up @@ -81,12 +81,12 @@ public static void stop() throws IOException {
}

private static void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Assertions.assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import com.datastrato.gravitino.catalog.hive.HiveSchemaPropertiesMetadata;
import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata;
import com.datastrato.gravitino.catalog.hive.HiveTablePropertiesMetadata.TableType;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.connector.BaseCatalog;
import com.datastrato.gravitino.dto.rel.expressions.FieldReferenceDTO;
import com.datastrato.gravitino.dto.rel.partitioning.IdentityPartitioningDTO;
Expand Down Expand Up @@ -123,7 +123,7 @@ public class CatalogHiveIT extends AbstractIT {
private static final String provider = "hive";
private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
private static HiveClientPool hiveClientPool;
private static GravitinoMetaLake metalake;
private static GravitinoMetalake metalake;
private static Catalog catalog;
private static SparkSession sparkSession;
private static FileSystem hdfs;
Expand Down Expand Up @@ -231,12 +231,12 @@ public void resetSchema() throws TException, InterruptedException {
}

private static void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Assertions.assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down Expand Up @@ -1061,7 +1061,7 @@ public void testDropHiveTable() {
public void testAlterSchema() throws TException, InterruptedException {
NameIdentifier ident = NameIdentifier.of(metalakeName, catalogName, schemaName);

GravitinoMetaLake metalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake metalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Catalog catalog = metalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName));
Schema schema = catalog.asSchemas().loadSchema(ident);
Assertions.assertNull(schema.auditInfo().lastModifier());
Expand Down Expand Up @@ -1089,7 +1089,7 @@ public void testAlterSchema() throws TException, InterruptedException {

@Test
void testLoadEntityWithSamePrefix() {
GravitinoMetaLake metalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake metalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Catalog catalog = metalake.loadCatalog(NameIdentifier.of(metalakeName, catalogName));
Assertions.assertNotNull(catalog);

Expand Down Expand Up @@ -1144,7 +1144,7 @@ void testLoadEntityWithSamePrefix() {
void testAlterEntityName() {
String metalakeName = GravitinoITUtils.genRandomName("CatalogHiveIT_metalake");
client.createMetalake(NameIdentifier.of(metalakeName), "", ImmutableMap.of());
final GravitinoMetaLake metalake = client.loadMetalake(NameIdentifier.of(metalakeName));
final GravitinoMetalake metalake = client.loadMetalake(NameIdentifier.of(metalakeName));
String newMetalakeName = GravitinoITUtils.genRandomName("CatalogHiveIT_metalake_new");

// Test rename metalake
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.auth.AuthenticatorType;
import com.datastrato.gravitino.catalog.hive.HiveClientPool;
import com.datastrato.gravitino.client.GravitinoClient;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoAdminClient;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.dto.rel.partitioning.Partitioning;
import com.datastrato.gravitino.integration.test.container.ContainerSuite;
import com.datastrato.gravitino.integration.test.container.HiveContainer;
Expand Down Expand Up @@ -62,14 +62,14 @@ public class ProxyCatalogHiveIT extends AbstractIT {
private static final String HADOOP_USER_NAME = "HADOOP_USER_NAME";

private static final ContainerSuite containerSuite = ContainerSuite.getInstance();
private static GravitinoMetaLake metalake;
private static GravitinoMetalake metalake;
private static Catalog catalog;
private static HiveClientPool hiveClientPool;

private static String HIVE_METASTORE_URIS;
private static FileSystem hdfs;
private static String originHadoopUser;
private static GravitinoClient anotherClient;
private static GravitinoAdminClient anotherClient;
private static Catalog anotherCatalog;

@BeforeAll
Expand Down Expand Up @@ -108,7 +108,7 @@ public static void startIntegrationTest() throws Exception {

String uri = "http://" + jettyServerConfig.getHost() + ":" + jettyServerConfig.getHttpPort();
System.setProperty("user.name", "test");
anotherClient = GravitinoClient.builder(uri).withSimpleAuth().build();
anotherClient = GravitinoAdminClient.builder(uri).withSimpleAuth().build();
createMetalake();
createCatalog();
loadCatalogWithAnotherClient();
Expand Down Expand Up @@ -286,12 +286,12 @@ private Column[] createColumns() {
}

private static void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(METALAKE_NAME), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(METALAKE_NAME));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(METALAKE_NAME));
Assertions.assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down Expand Up @@ -320,7 +320,7 @@ private static void createCatalog() {
}

private static void loadCatalogWithAnotherClient() {
GravitinoMetaLake metaLake = anotherClient.loadMetalake(NameIdentifier.of(METALAKE_NAME));
GravitinoMetalake metaLake = anotherClient.loadMetalake(NameIdentifier.of(METALAKE_NAME));
anotherCatalog = metaLake.loadCatalog(NameIdentifier.of(METALAKE_NAME, CATALOG_NAME));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import com.datastrato.gravitino.auth.AuthenticatorType;
import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig;
import com.datastrato.gravitino.catalog.mysql.integration.test.service.MysqlService;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.integration.test.util.AbstractIT;
import com.datastrato.gravitino.integration.test.util.GravitinoITUtils;
import com.datastrato.gravitino.integration.test.util.ITUtils;
Expand Down Expand Up @@ -50,7 +50,7 @@ public class AuditCatalogMysqlIT extends AbstractIT {

private static MysqlService mysqlService;
private static MySQLContainer<?> MYSQL_CONTAINER;
private static GravitinoMetaLake metalake;
private static GravitinoMetalake metalake;

@BeforeAll
public static void startIntegrationTest() throws Exception {
Expand Down Expand Up @@ -162,12 +162,12 @@ private static Catalog createCatalog(String catalogName) {
}

private static void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Assertions.assertEquals(createdMetalake, loadMetalake);
metalake = loadMetalake;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import com.datastrato.gravitino.Namespace;
import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig;
import com.datastrato.gravitino.catalog.mysql.integration.test.service.MysqlService;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.dto.rel.expressions.LiteralDTO;
import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
import com.datastrato.gravitino.exceptions.NotFoundException;
Expand Down Expand Up @@ -88,7 +88,7 @@ public class CatalogMysqlIT extends AbstractIT {
public String MYSQL_COL_NAME4 = "mysql_col_name4";
public String MYSQL_COL_NAME5 = "mysql_col_name5";

private GravitinoMetaLake metalake;
private GravitinoMetalake metalake;

protected Catalog catalog;

Expand Down Expand Up @@ -151,12 +151,12 @@ private void clearTableAndSchema() {
}

private void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Assertions.assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
import com.datastrato.gravitino.auth.AuthConstants;
import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig;
import com.datastrato.gravitino.catalog.postgresql.integration.test.service.PostgreSqlService;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
import com.datastrato.gravitino.exceptions.SchemaAlreadyExistsException;
import com.datastrato.gravitino.integration.test.util.AbstractIT;
Expand Down Expand Up @@ -83,7 +83,7 @@ public class CatalogPostgreSqlIT extends AbstractIT {
public String POSTGRESQL_COL_NAME3 = "postgresql_col_name3";
private final String provider = "jdbc-postgresql";

private GravitinoMetaLake metalake;
private GravitinoMetalake metalake;

private Catalog catalog;

Expand Down Expand Up @@ -140,12 +140,12 @@ private void clearTableAndSchema() {
}

private void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Assertions.assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.Namespace;
import com.datastrato.gravitino.catalog.jdbc.config.JdbcConfig;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.integration.test.util.AbstractIT;
import com.datastrato.gravitino.integration.test.util.ITUtils;
import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
Expand Down Expand Up @@ -90,7 +90,7 @@ public static void startup() throws IOException {
public void testCreateMultipleJdbc() throws URISyntaxException {
String metalakeName = RandomNameUtils.genRandomName("it_metalake");
String postgreSqlCatalogName = RandomNameUtils.genRandomName("it_postgresql");
GravitinoMetaLake metalake =
GravitinoMetalake metalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());

Map<String, String> pgConf = Maps.newHashMap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergSchemaPropertiesMetadata;
import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergTable;
import com.datastrato.gravitino.catalog.lakehouse.iceberg.ops.IcebergTableOpsHelper;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.dto.rel.partitioning.Partitioning;
import com.datastrato.gravitino.dto.util.DTOConverters;
import com.datastrato.gravitino.exceptions.NoSuchSchemaException;
Expand Down Expand Up @@ -96,7 +96,7 @@ public class CatalogIcebergIT extends AbstractIT {
private static String SELECT_ALL_TEMPLATE = "SELECT * FROM iceberg.%s";
private static String INSERT_BATCH_WITHOUT_PARTITION_TEMPLATE =
"INSERT INTO iceberg.%s VALUES %s";
private static GravitinoMetaLake metalake;
private static GravitinoMetalake metalake;

private static Catalog catalog;

Expand Down Expand Up @@ -160,12 +160,12 @@ private static void clearTableAndSchema() {
}

private static void createMetalake() {
GravitinoMetaLake[] gravitinoMetaLakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetaLakes.length);
GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes();
Assertions.assertEquals(0, gravitinoMetalakes.length);

GravitinoMetaLake createdMetalake =
GravitinoMetalake createdMetalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());
GravitinoMetaLake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
GravitinoMetalake loadMetalake = client.loadMetalake(NameIdentifier.of(metalakeName));
Assertions.assertEquals(createdMetalake, loadMetalake);

metalake = loadMetalake;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import com.datastrato.gravitino.NameIdentifier;
import com.datastrato.gravitino.Namespace;
import com.datastrato.gravitino.catalog.lakehouse.iceberg.IcebergConfig;
import com.datastrato.gravitino.client.GravitinoMetaLake;
import com.datastrato.gravitino.client.GravitinoMetalake;
import com.datastrato.gravitino.integration.test.util.AbstractIT;
import com.datastrato.gravitino.integration.test.util.ITUtils;
import com.datastrato.gravitino.integration.test.util.JdbcDriverDownloader;
Expand Down Expand Up @@ -87,7 +87,7 @@ public static void startup() throws IOException {
public void testCreateMultipleJdbcInIceberg() throws URISyntaxException {
String metalakeName = RandomNameUtils.genRandomName("it_metalake");
String postgreSqlCatalogName = RandomNameUtils.genRandomName("it_iceberg_postgresql");
GravitinoMetaLake metalake =
GravitinoMetalake metalake =
client.createMetalake(NameIdentifier.of(metalakeName), "comment", Collections.emptyMap());

Map<String, String> icebergPgConf = Maps.newHashMap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
class DTOConverters {
private DTOConverters() {}

static GravitinoMetaLake toMetaLake(MetalakeDTO metalake, RESTClient client) {
return new GravitinoMetaLake.Builder()
static GravitinoMetalake toMetaLake(MetalakeDTO metalake, RESTClient client) {
return new GravitinoMetalake.Builder()
.withName(metalake.name())
.withComment(metalake.comment())
.withProperties(metalake.properties())
Expand Down
Loading

0 comments on commit bc43ec8

Please sign in to comment.