Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#4024]Refactor: Reduce unnecessary queries in catalog JDBC implementation #6540

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,23 @@
public interface CatalogMetaMapper {
String TABLE_NAME = "catalog_meta";

@SelectProvider(
type = CatalogMetaSQLProviderFactory.class,
method = "listCatalogPOsByMetalakeName")
List<CatalogPO> listCatalogPOsByMetalakeName(@Param("metalakeName") String metalakeName);

@SelectProvider(
type = CatalogMetaSQLProviderFactory.class,
method = "listCatalogPOsByMetalakeAndCatalogName")
List<CatalogPO> listCatalogPOsByMetalakeAndCatalogName(
@Param("metalakeName") String metalakeName, @Param("catalogName") String catalogName);

@SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "listCatalogPOs")
List<CatalogPO> listCatalogPOs(
@Param("metalakeName") String metalakeName,
@Param("catalogName") String catalogName,
@Param("schemaName") String schemaName);

@SelectProvider(type = CatalogMetaSQLProviderFactory.class, method = "listCatalogPOsByMetalakeId")
List<CatalogPO> listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,22 @@ static class CatalogMetaMySQLProvider extends CatalogMetaBaseSQLProvider {}

static class CatalogMetaH2Provider extends CatalogMetaBaseSQLProvider {}

public static String listCatalogPOsByMetalakeName(@Param("metalakeName") String metalakeName) {
return getProvider().listCatalogPOsByMetalakeName(metalakeName);
}

public static String listCatalogPOsByMetalakeAndCatalogName(
@Param("metalakeName") String metalakeName, @Param("catalogName") String catalogName) {
return getProvider().listCatalogPOsByMetalakeAndCatalogName(metalakeName, catalogName);
}

public static String listCatalogPOs(
@Param("metalakeName") String metalakeName,
@Param("catalogName") String catalogName,
@Param("schemaName") String schemaName) {
return getProvider().listCatalogPOs(metalakeName, catalogName, schemaName);
}

public static String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) {
return getProvider().listCatalogPOsByMetalakeId(metalakeId);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,67 @@
import static org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper.TABLE_NAME;

import java.util.List;
import org.apache.gravitino.storage.relational.mapper.MetalakeMetaMapper;
import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper;
import org.apache.gravitino.storage.relational.po.CatalogPO;
import org.apache.ibatis.annotations.Param;

public class CatalogMetaBaseSQLProvider {
public String listCatalogPOsByMetalakeName(@Param("metalakeName") String metalakeName) {
return "SELECT cm.catalog_id as catalogId, cm.catalog_name as catalogName,"
+ " cm.metalake_id as metalakeId, cm.type, cm.provider,"
+ " cm.catalog_comment as catalogComment, cm.properties, cm.audit_info as auditInfo,"
+ " cm.current_version as currentVersion, cm.last_version as lastVersion,"
+ " cm.deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " cm JOIN "
+ MetalakeMetaMapper.TABLE_NAME
+ " mm ON cm.metalake_id = mm.metalake_id"
+ " WHERE mm.metalake_name = #{metalakeName}"
+ " AND mm.deleted_at = 0 AND cm.deleted_at = 0";
}

public String listCatalogPOsByMetalakeAndCatalogName(
@Param("metalakeName") String metalakeName, @Param("catalogName") String catalogName) {
return "SELECT cm.catalog_id as catalogId, cm.catalog_name as catalogName,"
+ " cm.metalake_id as metalakeId, cm.type, cm.provider,"
+ " cm.catalog_comment as catalogComment, cm.properties, cm.audit_info as auditInfo,"
+ " cm.current_version as currentVersion, cm.last_version as lastVersion,"
+ " cm.deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " cm JOIN "
+ MetalakeMetaMapper.TABLE_NAME
+ " mm ON cm.metalake_id = mm.metalake_id"
+ " WHERE mm.metalake_name = #{metalakeName}"
+ " AND cm.catalog_name = #{catalogName}"
+ " AND mm.deleted_at = 0 AND cm.deleted_at = 0";
}

public String listCatalogPOs(
@Param("metalakeName") String metalakeName,
@Param("catalogName") String catalogName,
@Param("schemaName") String schemaName) {
return "SELECT cm.catalog_id as catalogId, cm.catalog_name as catalogName,"
+ " cm.metalake_id as metalakeId, cm.type, cm.provider,"
+ " cm.catalog_comment as catalogComment, cm.properties, cm.audit_info as auditInfo,"
+ " cm.current_version as currentVersion, cm.last_version as lastVersion,"
+ " cm.deleted_at as deletedAt"
+ " FROM "
+ TABLE_NAME
+ " cm JOIN "
+ SchemaMetaMapper.TABLE_NAME
+ " sm ON cm.catalog_id = sm.catalog_id"
+ " JOIN "
+ MetalakeMetaMapper.TABLE_NAME
+ " mm ON cm.metalake_id = mm.metalake_id"
+ " WHERE mm.metalakeName = #{metalakeName}"
+ " AND cm.catalog_name = #{catalogName}"
+ " AND sm.schema_name = #{schemaName}"
+ " AND cm.deleted_at = 0 AND sm.deleted_at = 0 AND mm.deleted_at = 0";
}

public String listCatalogPOsByMetalakeId(@Param("metalakeId") Long metalakeId) {
return "SELECT catalog_id as catalogId, catalog_name as catalogName,"
+ " metalake_id as metalakeId, type, provider,"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import com.google.common.base.Preconditions;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Function;
Expand Down Expand Up @@ -119,12 +120,30 @@ public CatalogEntity getCatalogByIdentifier(NameIdentifier identifier) {

public List<CatalogEntity> listCatalogsByNamespace(Namespace namespace) {
NamespaceUtil.checkCatalog(namespace);
List<CatalogPO> catalogPOS = Collections.emptyList();
if (namespace.levels().length >= 1) {
catalogPOS =
SessionUtils.getWithoutCommit(
CatalogMetaMapper.class,
mapper -> mapper.listCatalogPOsByMetalakeName(namespace.level(0)));
}
if (namespace.levels().length >= 2) {
catalogPOS =
SessionUtils.getWithoutCommit(
CatalogMetaMapper.class,
mapper ->
mapper.listCatalogPOsByMetalakeAndCatalogName(
namespace.level(0), namespace.level(1)));
}

Long metalakeId = CommonMetaService.getInstance().getParentEntityIdByNamespace(namespace);

List<CatalogPO> catalogPOS =
SessionUtils.getWithoutCommit(
CatalogMetaMapper.class, mapper -> mapper.listCatalogPOsByMetalakeId(metalakeId));
if (namespace.levels().length >= 3) {
catalogPOS =
SessionUtils.getWithoutCommit(
CatalogMetaMapper.class,
mapper ->
mapper.listCatalogPOs(
namespace.level(0), namespace.level(1), namespace.level(2)));
}

return POConverters.fromCatalogPOs(catalogPOS, namespace);
}
Expand Down