Skip to content

Commit

Permalink
[apache#2388] feat(spark-connector): filter non relational catalogs b…
Browse files Browse the repository at this point in the history
…efore register to spark catalog manager (apache#2830)

### What changes were proposed in this pull request?
filter non relational catalogs before register to spark catalog manager

### Why are the changes needed?
Fix: apache#2388 

### Does this PR introduce _any_ user-facing change?
no

### How was this patch tested?
existing test
  • Loading branch information
FANNG1 authored Apr 8, 2024
1 parent 2b8f60e commit c88506f
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -73,15 +73,15 @@ public String getMetalakeName() {
return metalakeName;
}

public Set<String> listCatalogs() {
NameIdentifier[] catalogNames = metalake.listCatalogs(Namespace.ofCatalog(metalake.name()));
LOG.info(
"Load metalake {}'s catalogs. catalogs: {}.",
metalake.name(),
Arrays.toString(catalogNames));
return Arrays.stream(catalogNames)
.map(identifier -> identifier.name())
.collect(Collectors.toSet());
public void loadRelationalCatalogs() {
Catalog[] catalogs = metalake.listCatalogsInfo(Namespace.ofCatalog(metalake.name()));
Arrays.stream(catalogs)
.filter(catalog -> Catalog.Type.RELATIONAL.equals(catalog.type()))
.forEach(catalog -> gravitinoCatalogs.put(catalog.name(), catalog));
}

public Set<String> getCatalogNames() {
return gravitinoCatalogs.asMap().keySet().stream().collect(Collectors.toSet());
}

private Catalog loadCatalog(String catalogName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ public Map<String, String> init(SparkContext sc, PluginContext pluginContext) {
"%s:%s, should not be empty", GravitinoSparkConfig.GRAVITINO_METALAKE, metalake));

catalogManager = GravitinoCatalogManager.create(gravitinoUri, metalake);
Set<String> catalogs = catalogManager.listCatalogs();
registerGravitinoCatalogs(conf, catalogs);
catalogManager.loadRelationalCatalogs();
Set<String> catalogNames = catalogManager.getCatalogNames();
registerGravitinoCatalogs(conf, catalogNames);
registerSqlExtensions();
return Collections.emptyMap();
}
Expand Down

0 comments on commit c88506f

Please sign in to comment.