diff --git a/be/src/information_schema/schema_schemata_scanner.cpp b/be/src/information_schema/schema_schemata_scanner.cpp index e9d25684780..15e2528a89f 100644 --- a/be/src/information_schema/schema_schemata_scanner.cpp +++ b/be/src/information_schema/schema_schemata_scanner.cpp @@ -52,6 +52,8 @@ Status SchemaSchemataScanner::start(RuntimeState* state) { return Status::InternalError("used before initial."); } TGetDbsParams db_params; + // SCHEMATA never reads database IDs; avoid creating one database object per namespace in FE. + db_params.__set_include_db_ids(false); if (nullptr != _param->common_param->wild) { db_params.__set_pattern(*(_param->common_param->wild)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 23c3a74f1d4..30493552832 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -345,7 +345,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -363,6 +362,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.ObjLongConsumer; import java.util.stream.Collectors; // Frontend service used to serve all request for this frontend through @@ -496,7 +496,7 @@ public class FrontendServiceImpl implements FrontendService.Iface { List dbIds = Lists.newArrayList(); List catalogIds = Lists.newArrayList(); - PatternMatcher matcher = null; + PatternMatcher matcher; if (params.isSetPattern()) { try { matcher = PatternMatcher.createMysqlPattern(params.getPattern(), @@ -504,6 +504,8 @@ public class FrontendServiceImpl implements FrontendService.Iface { } catch (PatternMatcherException e) { throw new TException("Pattern is in bad format: " + params.getPattern()); } + } else { + matcher = null; } Env env = Env.getCurrentEnv(); @@ -517,55 +519,77 @@ public class FrontendServiceImpl implements FrontendService.Iface { catalog -> new TException("Unknown catalog " + catalog))); } + boolean includeDbIds = params.isIncludeDbIds(); for (CatalogIf catalog : catalogIfs) { - Collection dbs = new HashSet(); + Collection dbs = Collections.emptyList(); + List names = Collections.emptyList(); try { - dbs = catalog.getAllDbs(); + if (includeDbIds) { + dbs = catalog.getAllDbs(); + } else { + // SCHEMATA only needs names; loading every database churns the object cache on large catalogs. + names = catalog.getDbNames(); + } } catch (Exception e) { LOG.warn("failed to get database names for catalog {}", catalog.getName(), e); // Some external catalog may fail to get databases due to wrong connection info. } + int dbCount = includeDbIds ? dbs.size() : names.size(); if (LOG.isDebugEnabled()) { - LOG.debug("get db size: {}, in catalog: {}", dbs.size(), catalog.getName()); + LOG.debug("get db size: {}, in catalog: {}", dbCount, catalog.getName()); } - if (dbs.isEmpty() && params.isSetGetNullCatalog() && params.get_null_catalog) { + if (dbCount == 0 && params.isSetGetNullCatalog() && params.get_null_catalog) { catalogNames.add(catalog.getName()); dbNames.add("NULL"); catalogIds.add(catalog.getId()); - dbIds.add(-1L); + if (includeDbIds) { + dbIds.add(-1L); + } continue; } - if (dbs.isEmpty()) { + if (dbCount == 0) { continue; } - UserIdentity currentUser = null; + UserIdentity currentUser; if (params.isSetCurrentUserIdent()) { currentUser = UserIdentity.fromThrift(params.current_user_ident); } else { currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip); } - for (DatabaseIf db : dbs) { - String dbName = db.getFullName(); + ObjLongConsumer addDatabase = (dbName, dbId) -> { if (!env.getAccessManager() .checkDbPriv(currentUser, catalog.getName(), dbName, PrivPredicate.SHOW)) { - continue; + return; } if (matcher != null && !matcher.match(getMysqlTableSchema(catalog.getName(), dbName))) { - continue; + return; } catalogNames.add(catalog.getName()); dbNames.add(getMysqlTableSchema(catalog.getName(), dbName)); catalogIds.add(catalog.getId()); - dbIds.add(db.getId()); + if (includeDbIds) { + dbIds.add(dbId); + } + }; + if (includeDbIds) { + for (DatabaseIf db : dbs) { + addDatabase.accept(db.getFullName(), db.getId()); + } + } else { + for (String dbName : names) { + addDatabase.accept(dbName, -1L); + } } } result.setDbs(dbNames); result.setCatalogs(catalogNames); result.setCatalogIds(catalogIds); - result.setDbIds(dbIds); + if (includeDbIds) { + result.setDbIds(dbIds); + } return result; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceGetDbNamesTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceGetDbNamesTest.java new file mode 100644 index 00000000000..236569257fd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceGetDbNamesTest.java @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.service; + +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.thrift.TGetDbsParams; +import org.apache.doris.thrift.TGetDbsResult; + +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TSerializer; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class FrontendServiceGetDbNamesTest { + private final Env env = Mockito.mock(Env.class); + private final CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + private final CatalogIf catalog = Mockito.mock(CatalogIf.class); + private final FrontendServiceImpl service = new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class)); + private MockedStatic envMock; + + @BeforeEach + public void setUp() { + envMock = Mockito.mockStatic(Env.class); + envMock.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(catalogMgr.listCatalogs()).thenReturn(Collections.singletonList(catalog)); + Mockito.when(catalog.getName()).thenReturn("example_catalog"); + Mockito.when(catalog.getId()).thenReturn(42L); + Mockito.when(accessManager.checkDbPriv(Mockito.any(UserIdentity.class), Mockito.anyString(), + Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW))).thenReturn(true); + } + + @AfterEach + public void tearDown() { + envMock.close(); + } + + @Test + public void testNamesOnlyDoesNotMaterializeDatabases() throws Exception { + List names = new ArrayList<>(); + for (int i = 0; i < 100_000; i++) { + names.add("namespace_" + i); + } + names.add(InfoSchemaDb.DATABASE_NAME); + names.add("mysql"); + Mockito.when(catalog.getDbNames()).thenReturn(names); + Mockito.when(catalog.getAllDbs()).thenThrow(new AssertionError("Database objects must not be loaded")); + + TGetDbsParams params = params(); + params.setIncludeDbIds(false); + TGetDbsResult result = service.getDbNames(params); + + Assertions.assertEquals(names.size(), result.getDbsSize()); + Assertions.assertEquals(names.size(), result.getCatalogsSize()); + Assertions.assertEquals(names.size(), result.getCatalogIdsSize()); + Assertions.assertTrue(result.getCatalogs().stream().allMatch("example_catalog"::equals)); + Assertions.assertTrue(result.getCatalogIds().stream().allMatch(id -> id == 42L)); + Assertions.assertFalse(result.isSetDbIds()); + Mockito.verify(catalog, Mockito.times(1)).getDbNames(); + Mockito.verify(catalog, Mockito.never()).getAllDbs(); + } + + @Test + public void testNamesOnlyPreservesPrivilegesPatternAndDisplayNames() throws Exception { + Mockito.when(catalog.getDbNames()).thenReturn(Arrays.asList("visible_a", "visible_hidden", "other")); + Mockito.when(accessManager.checkDbPriv(Mockito.any(UserIdentity.class), Mockito.eq("example_catalog"), + Mockito.eq("visible_hidden"), Mockito.eq(PrivPredicate.SHOW))).thenReturn(false); + + boolean original = GlobalVariable.showFullDbNameInInfoSchemaDb; + try { + for (boolean showFullName : new boolean[] {false, true}) { + GlobalVariable.showFullDbNameInInfoSchemaDb = showFullName; + TGetDbsParams params = params(); + params.setIncludeDbIds(false); + params.setPattern(showFullName ? "example_catalog.visible%" : "visible%"); + TGetDbsResult result = service.getDbNames(params); + Assertions.assertEquals(Collections.singletonList( + showFullName ? "example_catalog.visible_a" : "visible_a"), result.getDbs()); + Assertions.assertFalse(result.isSetDbIds()); + } + } finally { + GlobalVariable.showFullDbNameInInfoSchemaDb = original; + } + } + + @Test + public void testExistingCallersKeepDatabaseIds() throws Exception { + DatabaseIf database = Mockito.mock(DatabaseIf.class); + Mockito.when(database.getFullName()).thenReturn("visible_a"); + Mockito.when(database.getId()).thenReturn(123L); + Mockito.when(catalog.getAllDbs()).thenReturn(Collections.singletonList(database)); + + for (boolean explicitlySet : new boolean[] {false, true}) { + TGetDbsParams params = params(); + if (explicitlySet) { + params.setIncludeDbIds(true); + } + TGetDbsResult result = service.getDbNames(params); + Assertions.assertEquals(Collections.singletonList(123L), result.getDbIds()); + Assertions.assertEquals(Collections.singletonList(42L), result.getCatalogIds()); + } + Mockito.verify(catalog, Mockito.never()).getDbNames(); + } + + @Test + public void testAbsentIncludeDbIdsOnWireKeepsLegacyDefault() throws Exception { + TGetDbsParams legacy = params(); + legacy.unsetIncludeDbIds(); + TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory()); + TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory()); + TGetDbsParams decoded = new TGetDbsParams(); + deserializer.deserialize(decoded, serializer.serialize(legacy)); + Assertions.assertTrue(decoded.isIncludeDbIds()); + + legacy.setIncludeDbIds(false); + deserializer.deserialize(decoded, serializer.serialize(legacy)); + Assertions.assertFalse(decoded.isIncludeDbIds()); + } + + @Test + public void testNamesOnlyPreservesEmptyCatalogPlaceholder() throws Exception { + Mockito.when(catalog.getDbNames()).thenReturn(Collections.emptyList()); + TGetDbsParams params = params(); + params.setIncludeDbIds(false); + params.setGetNullCatalog(true); + + TGetDbsResult result = service.getDbNames(params); + Assertions.assertEquals(Collections.singletonList("NULL"), result.getDbs()); + Assertions.assertEquals(Collections.singletonList("example_catalog"), result.getCatalogs()); + Assertions.assertEquals(Collections.singletonList(42L), result.getCatalogIds()); + Assertions.assertFalse(result.isSetDbIds()); + } + + @Test + public void testNamesOnlySkipsUnavailableCatalog() throws Exception { + Mockito.when(catalog.getDbNames()).thenThrow(new IllegalStateException("Metadata service unavailable")); + TGetDbsParams params = params(); + params.setIncludeDbIds(false); + + TGetDbsResult result = service.getDbNames(params); + Assertions.assertTrue(result.getDbs().isEmpty()); + Assertions.assertTrue(result.getCatalogs().isEmpty()); + Assertions.assertFalse(result.isSetDbIds()); + } + + private TGetDbsParams params() { + TGetDbsParams params = new TGetDbsParams(); + params.setCurrentUserIdent(UserIdentity.ROOT.toThrift()); + return params; + } +} diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 45e63623fa8..094a52d22ef 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -101,6 +101,8 @@ struct TGetDbsParams { // Reserved for downstream field `current_roles` to keep thrift field ids // wire-compatible across maintained branches. Do not reuse this id. 7: optional set reserved_field_7 + // Name-only scanners can avoid materializing database objects and omit db_ids in the response. + 8: optional bool include_db_ids = true } // getDbNames returns a list of database names , database ids and catalog names ,catalog ids