[#9429] improvement(core): Use join to improve query performance on table, topic and so on. (#9430)

### What changes were proposed in this pull request?

This pull request introduces major performance and functionality
improvements to the Metalake and table metadata management system. The
most significant change is the addition of a Caffeine-based cache for
Metalake entities, which reduces redundant storage access and improves
efficiency. Additionally, it adds a new method for querying tables by
their fully qualified names and optimizes database schema indexing for
faster lookups.

### Metalake caching and management improvements

* Introduced a static Caffeine cache (`metalakeCache`) in
`MetalakeManager` to store `BaseMetalake` objects, with automatic expiry
and scheduled cleanup. This cache is now used in `metalakeInUse`,
`loadMetalake`, and is invalidated on alter, drop, enable, and disable
operations to ensure consistency.
[[1]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R23-R36)
[[2]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R74-R86)
[[3]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0L87-R107)
[[4]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R138-R155)
[[5]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R194-R209)
[[6]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R301-R302)
[[7]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R353-R354)
[[8]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R381)
[[9]](diffhunk://#diff-21ed56786d4199a9ebee9969aae3195dff5037135c738ff4b8ba2072fa72d6a0R416)

### Table metadata querying enhancements

* Added a new method `selectTableByFullQualifiedName` to
`TableMetaMapper`, its SQL provider, and the service layer, allowing
retrieval of table metadata using metalake, catalog, schema, and table
names directly, improving query flexibility and performance.
[[1]](diffhunk://#diff-464816dcd4652ac061d317d0ed9ab3a98db358f2db517ce70bbf61fa35624db9R59-R67)
[[2]](diffhunk://#diff-e160fd550934e3ac047abde64991c6ab5edc44602bf8cfd8587596f10b0b58efR72-R80)
[[3]](diffhunk://#diff-19aff10a8b33dded8f085dc93250ba296ca800b5a68630a22537e83b0bd9b730R236-R292)
[[4]](diffhunk://#diff-c0bc429f18cd462c3a3c1fe65c7da0966c338a96737d69b6e32b17257d97a52cL88-L92)
[[5]](diffhunk://#diff-c0bc429f18cd462c3a3c1fe65c7da0966c338a96737d69b6e32b17257d97a52cR347-R363)

### Database schema and indexing optimizations

* Added composite indexes (`idx_name_da`) on the `metalake_name`,
`catalog_name`, `schema_name`, and `table_name` columns (with
`deleted_at`) in both H2 and MySQL schema scripts, significantly
improving query performance for lookups by name.
[[1]](diffhunk://#diff-3ca2c363f5538fb4838a31233c4c858b4928e9ec65b5b86bf35a8c5f653133bfR31)
[[2]](diffhunk://#diff-3ca2c363f5538fb4838a31233c4c858b4928e9ec65b5b86bf35a8c5f653133bfR49)
[[3]](diffhunk://#diff-3ca2c363f5538fb4838a31233c4c858b4928e9ec65b5b86bf35a8c5f653133bfR66)
[[4]](diffhunk://#diff-3ca2c363f5538fb4838a31233c4c858b4928e9ec65b5b86bf35a8c5f653133bfR83)
[[5]](diffhunk://#diff-6fac45e3da22ec7d24ca5ec0b78c5a03d653fabd65dd8040b2c59404fc49fb21R33-R38)
[[6]](diffhunk://#diff-ac6c56c7547bd70d3951716f2b543beea23964b906792b527b833e6229404f70R31)
[[7]](diffhunk://#diff-ac6c56c7547bd70d3951716f2b543beea23964b906792b527b833e6229404f70R48)
[[8]](diffhunk://#diff-ac6c56c7547bd70d3951716f2b543beea23964b906792b527b833e6229404f70R64)
[[9]](diffhunk://#diff-ac6c56c7547bd70d3951716f2b543beea23964b906792b527b833e6229404f70R83)
### Why are the changes needed?

To improve performance

Fix: #9429 

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

Existing test

Author: yuqi1129

authorizations/authorization-chain/build.gradle.kts

@@ -60,6 +60,7 @@ dependencies {
   testImplementation(project(":integration-test-common", "testArtifacts"))
   testImplementation(project(":authorizations:authorization-ranger"))
   testImplementation(project(":authorizations:authorization-ranger", "testArtifacts"))
+  testImplementation(libs.awaitility)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
   testImplementation(libs.testcontainers)

authorizations/authorization-ranger/build.gradle.kts

@@ -80,6 +80,7 @@ dependencies {
   testImplementation(project(":clients:client-java"))
   testImplementation(project(":server"))
   testImplementation(project(":integration-test-common", "testArtifacts"))
+  testImplementation(libs.awaitility)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.mockito.core)
   testImplementation(libs.mockito.inline)

build.gradle.kts

@@ -154,6 +154,11 @@ allprojects {
           "import\\s+.*\\.(Logger|LoggerFactory);",
           "import org.slf4j.${'$'}1;"
         )
+        replaceRegex(
+          "Remove Testcontainers shading",
+          "import\\s+org\\.testcontainers\\.shaded\\.([^;]+);",
+          "import $1;"
+        )
 
         targetExclude("**/build/**", "**/.pnpm/***")
       }

catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java

@@ -111,6 +111,7 @@
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
@@ -121,7 +122,6 @@
 import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.MockedStatic;
 import org.mockito.Mockito;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 
 public class TestFilesetCatalogOperations {
 
@@ -216,7 +216,7 @@ private static CatalogInfo randomCatalogInfo(
   }
 
   @BeforeAll
-  public static void setUp() {
+  public static void setUp() throws IllegalAccessException {
     Config config = Mockito.mock(Config.class);
     when(config.get(ENTITY_STORE)).thenReturn(RELATIONAL_ENTITY_STORE);
     when(config.get(ENTITY_RELATIONAL_STORE)).thenReturn(DEFAULT_ENTITY_RELATIONAL_STORE);
@@ -336,6 +336,8 @@ public static void setUp() {
     schemaMetaServiceMockedStatic
         .when(SchemaMetaService::getInstance)
         .thenReturn(spySchemaMetaService);
+
+    FieldUtils.writeField(GravitinoEnv.getInstance(), "config", config, true);
   }
 
   @AfterAll

catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/integration/test/FilesetS3CatalogIT.java

@@ -38,6 +38,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
@@ -46,7 +47,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.Container;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 
 @Tag("gravitino-docker-test")
 public class FilesetS3CatalogIT extends FilesetCatalogIT {

catalogs/catalog-hive/build.gradle.kts

@@ -65,6 +65,7 @@ dependencies {
   annotationProcessor(libs.immutables.value)
   annotationProcessor(libs.lombok)
 
+  testImplementation(libs.awaitility)
   testImplementation(project(":catalogs:hive-metastore-common", "testArtifacts"))
   testImplementation(project(":common"))
   testImplementation(project(":clients:client-java"))

catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveS3IT.java

@@ -29,11 +29,11 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.spark.sql.SparkSession;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.TestInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.testcontainers.containers.Container;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 public class CatalogHiveS3IT extends CatalogHive3IT {

catalogs/catalog-jdbc-doris/build.gradle.kts

@@ -49,6 +49,7 @@ dependencies {
   testImplementation(project(":server"))
   testImplementation(project(":server-common"))
 
+  testImplementation(libs.awaitility)
   testImplementation(libs.junit.jupiter.api)
   testImplementation(libs.junit.jupiter.params)
   testImplementation(libs.mysql.driver)

catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java

@@ -78,13 +78,13 @@
 import org.apache.gravitino.rel.partitions.RangePartition;
 import org.apache.gravitino.rel.types.Types;
 import org.apache.gravitino.utils.RandomNameUtils;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 
 @Tag("gravitino-docker-test")
 public class CatalogDorisIT extends BaseIT {

catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/operation/TestDorisTableOperations.java

@@ -55,11 +55,11 @@
 import org.apache.gravitino.rel.types.Type;
 import org.apache.gravitino.rel.types.Types;
 import org.apache.gravitino.utils.RandomNameUtils;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import org.testcontainers.shaded.org.awaitility.Awaitility;
 
 @Tag("gravitino-docker-test")
 public class TestDorisTableOperations extends TestDoris {