Skip to content

Commit 93bdc3b

Browse files
authored
[#9581] fix(hive): Perform resource cleanup in HiveClientPool close (#9617)
<!-- 1. Title: [#<issue>] <type>(<scope>): <subject> Examples: - "[#123] feat(operator): support xxx" - "[#233] fix: check null before access result in xxx" - "[MINOR] refactor: fix typo in variable name" - "[MINOR] docs: fix typo in README" - "[#255] test: fix flaky test NameOfTheTest" Reference: https://www.conventionalcommits.org/en/v1.0.0/ 2. If the PR is unfinished, please mark this PR as draft. --> ### What changes were proposed in this pull request? This PR performs actual resource cleanup in `HiveClientPool.close` ### Why are the changes needed? Any code path in the current implementation that close the pool via `ClientPoolImpl.close()` will cause the connection leaks because the HiveClientPool.close is a no-op. Fix: #9581 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add UTs.
1 parent 05c3312 commit 93bdc3b

File tree

3 files changed

+105
-1
lines changed

3 files changed

+105
-1
lines changed

catalogs/hive-metastore-common/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ dependencies {
120120
exclude("org.slf4j")
121121
}
122122
testImplementation(libs.junit.jupiter.api)
123+
testImplementation(libs.mockito.core)
123124
testImplementation(libs.woodstox.core)
124125
testImplementation(libs.testcontainers)
125126
testImplementation(project(":integration-test-common", "testArtifacts"))

catalogs/hive-metastore-common/src/main/java/org/apache/gravitino/hive/HiveClientPool.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public class HiveClientPool extends ClientPoolImpl<HiveClient, GravitinoRuntimeE
4141
* @param properties The configuration used to initialize the Hive Metastore clients.
4242
*/
4343
public HiveClientPool(String name, int poolSize, Properties properties) {
44-
// Do not allow retry by default as we rely on RetryingHiveClient
44+
// Do not allow retry by default as we rely on RetryingMetaStoreClient
4545
super(poolSize, GravitinoRuntimeException.class, false);
4646
this.clientFactory = new HiveClientFactory(properties, name);
4747
}
@@ -58,17 +58,20 @@ protected HiveClient newClient() {
5858

5959
@Override
6060
protected HiveClient reconnect(HiveClient client) {
61+
// No-op reconnect: RetryingMetaStoreClient handles reconnect logic.
6162
LOG.warn("Reconnecting to Hive Metastore");
6263
return client;
6364
}
6465

6566
@Override
6667
protected boolean isConnectionException(Exception e) {
68+
// Pool-level reconnection is not required by design.
6769
return false;
6870
}
6971

7072
@Override
7173
protected void close(HiveClient client) {
7274
LOG.info("Closing Hive Metastore client");
75+
client.close();
7376
}
7477
}
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.gravitino.hive;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertThrows;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
import com.google.common.collect.Lists;
27+
import java.util.List;
28+
import java.util.Properties;
29+
import org.apache.gravitino.exceptions.GravitinoRuntimeException;
30+
import org.apache.gravitino.hive.client.HiveClient;
31+
import org.junit.jupiter.api.AfterEach;
32+
import org.junit.jupiter.api.BeforeEach;
33+
import org.junit.jupiter.api.Test;
34+
import org.mockito.Mockito;
35+
36+
/**
37+
* Referenced from Apache Iceberg's {@code TestHiveClientPool} implementation.
38+
*
39+
* <p>Source: hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveClientPool.java
40+
*/
41+
public class TestHiveClientPool {
42+
43+
private HiveClientPool clients;
44+
45+
@BeforeEach
46+
public void before() {
47+
HiveClientPool clientPool = new HiveClientPool("hive", 2, new Properties());
48+
clients = Mockito.spy(clientPool);
49+
}
50+
51+
@AfterEach
52+
public void after() {
53+
clients.close();
54+
clients = null;
55+
}
56+
57+
@Test
58+
public void testNewClientFailure() {
59+
Mockito.doThrow(new RuntimeException("Connection exception")).when(clients).newClient();
60+
RuntimeException ex = assertThrows(RuntimeException.class, () -> clients.run(Object::toString));
61+
assertEquals("Connection exception", ex.getMessage());
62+
}
63+
64+
@Test
65+
public void testReconnect() {
66+
HiveClient hiveClient = newClient();
67+
68+
String metaMessage = "Got exception: org.apache.thrift.transport.TTransportException";
69+
Mockito.doThrow(new GravitinoRuntimeException(metaMessage))
70+
.when(hiveClient)
71+
.getAllDatabases("");
72+
73+
GravitinoRuntimeException ex =
74+
assertThrows(
75+
GravitinoRuntimeException.class,
76+
() -> clients.run(client -> client.getAllDatabases("")));
77+
assertEquals("Got exception: org.apache.thrift.transport.TTransportException", ex.getMessage());
78+
// Verify that the method is never called.
79+
Mockito.verify(clients, Mockito.never()).reconnect(hiveClient);
80+
}
81+
82+
@Test
83+
public void testClose() throws Exception {
84+
HiveClient hiveClient = newClient();
85+
86+
List<String> databases = Lists.newArrayList("db1", "db2");
87+
Mockito.doReturn(databases).when(hiveClient).getAllDatabases("");
88+
assertEquals(clients.run(client -> client.getAllDatabases("")), databases);
89+
90+
clients.close();
91+
assertTrue(clients.isClosed());
92+
Mockito.verify(hiveClient).close();
93+
}
94+
95+
private HiveClient newClient() {
96+
HiveClient hiveClient = Mockito.mock(HiveClient.class);
97+
Mockito.doReturn(hiveClient).when(clients).newClient();
98+
return hiveClient;
99+
}
100+
}

0 commit comments

Comments
 (0)