Skip to content

Commit 5cadde6

Browse files
igorbernstein2igorbernstein2
authored
test: push hbase minicluster out to a separate classpath (#4270)
* test: push hbase minicluster out to a separate classpath This should allow for much more accurate testing of dependencies Change-Id: Ifd2a81fd7a465705230e5f341b00394ecb62de43 * format Change-Id: Ib7fd31fa4b1468f62094041f0a7e10d960a55181 * missing version bits Change-Id: I3e18340f0ddf46d27ff16dd0ef25a60c826f272c * deps: remove unused test dependency Change-Id: Iff9c565a89bc94a32712533a7bed56f6c3b4019f * oops, forgot to propagate the miniclsuter start error Change-Id: I762c03ad271d60e08704a2f93ccb3b99293b1b3f
1 parent 0d9ec40 commit 5cadde6

File tree

13 files changed

+473
-105
lines changed

13 files changed

+473
-105
lines changed

‎bigtable-client-core-parent/bigtable-hbase-integration-tests-common/pom.xml‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,29 @@ limitations under the License.
4242
</dependencyManagement>
4343

4444
<dependencies>
45+
<dependency>
46+
<groupId>org.apache.hbase</groupId>
47+
<artifactId>hbase-shaded-client</artifactId>
48+
<version>${hbase1.version}</version>
49+
<exclusions>
50+
<exclusion>
51+
<groupId>log4j</groupId>
52+
<artifactId>log4j</artifactId>
53+
</exclusion>
54+
</exclusions>
55+
</dependency>
56+
<dependency>
57+
<groupId>ch.qos.reload4j</groupId>
58+
<artifactId>reload4j</artifactId>
59+
<version>${reload4j.version}</version>
60+
</dependency>
61+
62+
<!-- Only used for BigtableConfiguration -->
4563
<dependency>
4664
<groupId>com.google.cloud.bigtable</groupId>
4765
<artifactId>bigtable-hbase</artifactId>
4866
<version>${project.version}</version>
49-
<scope>test</scope>
67+
<scope>provided</scope>
5068
<exclusions>
5169
<exclusion>
5270
<groupId>org.apache.hbase</groupId>
@@ -61,20 +79,6 @@ limitations under the License.
6179
<scope>test</scope>
6280
</dependency>
6381

64-
<dependency>
65-
<groupId>org.apache.hbase</groupId>
66-
<artifactId>hbase-shaded-testing-util</artifactId>
67-
<version>${hbase1.version}</version>
68-
<scope>test</scope>
69-
<exclusions>
70-
<exclusion>
71-
<groupId>log4j</groupId>
72-
<artifactId>log4j</artifactId>
73-
</exclusion>
74-
</exclusions>
75-
</dependency>
76-
77-
7882
<dependency>
7983
<groupId>commons-lang</groupId>
8084
<artifactId>commons-lang</artifactId>

‎bigtable-client-core-parent/bigtable-hbase-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/test_env/EmulatorEnv.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
2020
import org.apache.hadoop.hbase.HBaseConfiguration;
2121

22-
public class EmulatorEnv extends SharedTestEnv {
22+
class EmulatorEnv extends SharedTestEnv {
2323
private EmulatorController emulator;
2424

2525
@Override

‎bigtable-client-core-parent/bigtable-hbase-integration-tests-common/src/test/java/com/google/cloud/bigtable/hbase/test_env/MiniClusterEnv.java‎

Lines changed: 5 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -19,50 +19,20 @@
1919
import org.apache.commons.logging.Log;
2020
import org.apache.commons.logging.LogFactory;
2121
import org.apache.hadoop.hbase.HBaseConfiguration;
22-
import org.apache.hadoop.hbase.HBaseTestingUtility;
23-
import org.apache.hadoop.hbase.ipc.HBaseRpcController;
2422

2523
class MiniClusterEnv extends SharedTestEnv {
24+
private static final String PORT_KEY = "hbase.zookeeper.property.clientPort";
2625
private static final Log LOG = LogFactory.getLog(MiniClusterEnv.class);
2726

28-
static {
29-
try {
30-
HBaseRpcController.class.getName();
31-
} catch (Throwable t) {
32-
t.printStackTrace();
33-
}
34-
}
35-
36-
private HBaseTestingUtility helper;
37-
3827
@Override
3928
protected void setup() throws Exception {
40-
LOG.info("Starting hbase minicluster");
41-
42-
System.setProperty(
43-
"org.apache.hadoop.hbase.shaded.io.netty.packagePrefix", "org.apache.hadoop.hbase.shaded.");
44-
45-
helper = HBaseTestingUtility.createLocalHTU();
46-
helper.startMiniCluster();
47-
48-
// Need to create a separate config for the client to avoid
49-
// leaking hadoop configs, which messes up local mapreduce jobs
5029
configuration = HBaseConfiguration.create();
5130

52-
String[] keys = new String[] {"hbase.zookeeper.quorum", "hbase.zookeeper.property.clientPort"};
53-
for (String key : keys) {
54-
configuration.set(key, helper.getConfiguration().get(key));
55-
}
56-
LOG.info("Test dir: " + helper.getDataTestDir());
31+
int miniClusterPort = Integer.getInteger(PORT_KEY);
32+
LOG.info("MiniCluster port: " + miniClusterPort);
33+
configuration.setInt(PORT_KEY, miniClusterPort);
5734
}
5835

5936
@Override
60-
protected void teardown() throws IOException {
61-
helper.shutdownMiniHBaseCluster();
62-
System.out.println("Cleaning up testDir: " + helper.getDataTestDir());
63-
if (!helper.cleanupTestDir()) {
64-
LOG.warn("Failed to clean up testDir");
65-
}
66-
helper = null;
67-
}
37+
protected void teardown() throws IOException {}
6838
}

‎bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/pom.xml‎

Lines changed: 58 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,22 @@ limitations under the License.
8989
<id>hbaseLocalMiniClusterTest</id>
9090
<build>
9191
<plugins>
92+
<!-- start & stop the minicluster around the integration tests -->
93+
<plugin>
94+
<groupId>com.google.cloud.bigtable</groupId>
95+
<artifactId>hbase-minicluster-maven-plugin</artifactId>
96+
<version>2.12.1-SNAPSHOT</version>
97+
98+
<executions>
99+
<execution>
100+
<goals>
101+
<goal>start</goal>
102+
<goal>stop</goal>
103+
</goals>
104+
</execution>
105+
</executions>
106+
</plugin>
107+
92108
<plugin>
93109
<groupId>org.apache.maven.plugins</groupId>
94110
<artifactId>maven-failsafe-plugin</artifactId>
@@ -113,6 +129,8 @@ limitations under the License.
113129
<reportsDirectory>${project.build.directory}/failsafe-reports/minicluster-tests</reportsDirectory>
114130
<systemPropertyVariables>
115131
<google.bigtable.test_env>minicluster</google.bigtable.test_env>
132+
<!-- populated by hbase-minicluster-maven-plugin -->
133+
<hbase.zookeeper.property.clientPort>${hbase.zookeeper.property.clientPort}</hbase.zookeeper.property.clientPort>
116134
</systemPropertyVariables>
117135
</configuration>
118136
</execution>
@@ -159,33 +177,12 @@ limitations under the License.
159177
</profile>
160178
</profiles>
161179

162-
<dependencyManagement>
163-
<dependencies>
164-
<dependency>
165-
<groupId>com.google.cloud</groupId>
166-
<artifactId>google-cloud-bigtable-bom</artifactId>
167-
<version>${bigtable.version}</version>
168-
<type>pom</type>
169-
<scope>import</scope>
170-
</dependency>
171-
172-
<dependency>
173-
<groupId>com.google.cloud</groupId>
174-
<artifactId>google-cloud-bigtable-deps-bom</artifactId>
175-
<version>${bigtable.version}</version>
176-
<type>pom</type>
177-
<scope>import</scope>
178-
</dependency>
179-
</dependencies>
180-
</dependencyManagement>
181180

182181
<dependencies>
183-
<!-- HBase testing tools -->
184182
<dependency>
185183
<groupId>org.apache.hbase</groupId>
186-
<artifactId>hbase-shaded-testing-util</artifactId>
184+
<artifactId>hbase-shaded-client</artifactId>
187185
<version>${hbase1.version}</version>
188-
<scope>test</scope>
189186
<exclusions>
190187
<exclusion>
191188
<groupId>log4j</groupId>
@@ -197,7 +194,6 @@ limitations under the License.
197194
<groupId>ch.qos.reload4j</groupId>
198195
<artifactId>reload4j</artifactId>
199196
<version>${reload4j.version}</version>
200-
<scope>runtime</scope>
201197
</dependency>
202198

203199
<!-- Project Modules -->
@@ -206,13 +202,6 @@ limitations under the License.
206202
<artifactId>bigtable-hbase-1.x</artifactId>
207203
<version>2.12.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
208204
<scope>test</scope>
209-
<exclusions>
210-
<!-- included in hbase-shaded-testing-util -->
211-
<exclusion>
212-
<groupId>org.apache.hbase</groupId>
213-
<artifactId>hbase-shaded-client</artifactId>
214-
</exclusion>
215-
</exclusions>
216205
</dependency>
217206

218207
<dependency>
@@ -236,13 +225,6 @@ limitations under the License.
236225
<scope>test</scope>
237226
</dependency>
238227

239-
<!-- java-bigtable Modules -->
240-
<dependency>
241-
<groupId>com.google.api.grpc</groupId>
242-
<artifactId>grpc-google-cloud-bigtable-v2</artifactId>
243-
<scope>test</scope>
244-
</dependency>
245-
246228
<!-- Misc -->
247229
<dependency>
248230
<groupId>commons-lang</groupId>
@@ -323,7 +305,46 @@ limitations under the License.
323305
<!-- End Skip publishing -->
324306
</plugins>
325307
</pluginManagement>
308+
326309
<plugins>
310+
<plugin>
311+
<groupId>com.google.cloud.bigtable</groupId>
312+
<artifactId>hbase-minicluster-maven-plugin</artifactId>
313+
<version>2.12.1-SNAPSHOT</version> <!-- {x-version-update:bigtable-client-parent:current} -->
314+
315+
<dependencies>
316+
<dependency>
317+
<groupId>org.apache.hbase</groupId>
318+
<artifactId>hbase-shaded-testing-util</artifactId>
319+
<version>${hbase1.version}</version>
320+
<exclusions>
321+
<!-- Reroute all logging to slf4j & simple-logger in maven -->
322+
<exclusion>
323+
<groupId>org.slf4j</groupId>
324+
<artifactId>slf4j-log4j12</artifactId>
325+
</exclusion>
326+
<exclusion>
327+
<groupId>log4j</groupId>
328+
<artifactId>log4j</artifactId>
329+
</exclusion>
330+
</exclusions>
331+
</dependency>
332+
333+
<!-- Force upgrade commons-logging to 1.3.0 so that it re-routes over slf4j which maven uses -->
334+
<dependency>
335+
<groupId>commons-logging</groupId>
336+
<artifactId>commons-logging</artifactId>
337+
<version>1.3.0</version>
338+
</dependency>
339+
<!-- Force log4j messages -> slf4j -> maven's simplelogger -->
340+
<dependency>
341+
<groupId>org.slf4j</groupId>
342+
<artifactId>log4j-over-slf4j</artifactId>
343+
<version>1.7.36</version>
344+
</dependency>
345+
</dependencies>
346+
</plugin>
347+
327348
<plugin>
328349
<groupId>org.apache.maven.plugins</groupId>
329350
<artifactId>maven-surefire-plugin</artifactId>

‎bigtable-hbase-1.x-parent/bigtable-hbase-1.x-integration-tests/src/test/java/com/google/cloud/bigtable/hbase/TestFilters.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,11 @@
3434
import org.apache.hadoop.hbase.util.Bytes;
3535
import org.junit.Assert;
3636
import org.junit.Test;
37+
import org.junit.experimental.categories.Category;
3738

3839
public class TestFilters extends AbstractTestFilters {
39-
40+
// This test can't run against the minicluster because its a Bigtable extension
41+
@Category(KnownHBaseGap.class)
4042
@Test
4143
public void testTimestampRangeFilter() throws IOException {
4244
// Initialize
@@ -66,6 +68,8 @@ public void testTimestampRangeFilter() throws IOException {
6668
table.close();
6769
}
6870

71+
// This test can't run against the minicluster because its a Bigtable extension
72+
@Category(KnownHBaseGap.class)
6973
@Test
7074
public void testBigtableFilter() throws IOException {
7175
if (!sharedTestEnv.isBigtable()) {
@@ -99,6 +103,8 @@ public void testBigtableFilter() throws IOException {
99103
*
100104
* @throws IOException
101105
*/
106+
// This test can't run against the minicluster because its a Bigtable extension
107+
@Category(KnownHBaseGap.class)
102108
@Test
103109
public void testTimestampRangeFilterWithMaxVal() throws IOException {
104110
// Initialize

0 commit comments

Comments
 (0)