keep trying to make tests pass

Author: charlesconnell

.github/workflows/precommit.yml

@@ -37,14 +37,14 @@ env:
   TESTS_FILTER: 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
   SUREFIRE_FIRST_PART_FORK_COUNT: '1C'
   SUREFIRE_SECOND_PART_FORK_COUNT: '1C'
+  BUILD_THREAD: '4'
   ASF_NIGHTLIES: 'https://nightlies.apache.org'
   EXCLUDE_TESTS_URL: 'https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes'
   BUILD_URL_ARTIFACTS: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'
 
 jobs:
   precommit-checks:
     runs-on: ubuntu-latest
-    timeout-minutes: 600
     continue-on-error: true
     strategy:
       fail-fast: false
@@ -78,7 +78,7 @@ jobs:
           echo "SOURCEDIR=${WORKDIR}/${SRC_REL}" >> $GITHUB_ENV
           echo "PATCHDIR=${WORKDIR}/${PATCH_REL}" >> $GITHUB_ENV
           echo "DOCKERFILE=${WORKDIR}/${SRC_REL}/dev-support/docker/Dockerfile" >> $GITHUB_ENV
-          echo "YETUSDIR=${WORKDIR}/yetus" >> $GITHUB_ENV
+          echo "YETUSDIR=${WORKDIR}/${YETUS_REL}" >> $GITHUB_ENV
           echo "CACHE_APACHE_ARTIFACT=${WORKDIR}/${SRC_REL}/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" >> $GITHUB_ENV
           echo "YETUS_DRIVER=${WORKDIR}/${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh" >> $GITHUB_ENV
           echo "HIDE_OLD_COMMENTS=${WORKDIR}/${SRC_REL}/dev-support/gh_hide_old_comments.sh" >> $GITHUB_ENV
@@ -108,6 +108,7 @@ jobs:
       - name: Run Yetus test-patch
         id: test-patch
         continue-on-error: true
+        timeout-minutes: 300
         working-directory: ${{ env.WORKDIR }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dev-support/hbase-personality.sh

@@ -168,6 +168,8 @@ function personality_modules
   # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
   if [[ "${testtype}" == mvnsite ]]; then
     yetus_debug "Skip specifying --threads since maven-site-plugin does not support building in parallel."
+  elif [[ "${testtype}" == unit ]]; then
+    yetus_debug "Skip specifying --threads since unit tests are parallelized via surefire test runner."
   else
     if [[ -n "${BUILD_THREAD}" ]]; then
       extra="--threads=${BUILD_THREAD}"

hbase-client/src/test/java/org/apache/hadoop/hbase/security/token/TestClientTokenUtil.java

@@ -29,7 +29,7 @@
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.client.AsyncConnection;
 import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.ClassRule;
@@ -41,7 +41,8 @@
 
 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
 
-@Category(SmallTests.class)
+// Use MediumTests because this test class needs its own JVM
+@Category(MediumTests.class)
 public class TestClientTokenUtil {
 
   @ClassRule

hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestThreads.java

@@ -21,15 +21,16 @@
 
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@Category({ MiscTests.class, SmallTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ MiscTests.class, MediumTests.class })
 public class TestThreads {
 
   @ClassRule

hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java

@@ -31,16 +31,17 @@
 import javax.servlet.http.HttpServletRequest;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
 import org.apache.hadoop.net.NetUtils;
 import org.junit.ClassRule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@Category({ MiscTests.class, SmallTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ MiscTests.class, MediumTests.class })
 public class TestGlobalFilter extends HttpServerFunctionalTest {
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAccessControlAdminApi.java

@@ -33,7 +33,7 @@
 import org.apache.hadoop.hbase.security.access.SecureTestUtil.AccessTestAction;
 import org.apache.hadoop.hbase.security.access.UserPermission;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
@@ -44,7 +44,8 @@
 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 
 @RunWith(Parameterized.class)
-@Category({ ClientTests.class, SmallTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ ClientTests.class, MediumTests.class })
 public class TestAsyncAccessControlAdminApi extends TestAsyncAdminBase {
 
   @ClassRule

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestClientSideRegionScanner.java

@@ -47,7 +47,7 @@
 import org.apache.hadoop.hbase.regionserver.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.StoreScanner;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -57,7 +57,8 @@
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-@Category({ SmallTests.class, ClientTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ MediumTests.class, ClientTests.class })
 public class TestClientSideRegionScanner {
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaRegionLocationCache.java

@@ -37,7 +37,7 @@
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.JVMClusterUtil;
 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
@@ -51,7 +51,8 @@
 
 import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
 
-@Category({ SmallTests.class, MasterTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ MediumTests.class, MasterTests.class })
 public class TestMetaRegionLocationCache {
   @ClassRule
   public static final HBaseClassTestRule CLASS_RULE =

hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java

@@ -60,7 +60,7 @@
 import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
 import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
 import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -70,7 +70,8 @@
 
 import org.apache.hbase.thirdparty.io.netty.util.ResourceLeakDetector;
 
-@Category({ IOTests.class, SmallTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ IOTests.class, MediumTests.class })
 public class TestHalfStoreFileReader {
 
   @ClassRule

hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestChecksum.java

@@ -45,7 +45,7 @@
 import org.apache.hadoop.hbase.nio.MultiByteBuff;
 import org.apache.hadoop.hbase.nio.SingleByteBuff;
 import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.ChecksumType;
 import org.junit.Before;
 import org.junit.ClassRule;
@@ -54,7 +54,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-@Category({ IOTests.class, SmallTests.class })
+// Use MediumTests because this test class needs its own JVM
+@Category({ IOTests.class, MediumTests.class })
 public class TestChecksum {
 
   @ClassRule