Speedup CI tests (#179)

speed up tests

Author: lehvolk

.github/workflows/build-and-test.yml

@@ -19,7 +19,7 @@ jobs:
     #       which requires 'pull_request' trigger for the bot to be able to add a comment to PR.
     # if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository
 
-    name: Build on JDK ${{ matrix.jdk }}
+    name: Run tests on JDK ${{ matrix.jdk }}
 
     runs-on: ubuntu-latest
     strategy:
@@ -57,7 +57,6 @@ jobs:
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
-        if: matrix.jdk == env.main_jdk
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
@@ -74,3 +73,36 @@ jobs:
           name: gradle-reports
           path: '**/build/reports/'
           retention-days: 1
+  lifecycleTests:
+    name: Run lifecycle tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: 'zulu'
+
+      - name: Set up Gradle
+        uses: gradle/gradle-build-action@v2
+        with:
+          # Only write to the cache for builds on the specific branches. (Default is 'main' only.)
+          # Builds on other branches will only read existing entries from the cache.
+          cache-read-only: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/develop' }}
+
+      - name: Build and run lifecycle tests
+        run: ./gradlew lifecycleTest --stacktrace --scan
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          files: "**/build/test-results/**/*.xml"
+          check_name: "Lifecycle test results"

build.gradle.kts

@@ -89,17 +89,21 @@ allprojects {
             }
         }
 
-        withType<Test> {
-            useJUnitPlatform()
-            testLogging {
-                events("passed", "skipped", "failed")
+        test {
+            useJUnitPlatform {
+                excludeTags(Tests.lifecycleTag)
             }
-            finalizedBy(jacocoTestReport) // report is always generated after tests run
-            jvmArgs = listOf("-Xmx2g", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=heapdump.hprof")
+            setup(jacocoTestReport)
+        }
+
+        val lifecycleTest by creating(Test::class) {
+            useJUnitPlatform {
+                includeTags(Tests.lifecycleTag)
+            }
+            setup(jacocoTestReport)
         }
 
         jacocoTestReport {
-            dependsOn(test) // tests are required to run before generating the report
             classDirectories.setFrom(files(classDirectories.files.map {
                 fileTree(it) {
                     excludes.add("org/jacodb/impl/storage/jooq/**")

buildSrc/src/main/kotlin/Tests.kt

@@ -0,0 +1,15 @@
+import org.gradle.api.tasks.TaskProvider
+import org.gradle.api.tasks.testing.Test
+
+object Tests {
+    val lifecycleTag = "lifecycle"
+
+}
+
+fun Test.setup(jacocoTestReport: TaskProvider<*>) {
+    testLogging {
+        events("passed", "skipped", "failed")
+    }
+    finalizedBy(jacocoTestReport) // report is always generated after tests run
+    jvmArgs = listOf("-Xmx2g", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=heapdump.hprof")
+}

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/AliasAnalysisTest.kt

@@ -31,10 +31,8 @@ import org.jacodb.api.cfg.JcExpr
 import org.jacodb.api.cfg.JcInst
 import org.jacodb.api.ext.cfg.callExpr
 import org.jacodb.api.ext.findClass
-import org.jacodb.impl.features.InMemoryHierarchy
-import org.jacodb.impl.features.Usages
 import org.jacodb.testing.BaseTest
-import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithGlobalDB
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.params.ParameterizedTest
@@ -45,7 +43,7 @@ import kotlin.streams.asStream
 
 @Disabled("Needs modifications after introduction of summaries")
 class AliasAnalysisTest : BaseTest() {
-    companion object : WithDB(Usages, InMemoryHierarchy) {
+    companion object : WithGlobalDB() {
 
         @JvmStatic
         fun provideForPointerBenchBasic(): Stream<Arguments> = listOf(

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/BaseAnalysisTest.kt

@@ -23,12 +23,10 @@ import org.jacodb.api.JcClassOrInterface
 import org.jacodb.api.JcMethod
 import org.jacodb.api.ext.findClass
 import org.jacodb.api.ext.methods
-import org.jacodb.impl.features.InMemoryHierarchy
-import org.jacodb.impl.features.Usages
 import org.jacodb.impl.features.classpaths.UnknownClasses
 import org.jacodb.impl.features.hierarchyExt
 import org.jacodb.testing.BaseTest
-import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithGlobalDB
 import org.jacodb.testing.allClasspath
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertTrue
@@ -37,7 +35,7 @@ import java.util.stream.Stream
 import kotlin.streams.asStream
 
 abstract class BaseAnalysisTest : BaseTest() {
-    companion object : WithDB(UnknownClasses, Usages, InMemoryHierarchy) {
+    companion object : WithGlobalDB(UnknownClasses) {
         @JvmStatic
         fun provideClassesForJuliet(cweNum: Int, cweSpecificBans: List<String> = emptyList()): Stream<Arguments> = runBlocking {
             val cp = db.classpath(allClasspath)

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/JodaDateTimeAnalysisTest.kt

@@ -27,15 +27,13 @@ import org.jacodb.analysis.library.newNpeRunnerFactory
 import org.jacodb.analysis.runAnalysis
 import org.jacodb.analysis.sarif.SarifReport
 import org.jacodb.api.ext.findClass
-import org.jacodb.impl.features.InMemoryHierarchy
-import org.jacodb.impl.features.Usages
 import org.jacodb.testing.BaseTest
-import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithGlobalDB
 import org.joda.time.DateTime
 import org.junit.jupiter.api.Test
 
 class JodaDateTimeAnalysisTest : BaseTest() {
-    companion object : WithDB(Usages, InMemoryHierarchy)
+    companion object : WithGlobalDB()
 
     private fun <UnitType> testOne(unitResolver: UnitResolver<UnitType>, ifdsUnitRunnerFactory: IfdsUnitRunnerFactory) {
         val clazz = cp.findClass<DateTime>()

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/NullabilityAssumptionAnalysisTest.kt

@@ -23,16 +23,15 @@ import org.jacodb.api.cfg.JcAssignInst
 import org.jacodb.api.cfg.JcInstanceCallExpr
 import org.jacodb.api.cfg.JcLocal
 import org.jacodb.api.ext.findClass
-import org.jacodb.impl.features.InMemoryHierarchy
 import org.jacodb.testing.BaseTest
-import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithGlobalDB
 import org.jacodb.testing.cfg.NullAssumptionAnalysisExample
 import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.Test
 
 class NullabilityAssumptionAnalysisTest : BaseTest() {
 
-    companion object : WithDB(InMemoryHierarchy)
+    companion object : WithGlobalDB()
 
     @Test
     fun `null-assumption analysis should work`() {

jacodb-core/src/test/java/org/jacodb/testing/JavaApi.java

@@ -16,10 +16,14 @@
 
 package org.jacodb.testing;
 
+import org.jacodb.api.JcDatabase;
 import org.jacodb.api.cfg.JcArgument;
 import org.jacodb.api.cfg.JcExpr;
 import org.jacodb.api.cfg.TypedExprResolver;
+import org.jacodb.impl.JacoDB;
 import org.jacodb.impl.JcCacheSettings;
+import org.jacodb.impl.JcSettings;
+import org.jacodb.impl.features.Usages;
 import org.jetbrains.annotations.NotNull;
 
 import java.time.Duration;
@@ -41,4 +45,12 @@ public static void cacheSettings() {
         new JcCacheSettings().types(10, Duration.of(1, ChronoUnit.MINUTES));
     }
 
+    public static void getDatabase() {
+        try {
+            JcDatabase instance = JacoDB.async(new JcSettings().installFeatures(Usages.INSTANCE)).get();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }

jacodb-core/src/test/java/org/jacodb/testing/JavaApiTest.java

@@ -16,13 +16,12 @@
 
 package org.jacodb.testing;
 
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
 import com.google.common.collect.Lists;
 import org.jacodb.api.JcClassOrInterface;
 import org.jacodb.api.JcClasspath;
 import org.jacodb.api.JcDatabase;
-import org.jacodb.impl.JacoDB;
-import org.jacodb.impl.JcSettings;
-import org.jacodb.impl.features.Usages;
 import org.junit.jupiter.api.Test;
 
 import java.io.IOException;
@@ -33,39 +32,45 @@
 
 public class JavaApiTest {
 
+    private final Supplier<JcDatabase> db = Suppliers.memoize(() -> {
+        try {
+            return BaseTestKt.getGlobalDb();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    });
+
     @Test
-    public void createJcdb() throws ExecutionException, InterruptedException, IOException {
+    public void createJcdb() {
         System.out.println("Creating database");
-        try (JcDatabase instance = JacoDB.async(new JcSettings().installFeatures(Usages.INSTANCE)).get()) {
-            System.out.println("Database is ready: " + instance);
-        }
+        JcDatabase database = db.get();
+        assertNotNull(database);
+        System.out.println("Database is ready: " + database);
     }
 
     @Test
     public void createClasspath() throws ExecutionException, InterruptedException, IOException {
         System.out.println("Creating database");
-        try (JcDatabase instance = JacoDB.async(new JcSettings().installFeatures(Usages.INSTANCE)).get()) {
-            try (JcClasspath classpath = instance.asyncClasspath(Lists.newArrayList()).get()) {
-                JcClassOrInterface clazz = classpath.findClassOrNull("java.lang.String");
-                assertNotNull(clazz);
-                assertNotNull(classpath.asyncRefreshed(false).get());
-            }
-            System.out.println("Database is ready: " + instance);
+        JcDatabase instance = db.get();
+        try (JcClasspath classpath = instance.asyncClasspath(Lists.newArrayList()).get()) {
+            JcClassOrInterface clazz = classpath.findClassOrNull("java.lang.String");
+            assertNotNull(clazz);
+            assertNotNull(classpath.asyncRefreshed(false).get());
         }
+        System.out.println("Database is ready: " + instance);
     }
 
     @Test
     public void jcdbOperations() throws ExecutionException, InterruptedException, IOException {
         System.out.println("Creating database");
-        try (JcDatabase instance = JacoDB.async(new JcSettings().installFeatures(Usages.INSTANCE)).get()) {
-            instance.asyncLoad(getAllClasspath()).get();
-            System.out.println("asyncLoad finished");
-            instance.asyncRefresh().get();
-            System.out.println("asyncRefresh finished");
-            instance.asyncRebuildFeatures().get();
-            System.out.println("asyncRebuildFeatures finished");
-            instance.asyncAwaitBackgroundJobs().get();
-            System.out.println("asyncAwaitBackgroundJobs finished");
-        }
+        JcDatabase instance = db.get();
+        instance.asyncLoad(getAllClasspath()).get();
+        System.out.println("asyncLoad finished");
+        instance.asyncRefresh().get();
+        System.out.println("asyncRefresh finished");
+        instance.asyncRebuildFeatures().get();
+        System.out.println("asyncRebuildFeatures finished");
+        instance.asyncAwaitBackgroundJobs().get();
+        System.out.println("asyncAwaitBackgroundJobs finished");
     }
 }

jacodb-core/src/test/kotlin/org/jacodb/testing/AnnotationsTest.kt

@@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test
 
 class AnnotationsTest : BaseTest() {
 
-    companion object : WithDB()
+    companion object : WithGlobalDB()
 
     @Test
     fun `field annotations`() = runBlocking {