♻️ Add OpenAIIntegrationTest Extension

gilday · gilday · commit 7136fcceb7b1 · 2024-06-26T09:10:38.000-04:00
Extension encapsulates the logic for determining if an OpenAI test should be skipped.
diff --git a/README.md b/README.md
@@ -58,6 +58,17 @@ python -m site
 
 You need to be able to run [Docker](https://www.docker.com/) for some tests. Make sure you have it installed and have the necessary permissions to run with your user.
 
+#### OpenAI Integration Tests
+
+Some tests integrate with the OpenAI platform. To run these tests, configure an OpenAI key in the Gradle
+property `codemodderOpenAIKey`. When no API key is configured, the tests will be skipped. 
+
+You can add the Gradle property to your `~/.gradle/gradle.properties` file:
+
+```properties
+codemodderOpenAIKey=your-openai-key
+```
+
 ### Run the Core Codemods
 You can download and run the latest release from this repository in order to run the core codemods as a CLI:
 
diff --git a/core-codemods/build.gradle.kts b/core-codemods/build.gradle.kts
@@ -3,6 +3,7 @@ plugins {
     id("io.codemodder.runner")
     id("io.codemodder.maven-publish")
     id("io.codemodder.core-codemods-docs")
+    id("io.codemodder.openai")
     `jvm-test-suite`
 }
 
@@ -30,12 +31,12 @@ dependencies {
     implementation(libs.dom4j)
     implementation(libs.commons.jexl)
     implementation(libs.tuples)
+    testImplementation(testFixtures(project(":plugins:codemodder-plugin-llm")))
     testImplementation(testlibs.bundles.junit.jupiter)
     testImplementation(testlibs.bundles.hamcrest)
     testImplementation(testlibs.assertj)
     testImplementation(testlibs.mockito)
     testImplementation(project(":framework:codemodder-testutils"))
-    testImplementation(project(":framework:codemodder-testutils-llm"))
     testRuntimeOnly(testlibs.junit.jupiter.engine)
     testImplementation(testlibs.jgit)
     testImplementation(testlibs.assertj)
diff --git a/core-codemods/src/test/java/io/codemodder/codemods/LogFailedLoginCodemodTest.java b/core-codemods/src/test/java/io/codemodder/codemods/LogFailedLoginCodemodTest.java
@@ -1,14 +1,14 @@
 package io.codemodder.codemods;
 
+import io.codemodder.plugins.llm.test.LLMVerifyingCodemodTestMixin;
+import io.codemodder.plugins.llm.test.OpenAIIntegrationTest;
 import io.codemodder.testutils.Metadata;
-import io.codemodder.testutils.llm.LLMVerifyingCodemodTestMixin;
-import org.junit.jupiter.api.condition.EnabledIf;
 
 @Metadata(
     codemodType = LogFailedLoginCodemod.class,
     testResourceDir = "log-failed-login",
     dependencies = {})
-@EnabledIf("io.codemodder.testutils.llm.CodemodderOpenAIKeys#isAvailable")
+@OpenAIIntegrationTest
 public final class LogFailedLoginCodemodTest implements LLMVerifyingCodemodTestMixin {
 
   @Override
diff --git a/core-codemods/src/test/java/io/codemodder/codemods/SensitiveDataLoggingCodemodTest.java b/core-codemods/src/test/java/io/codemodder/codemods/SensitiveDataLoggingCodemodTest.java
@@ -1,14 +1,14 @@
 package io.codemodder.codemods;
 
+import io.codemodder.plugins.llm.test.LLMVerifyingCodemodTestMixin;
+import io.codemodder.plugins.llm.test.OpenAIIntegrationTest;
 import io.codemodder.testutils.Metadata;
-import io.codemodder.testutils.llm.LLMVerifyingCodemodTestMixin;
-import org.junit.jupiter.api.condition.EnabledIf;
 
 @Metadata(
     codemodType = SensitiveDataLoggingCodemod.class,
     testResourceDir = "sensitive-data-logging",
     dependencies = {})
-@EnabledIf("io.codemodder.testutils.llm.CodemodderOpenAIKeys#isAvailable")
+@OpenAIIntegrationTest
 final class SensitiveDataLoggingCodemodTest implements LLMVerifyingCodemodTestMixin {
 
   @Override
diff --git a/framework/codemodder-testutils-llm/build.gradle.kts b/framework/codemodder-testutils-llm/build.gradle.kts
diff --git a/framework/codemodder-testutils-llm/src/main/java/io/codemodder/testutils/llm/CodemodderOpenAIKeys.java b/framework/codemodder-testutils-llm/src/main/java/io/codemodder/testutils/llm/CodemodderOpenAIKeys.java
diff --git a/gradle/build-plugins/src/main/kotlin/io.codemodder.java-test-fixtures.gradle.kts b/gradle/build-plugins/src/main/kotlin/io.codemodder.java-test-fixtures.gradle.kts
@@ -0,0 +1,3 @@
+plugins {
+    `java-test-fixtures`
+}
diff --git a/gradle/build-plugins/src/main/kotlin/io.codemodder.openai.gradle.kts b/gradle/build-plugins/src/main/kotlin/io.codemodder.openai.gradle.kts
@@ -0,0 +1,23 @@
+plugins {
+    id("io.codemodder.java")
+    `jvm-test-suite`
+}
+
+testing {
+    suites {
+        @Suppress("UnstableApiUsage")
+        val test by getting(JvmTestSuite::class) {
+            useJUnitJupiter()
+            targets {
+                all {
+                    testTask {
+                        val openAIKey = project.findProperty("codemodderOpenAIKey") as String?
+                        openAIKey?.let {
+                            environment("CODEMODDER_OPENAI_API_KEY", it)
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/plugins/codemodder-plugin-llm/build.gradle.kts b/plugins/codemodder-plugin-llm/build.gradle.kts
@@ -1,5 +1,6 @@
 plugins {
     id("io.codemodder.java-library")
+    id("io.codemodder.java-test-fixtures")
     id("io.codemodder.maven-publish")
 }
 
@@ -12,6 +13,10 @@ dependencies {
     }
     implementation("com.google.inject:guice:5.1.0")
 
+    testFixturesApi(testlibs.junit.jupiter.api)
+    testFixturesApi(project(":framework:codemodder-testutils"))
+    testFixturesImplementation(testlibs.bundles.hamcrest)
+
     testImplementation(testlibs.bundles.junit.jupiter)
     testImplementation(testlibs.bundles.hamcrest)
     testImplementation(testlibs.assertj)
diff --git a/plugins/codemodder-plugin-llm/src/testFixtures/java/io/codemodder/plugins/llm/test/LLMVerifyingCodemodTestMixin.java b/plugins/codemodder-plugin-llm/src/testFixtures/java/io/codemodder/plugins/llm/test/LLMVerifyingCodemodTestMixin.java
@@ -1,4 +1,4 @@
-package io.codemodder.testutils.llm;
+package io.codemodder.plugins.llm.test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.is;
diff --git a/plugins/codemodder-plugin-llm/src/testFixtures/java/io/codemodder/plugins/llm/test/OpenAIIntegrationTest.java b/plugins/codemodder-plugin-llm/src/testFixtures/java/io/codemodder/plugins/llm/test/OpenAIIntegrationTest.java
@@ -0,0 +1,16 @@
+package io.codemodder.plugins.llm.test;
+
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+/** Marks a test as an OpenAI integration test. */
+@Target({ElementType.TYPE, ElementType.METHOD})
+@Retention(RUNTIME)
+@Documented
+@ExtendWith(OpenAIIntegrationTestExtension.class)
+public @interface OpenAIIntegrationTest {}
diff --git a/plugins/codemodder-plugin-llm/src/testFixtures/java/io/codemodder/plugins/llm/test/OpenAIIntegrationTestExtension.java b/plugins/codemodder-plugin-llm/src/testFixtures/java/io/codemodder/plugins/llm/test/OpenAIIntegrationTestExtension.java
@@ -0,0 +1,38 @@
+package io.codemodder.plugins.llm.test;
+
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+/** Extension for writing OpenAI integration tests. */
+public final class OpenAIIntegrationTestExtension implements ExecutionCondition {
+  /**
+   * The OpenAI integration tests only run when the test suite has been configured with an OpenAI
+   * API key. This is a convenience to developers who may want to avoid running these relatively
+   * slow and expensive tests. However, we always want to run these tests in CI; therefore, this
+   * check includes backstop logic that ensures we never accidentally skip these tests in CI.
+   *
+   * <p>This extension reads the OpenAI key from the standard environment variable {@code
+   * CODEMODDER_OPENAI_API_KEY} vs the recommended method of reading configuration from the JUnit
+   * context. This is necessary, because the code under test reads the OpenAI key from an
+   * environment variable directly vs requiring the key to be injected, and this makes test
+   * configuration less flexible than it could be. Improving this is out-of-scope for the initial
+   * development of this extension.
+   *
+   * @return a result indicating whether the test should be enabled or disabled
+   */
+  @Override
+  public ConditionEvaluationResult evaluateExecutionCondition(final ExtensionContext ignored) {
+    String key = System.getenv("CODEMODDER_OPENAI_API_KEY");
+    final var available = key != null && !key.isBlank();
+    if (System.getenv("CI") != null && !available) {
+      throw new IllegalStateException(
+          "The CODEMODDER_OPENAI_API_KEY environment variable must be set in the CI/CD environment.");
+    }
+    return available
+        ? ConditionEvaluationResult.enabled(
+            "Using OpenAI API key from env var CODEMODDER_OPENAI_API_KEY")
+        : ConditionEvaluationResult.disabled(
+            "OpenAI API key not found in env var CODEMODDER_OPENAI_API_KEY");
+  }
+}
diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -41,7 +41,6 @@ include("core-codemods")
 include("community-codemods")
 include("framework:codemodder-base")
 include("framework:codemodder-testutils")
-include("framework:codemodder-testutils-llm")
 include("plugins:codemodder-plugin-aws")
 include("plugins:codemodder-plugin-codeql")
 include("plugins:codemodder-plugin-llm")

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+plugins {`
	`2`	+ `java-test-fixtures`
	`3`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-package io.codemodder.testutils.llm;`
	`1`	`+package io.codemodder.plugins.llm.test;`
`2`	`2`
`3`	`3`	`import static org.hamcrest.MatcherAssert.assertThat;`
`4`	`4`	`import static org.hamcrest.Matchers.is;`