When tests fail, generate a playground link

martypitt · martypitt · commit 94a65e0a83b9 · 2025-06-26T10:32:54.000+01:00
diff --git a/example-projects/simple-project/test/PersonTest.kt b/example-projects/simple-project/test/PersonTest.kt
@@ -44,5 +44,19 @@ class PersonTest : OrbitalSpec({
                 ))
         }
 
+
+        xit("should generate a playground link when a test fails") {
+            """
+            find { Person(PersonId == 1) } as PossibleAdult
+        """.queryForObject(
+                stub("getPerson").returns("""{ "id" : 123, "age" : 36 }""")
+            )
+                .shouldBe(mapOf(
+                    "id" to 123,
+                    "age" to 12, // This will intentionally fail
+                    "isAdult" to true
+                ))
+        }
+
     }
 })
diff --git a/preflight-core/preflight-runtime/build.gradle.kts b/preflight-core/preflight-runtime/build.gradle.kts
@@ -18,6 +18,17 @@ dependencies {
         // Not published to maven central, and not needed for testing
         // as it relates to saml auth
         exclude(group = "org.pac4j")
+
+        // This might need adding (and shading), but later...
+        // Can we avoid by just using the OSS version in Preflight?
+        exclude(group = "org.jooq.pro")
+        exclude(group = "org.pac4j")
+    }
+    api("org.opentest4j:opentest4j:1.3.0")
+    api("com.orbitalhq:taxi-playground-core:$orbitalVersion") {
+        exclude(group = "io.confluent")
+        exclude(group = "org.jooq.pro")
+        exclude(group = "org.pac4j")
     }
     api("com.orbitalhq:taxiql-query-engine:$orbitalVersion") {
         artifact { classifier = "tests" }
diff --git a/preflight-core/preflight-runtime/src/main/kotlin/com/orbitalhq/preflight/dsl/PlaygroundScenarioFactory.kt b/preflight-core/preflight-runtime/src/main/kotlin/com/orbitalhq/preflight/dsl/PlaygroundScenarioFactory.kt
@@ -0,0 +1,132 @@
+package com.orbitalhq.preflight.dsl
+
+import arrow.core.Either
+import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
+import com.orbitalhq.SourcePackage
+import com.orbitalhq.playground.OperationStub
+import com.orbitalhq.schemas.taxi.TaxiSchema
+import io.kotest.assertions.AssertionFailedError
+import io.kotest.core.test.TestCase
+import java.io.ByteArrayOutputStream
+import java.util.zip.GZIPOutputStream
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+object PlaygroundScenarioFactory {
+    private val objectMapper = jacksonObjectMapper()
+    fun buildPlaygroundScenario(
+        capturedScenario: CapturedQuery,
+        sourcePackage: SourcePackage,
+        schema: TaxiSchema,
+        failure: AssertionFailedError,
+        testCase: TestCase,
+        playgroundHost: String = "https://playground.taxilang.org"
+    ): Pair<PlaygroundQueryMessage, String> {
+
+        val sources = sourcePackage.sources.joinToString("\n") { it.content }
+        val stubs = buildOperationStubs(capturedScenario, schema)
+        val readme = """
+            ## ❌ Failed test scenario
+            
+            Your test `${testCase.name.testName}` failed. Bummer. 👎 
+            
+            Don't worry, it happens to the best of us.
+
+            The test failed with the following error:
+            
+            ```
+            ${failure.message.orEmpty()}
+            ```
+            
+            Here's the query you were trying to run:
+            
+            ```taxiql
+            ${capturedScenario.query.trim()}
+            ```
+            
+            Stubs have been wired up, so you can re-run your query here. Either click the run button above,
+            or the in the Query panel.
+        """.trimIndent()
+        val playgroundMessage = PlaygroundQueryMessage(
+            sources,
+            capturedScenario.query.trim(),
+            emptyMap(),
+            stubs,
+            null, // TODO : expected json doesn't actually do anything yet.
+            readme = readme
+        )
+
+        return playgroundMessage to getPlaygroundUrl(playgroundMessage,playgroundHost )
+
+    }
+
+    private fun buildOperationStubs(
+        capturedScenario: CapturedQuery,
+        schema: TaxiSchema
+    ): List<OperationStub> {
+        return capturedScenario.stub.responses.mapNotNull { (operationKey, response) ->
+            when (response) {
+                is Either.Right -> {
+                    val resultTypedInstances = response.value
+                    val listOfResults = resultTypedInstances.map { it.toRawObject() }
+
+                    // find the operation this was intended for
+                    capturedScenario.stub
+                    val operation = schema.operations.firstOrNull { it.name == operationKey } ?: return@mapNotNull null
+                    val responseAsJson = if (operation.returnType.isCollection) {
+                        objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(listOfResults)
+                    } else {
+                        objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(listOfResults.firstOrNull())
+                    }
+                    OperationStub(
+                        operationKey,
+                        responseAsJson
+                    )
+                }
+
+                else -> null
+            }
+        }
+    }
+
+    @OptIn(ExperimentalEncodingApi::class)
+    fun getPlaygroundUrl(
+        queryMessage: PlaygroundQueryMessage,
+        playgroundHost: String = "https://playground.taxilang.org",
+        enableDevTools: Boolean = false
+    ): String {
+        val json = jacksonObjectMapper().writeValueAsString(queryMessage)
+
+        // Compress the JSON with GZIP
+        val gzipOutput = ByteArrayOutputStream()
+        GZIPOutputStream(gzipOutput).use {
+            it.write(json.toByteArray())
+        }
+        val compressed = gzipOutput.toByteArray()
+
+        // Base64 encode:
+        val base64Encoded = Base64.encode(compressed)
+        val devToolsPart = if (enableDevTools) {
+            "?enableDevTools=true"
+        } else ""
+
+        return "$playgroundHost/$devToolsPart#pako:$base64Encoded"
+    }
+
+}
+
+// TODO : This is duplicated here because I need to add the readme property upstream on the
+// orbital instance of this message
+data class PlaygroundQueryMessage(
+    val schema: String,
+    val query: String,
+    val parameters: Map<String, Any> = emptyMap(),
+    val stubs: List<OperationStub> = emptyList(),
+    val expectedJson: String? = null,
+    /**
+     * The nebula stack Id that's currently running for this session.
+     * Null if one doesn't exist yet.
+     */
+    val stackId: String? = null,
+    val readme: String?
+)
diff --git a/preflight-core/preflight-runtime/src/main/kotlin/com/orbitalhq/preflight/dsl/PreflightExtension.kt b/preflight-core/preflight-runtime/src/main/kotlin/com/orbitalhq/preflight/dsl/PreflightExtension.kt
@@ -5,6 +5,7 @@ import com.orbitalhq.firstRawObject
 import com.orbitalhq.firstRawValue
 import com.orbitalhq.firstTypedInstace
 import com.orbitalhq.models.TypedInstance
+import com.orbitalhq.preflight.dsl.PreflightExtension.Companion.PreflightTestCaseKey
 import com.orbitalhq.query.QueryResult
 import com.orbitalhq.rawObjects
 import com.orbitalhq.schemaServer.core.adaptors.taxi.TaxiSchemaSourcesAdaptor
@@ -25,9 +26,24 @@ import java.nio.file.Paths
 import kotlin.io.path.absolute
 import com.orbitalhq.schemaServer.core.file.packages.FileSystemPackageLoader
 import com.orbitalhq.utils.files.ReactivePollingFileSystemMonitor
+import io.kotest.assertions.AssertionFailedError
+import io.kotest.core.extensions.TestCaseExtension
+import io.kotest.core.listeners.AfterTestListener
+import io.kotest.core.test.TestCase
+import io.kotest.core.test.TestResult
+import kotlinx.coroutines.withContext
+import lang.taxi.query.TaxiQLQueryString
 import java.time.Duration
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.coroutineContext
+
+class PreflightExtension(val projectRoot: Path = Paths.get("./")) : BeforeSpecListener, AfterTestListener,
+    TestCaseExtension {
+
+    companion object {
+        val PreflightTestCaseKey = object : CoroutineContext.Key<PreflightTestCaseContext> {}
+    }
 
-class PreflightExtension(val projectRoot: Path = Paths.get("./")) : BeforeSpecListener {
     /**
      * Provides access to the compiled taxi document.
      * This is lower-level than Orbital's schema object
@@ -38,22 +54,31 @@ class PreflightExtension(val projectRoot: Path = Paths.get("./")) : BeforeSpecLi
     lateinit var schema: TaxiSchema
         private set
 
+    lateinit var sourcePackage: SourcePackage
+        private set
+
     /**
      * Provides access to the actual Taxi project (the equivalent of the
      * taxi.conf file)
      */
     lateinit var taxiProject: TaxiPackageProject
         private set
 
+    private val capturedScenarios = mutableMapOf<TestCase, CapturedQuery>()
+
+
     override suspend fun beforeSpec(spec: Spec) {
         val loader = TaxiPackageLoader.forDirectoryContainingTaxiFile(projectRoot.absolute().normalize())
         this.taxiProject = loader.load()
 
-        val sourcePackage = loadSourcePackage(this.taxiProject.packageRootPath!!)
+        sourcePackage = loadSourcePackage(this.taxiProject.packageRootPath!!)
 
         withClue("Taxi project should compile without errors") {
             val taxiSchema = try {
-                TaxiSchema.from(sourcePackage, onErrorBehaviour = TaxiSchema.Companion.TaxiSchemaErrorBehaviour.THROW_EXCEPTION)
+                TaxiSchema.from(
+                    sourcePackage,
+                    onErrorBehaviour = TaxiSchema.Companion.TaxiSchemaErrorBehaviour.THROW_EXCEPTION
+                )
             } catch (e: CompilationException) {
                 fail("Taxi project has errors: \n${e.message}")
             }
@@ -79,7 +104,7 @@ class PreflightExtension(val projectRoot: Path = Paths.get("./")) : BeforeSpecLi
         return sourcePackage
     }
 
-    fun orbital():Pair<Orbital, StubService> {
+    fun orbital(): Pair<Orbital, StubService> {
         return testVyne(this.schema)
     }
 
@@ -91,32 +116,96 @@ class PreflightExtension(val projectRoot: Path = Paths.get("./")) : BeforeSpecLi
         }
     }
 
-    suspend fun queryForScalar(taxiQl: String, vararg stubScenarios: StubScenario) = queryForScalar(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
-    suspend fun queryForScalar(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}):Any? {
+    suspend fun queryForScalar(taxiQl: String, vararg stubScenarios: StubScenario) =
+        queryForScalar(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
+
+    suspend fun queryForScalar(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}): Any? {
         return query(taxiQl, stubCustomizer)
             .firstRawValue()
     }
 
 
-    suspend fun queryForObject(taxiQl: String, vararg stubScenarios: StubScenario) = queryForObject(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
-    suspend fun queryForObject(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}):Map<String,Any?> {
+    suspend fun queryForObject(taxiQl: String, vararg stubScenarios: StubScenario) =
+        queryForObject(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
+
+    suspend fun queryForObject(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}): Map<String, Any?> {
         return query(taxiQl, stubCustomizer)
             .firstRawObject()
     }
-    suspend fun queryForCollection(taxiQl: String, vararg stubScenarios: StubScenario) = queryForCollection(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
-    suspend fun queryForCollection(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}): List<Map<String, Any?>> {
+
+    suspend fun queryForCollection(taxiQl: String, vararg stubScenarios: StubScenario) =
+        queryForCollection(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
+
+    suspend fun queryForCollection(
+        taxiQl: String,
+        stubCustomizer: (StubService) -> Unit = {}
+    ): List<Map<String, Any?>> {
         return query(taxiQl, stubCustomizer)
             .rawObjects()
     }
-    suspend fun queryForTypedInstance(taxiQl: String, vararg stubScenarios: StubScenario) = queryForTypedInstance(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
-    suspend fun queryForTypedInstance(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}):TypedInstance {
+
+    suspend fun queryForTypedInstance(taxiQl: String, vararg stubScenarios: StubScenario) =
+        queryForTypedInstance(taxiQl, stubScenariosToCustomizer(stubScenarios.toList()))
+
+    suspend fun queryForTypedInstance(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}): TypedInstance {
         return query(taxiQl, stubCustomizer)
             .firstTypedInstace()
     }
 
     suspend fun query(taxiQl: String, stubCustomizer: (StubService) -> Unit = {}): QueryResult {
-        val (orbital,stub) = orbital()
+        val (orbital, stub) = orbital()
         stubCustomizer(stub)
+        val testContext = coroutineContext[PreflightTestCaseKey]
+        if (testContext != null) {
+            capturedScenarios[testContext.testCase] = CapturedQuery(stub, taxiQl)
+        } else {
+            println("A test is executing without a context - this shouldn't happen")
+        }
         return orbital.query(taxiQl)
     }
+
+    override suspend fun intercept(testCase: TestCase, execute: suspend (TestCase) -> TestResult): TestResult {
+        val context = PreflightTestCaseContext(testCase)
+        return withContext(context) {
+            val testResult = execute(testCase)
+            val capturedScenario = capturedScenarios[testCase]
+            if (testResult.isErrorOrFailure && capturedScenario != null) {
+
+                val failure = testResult as TestResult.Failure
+                val originalError = failure.cause as AssertionFailedError
+                val (_, playgroundLink) = PlaygroundScenarioFactory.buildPlaygroundScenario(
+                    capturedScenario,
+                    sourcePackage,
+                    schema,
+                    originalError,
+                    testCase
+                )
+                val errorMessageWithPlaygroundLink = """${originalError.message}
+                        |
+                        |This error is explorable in Taxi Playground at the following link: $playgroundLink
+                    """.trimMargin()
+                val failureWithPlaygroundLink = failure.copy(
+                    cause = AssertionFailedError(
+                        message = errorMessageWithPlaygroundLink,
+                        cause = originalError.cause,
+                        expectedValue = originalError.expectedValue,
+                        actualValue = originalError.actualValue,
+                    )
+                )
+                failureWithPlaygroundLink
+            } else {
+                testResult
+            }
+
+        }
+    }
+}
+
+data class CapturedQuery(
+    val stub: StubService,
+    val query: TaxiQLQueryString
+)
+
+data class PreflightTestCaseContext(val testCase: TestCase) : CoroutineContext.Element {
+    override val key = PreflightTestCaseKey
 }
