Prevent recursive detection for the complicated type structures

compose-stability-analyzer-idea/CHANGELOG.md

@@ -6,11 +6,14 @@ All notable changes to the IntelliJ IDEA plugin will be documented in this file.
 
 ### Fixed
 - Fixed stability analysis for Compose shape types (RoundedCornerShape, CircleShape, etc.) to correctly show as STABLE instead of RUNTIME
+- Fixed StackOverflowError when analyzing recursive or self-referential types (Issue #11)
 - Improved consistency between IDEA plugin and compiler plugin stability inference
 - Added Compose Foundation shapes to known stable types list
+- Added cycle detection to prevent infinite recursion in type stability analysis
 
 ### Improved
 - Enhanced accuracy of stability analysis to match compiler plugin behavior
+- Better handling of complex function type aliases and deeply nested generic types
 
 ---
 

compose-stability-analyzer-idea/src/main/kotlin/com/skydoves/compose/stability/idea/k2/KtStabilityInferencer.kt

@@ -58,15 +58,45 @@ internal class KtStabilityInferencer {
   private val settings: StabilitySettingsState
     get() = StabilitySettingsState.getInstance()
 
+  // Cycle detection for recursive types
+  private val analyzingTypes = ThreadLocal.withInitial { mutableSetOf<String>() }
+
   /**
    * Analyzes a Kotlin type to determine its stability.
    * Main entry point for K2-based stability analysis.
    */
   context(KaSession)
   internal fun ktStabilityOf(type: KaType): KtStability {
-    // Get the original type string BEFORE stripping nullability (preserves annotations)
-    val originalTypeString = type.render(position = org.jetbrains.kotlin.types.Variance.INVARIANT)
+    val originalTypeString = try {
+      type.render(position = org.jetbrains.kotlin.types.Variance.INVARIANT)
+    } catch (e: StackOverflowError) {
+      return KtStability.Runtime(
+        className = "Unknown",
+        reason = "Unable to render type due to complexity",
+      )
+    }
+
+    val currentlyAnalyzing = analyzingTypes.get()
+    if (originalTypeString in currentlyAnalyzing) {
+      return KtStability.Runtime(
+        className = originalTypeString,
+        reason = StabilityConstants.Messages.CIRCULAR_REFERENCE,
+      )
+    }
+
+    currentlyAnalyzing.add(originalTypeString)
+    try {
+      return ktStabilityOfInternal(type, originalTypeString)
+    } finally {
+      currentlyAnalyzing.remove(originalTypeString)
+    }
+  }
 
+  /**
+   * Internal implementation separated for proper cleanup.
+   */
+  context(KaSession)
+  private fun ktStabilityOfInternal(type: KaType, originalTypeString: String): KtStability {
     // 1. Nullable types - MUST be checked first to strip nullability
     // Use KaTypeNullability enum for compatibility with Android Studio AI-243
     val nonNullableType = if (type.isMarkedNullable) {

stability-compiler/src/main/kotlin/com/skydoves/compose/stability/compiler/lower/StabilityAnalyzerTransformer.kt

@@ -55,6 +55,9 @@ public class StabilityAnalyzerTransformer(
   private val irBuilder = RecompositionIrBuilder(context)
   private var irBuilderInitialized = false
 
+  // Cycle detection for recursive types
+  private val analyzingTypes = ThreadLocal.withInitial { mutableSetOf<String>() }
+
   override fun visitFunctionNew(declaration: IrFunction): IrStatement {
     val functionName = declaration.name.asString()
     val fqName = declaration.kotlinFqName.asString()
@@ -248,8 +251,34 @@ public class StabilityAnalyzerTransformer(
     }
 
     val classSymbol = type.classOrNull
-    val fqName = type.classFqName?.asString()
+    val fqName = try {
+      type.classFqName?.asString()
+    } catch (e: StackOverflowError) {
+      return ParameterStability.RUNTIME
+    }
+
+    val typeId = fqName ?: classSymbol?.owner?.name?.asString() ?: type.render()
+    val currentlyAnalyzing = analyzingTypes.get()
+    if (typeId in currentlyAnalyzing) {
+      return ParameterStability.RUNTIME
+    }
 
+    currentlyAnalyzing.add(typeId)
+    try {
+      return analyzeTypeStabilityInternal(type, classSymbol, fqName)
+    } finally {
+      currentlyAnalyzing.remove(typeId)
+    }
+  }
+
+  /**
+   * Internal implementation separated for proper cleanup.
+   */
+  private fun analyzeTypeStabilityInternal(
+    type: IrType,
+    classSymbol: org.jetbrains.kotlin.ir.symbols.IrClassSymbol?,
+    fqName: String?,
+  ): ParameterStability {
     // 2b. Type parameters (T, E, K, V in generics) - RUNTIME
     // If we can't resolve to a class, it's likely a type parameter
     if (classSymbol == null) {