Avoid recursion when transforming types used in Holes

Previously, the AvoidMap used in getPicklableHoleType would try to
replace recursive type parameters obtained via F-bounds) with LazyRefs,
which would expand to include another LazyRef, etc.
Now we explicitly guard against recursing over already visited TypeRefs,
instead replacing them with RecTypes (which are safe to pickle).

Author: jchyb

compiler/src/dotty/tools/dotc/transform/PickleQuotes.scala

@@ -199,10 +199,47 @@ class PickleQuotes extends MacroTransform {
     ctx.typeAssigner.assignType(untpd.TypeDef(local.name, hole), local).withSpan(typeArg.span)
   }
 
-  /** Avoid all non-static types except those defined in the quote. */
+  /** Avoid all non-static types except those defined in the quote.
+   *  Also avoid recursive LazyRef references that can be generated by TypeBounds.
+  */
   private def getPicklableHoleType(tpe: Type, isStagedClasses: Symbol => Boolean)(using Context) =
     new TypeOps.AvoidMap {
+      val processedTypeRefs = util.HashSet[Type]()
       def toAvoid(tp: NamedType) = !isStagedClasses(tp.typeSymbol) && !isStaticPrefix(tp)
+      override def apply(tp: Type) = tp match
+        case typeRef: TypeRef if toAvoid(typeRef) =>
+          if processedTypeRefs.contains(typeRef) then typeRef
+          else
+            typeRef.info match
+              case TypeBounds(lo, hi) =>
+                val typeRefRemapper = (recType: RecType) => new TypeMap {
+                  def apply(t: Type) = t match
+                    case lr: LazyRef if lr.ref == typeRef => recType.recThis
+                    case tr: TypeRef if tr == typeRef => recType.recThis
+                    case lr: LazyRef => lr
+                    case _ => mapOver(t)
+                }
+                processedTypeRefs.add(typeRef)
+                val newLo =
+                  RecType.closeOver(typeRefRemapper(_).apply(atVariance(-variance)(apply(lo))))
+                val newHi =
+                  apply(hi) match
+                    case Range(l, h) =>
+                      Range(
+                        RecType.closeOver(typeRefRemapper(_).apply(l)),
+                        RecType.closeOver(typeRefRemapper(_).apply(h))
+                      )
+                    case other =>
+                      RecType.closeOver(typeRefRemapper(_).apply(other))
+                processedTypeRefs.remove(typeRef)
+                range(newLo, newHi)
+              case _ =>
+                super.apply(tp)
+        case lazyRef: LazyRef =>
+          if processedTypeRefs.contains(lazyRef.ref) then lazyRef
+          else super.apply(lazyRef)
+        case _ =>
+          super.apply(tp)
     }.apply(tpe)
 }
 

tests/pos/i22996.scala

@@ -0,0 +1,14 @@
+import scala.quoted.*
+
+trait Appendable[T]
+trait Format[T] { def empty: T }
+
+object FormatMacros {
+
+  def test[T <: Appendable[T]: Type, F <: Format[T]: Type](f: Expr[F])(using Quotes): Expr[T] =
+    '{ $f.empty }
+
+  def test2[T <: Appendable[T2]: Type, T2 <: Appendable[T], F <: Format[T]: Type](f: Expr[F])(using Quotes): Expr[T] =
+    '{ $f.empty }
+
+}