Move most of self type setup to transformSym

We need to do it for compiled as well as imported types. We only
keep adding variables to selftypes of local classes in postProcess.

This allows us to drop one occurrence of LooseRootChecking

Author: odersky

compiler/src/dotty/tools/dotc/cc/CaptureOps.scala

@@ -95,7 +95,7 @@ trait FollowAliases extends TypeMap:
         if t2 ne t1 then return t2
       mapOver(t)
 
-class mapRoots(from0: CaptureRoot, to: CaptureRoot)(using Context) extends BiTypeMap, FollowAliases:
+class mapRoots(from0: CaptureRoot, to: CaptureRoot)(using Context) extends DeepTypeMap, BiTypeMap, FollowAliases:
   val from = from0.followAlias
 
   //override val toString = i"mapRoots($from, $to)"

compiler/src/dotty/tools/dotc/cc/CheckCaptures.scala

@@ -1110,12 +1110,7 @@ class CheckCaptures extends Recheck, SymTransformer:
         }
         assert(roots.nonEmpty)
         for case root: ClassSymbol <- roots do
-          inContext(ctx.fresh.setOwner(root).withProperty(LooseRootChecking, Some(()))):
-            // Without LooseRootChecking, we get problems with F-bounded parent types.
-            // These can make `cap` "pop out" in ways that are hard to prevent. I believe
-            // to prevent it we'd have to map `cap` in a whole class graph with all parent
-            // classes, which would be very expensive. So for now we approximate by assuming
-            // different roots are compatible for self type conformance checking.
+          inContext(ctx.fresh.setOwner(root)):
             checkSelfAgainstParents(root, root.baseClasses)
             val selfType = root.asClass.classInfo.selfType
             interpolator(startingVariance = -1).traverse(selfType)
@@ -1238,16 +1233,15 @@ class CheckCaptures extends Recheck, SymTransformer:
       setup.postCheck()
 
       if !ctx.reporter.errorsReported then
-        //inContext(ctx.withProperty(LooseRootChecking, Some(()))):
-          // We dont report errors here if previous errors were reported, because other
-          // errors often result in bad applied types, but flagging these bad types gives
-          // often worse error messages than the original errors.
-          val checkApplied = new TreeTraverser:
-            def traverse(t: Tree)(using Context) = t match
-              case tree: InferredTypeTree =>
-              case tree: New =>
-              case tree: TypeTree => checkAppliedTypesIn(tree.withKnownType)
-              case _ => traverseChildren(t)
-          checkApplied.traverse(unit)
+        // We dont report errors here if previous errors were reported, because other
+        // errors often result in bad applied types, but flagging these bad types gives
+        // often worse error messages than the original errors.
+        val checkApplied = new TreeTraverser:
+          def traverse(t: Tree)(using Context) = t match
+            case tree: InferredTypeTree =>
+            case tree: New =>
+            case tree: TypeTree => checkAppliedTypesIn(tree.withKnownType)
+            case _ => traverseChildren(t)
+        checkApplied.traverse(unit)
   end CaptureChecker
 end CheckCaptures

compiler/src/dotty/tools/dotc/cc/Setup.scala

@@ -113,24 +113,39 @@ class Setup extends PreRecheck, SymTransformer, SetupAPI:
   def transformSym(symd: SymDenotation)(using Context): SymDenotation =
     if !pastRecheck && Feature.ccEnabledSomewhere then
       val sym = symd.symbol
+      def rootTarget =
+        if sym.isAliasType && sym.isStatic then NoSymbol else sym
       def mappedInfo =
         if toBeUpdated.contains(sym) then symd.info
-        else transformExplicitType(symd.info, rootTarget = sym)
+        else transformExplicitType(symd.info, rootTarget)
       // TODO if sym is class && level owner: add a capture root
       if Synthetics.needsTransform(symd) then
         Synthetics.transform(symd, mappedInfo)
       else if isPreCC(sym) then
         symd.copySymDenotation(info = fluidify(sym.info))
       else if symd.owner.isTerm || symd.is(CaptureChecked) || symd.owner.is(CaptureChecked) then
         val newFlags = newFlagsFor(symd)
-        val newInfo = mappedInfo
+        var newInfo = mappedInfo
+        if sym.is(ModuleVal) && !sym.isStatic then
+          val selfType1 = transformSelfType(sym.moduleClass)
+          if selfType1.exists then
+            newInfo = newInfo.capturing(selfType1.captureSet)
+            sym.termRef.invalidateCaches()
+        if sym.isClass then
+          sym.thisType.asInstanceOf[ThisType].invalidateCaches()
         if newFlags != symd.flags || (newInfo ne sym.info)
         then symd.copySymDenotation(initFlags = newFlags, info = newInfo)
         else symd
       else symd
     else symd
   end transformSym
 
+  def transformSelfType(sym: Symbol)(using Context) = (sym.info: @unchecked) match
+    case ClassInfo(_, cls, _, _, selfInfo: Type) =>
+      inContext(ctx.withOwner(cls)):
+        transformExplicitType(selfInfo, rootTarget = ctx.owner)
+    case _ => NoType
+
   /** If `tp` is an unboxed capturing type or a function returning an unboxed capturing type,
    *  convert it to be boxed.
    */
@@ -490,15 +505,12 @@ class Setup extends PreRecheck, SymTransformer, SetupAPI:
       postProcess(tree)
     end traverse
 
-    extension (sym: Symbol)(using Context) def unlessStatic =
-      val owner = sym.levelOwner
-      if sym.isStaticOwner then NoSymbol else owner
-
     def postProcess(tree: Tree)(using Context): Unit = tree match
       case tree: TypeTree =>
         val lowner =
         transformTT(tree, boxed = false, exact = false,
-            rootTarget = ctx.owner.unlessStatic // roots of other types in static locations are not mapped
+            rootTarget = if ctx.owner.isStaticOwner then NoSymbol else ctx.owner
+            // roots of other types in static locations are not mapped
           )
       case tree: ValOrDefDef =>
         val sym = tree.symbol
@@ -594,22 +606,13 @@ class Setup extends PreRecheck, SymTransformer, SetupAPI:
         tree.symbol match
           case cls: ClassSymbol =>
             val cinfo @ ClassInfo(prefix, _, ps, decls, selfInfo) = cls.classInfo
-            val newSelfType =
-              if (selfInfo eq NoType) || cls.is(ModuleClass) && !cls.isStatic then
-                // add capture set to self type of nested classes if no self type is given explicitly.
-                // It's unclear what the right level owner should be. A self type should
-                // be able to mention class parameters, which are owned by the class; that's
-                // why the class was picked as level owner. But self types should not be able
-                // to mention other fields.
-                CapturingType(cinfo.selfType, CaptureSet.Var(cls))
-              else selfInfo match
-                case selfInfo: Type =>
-                  val selfInfo1 = inContext(ctx.withOwner(cls)):
-                    transformExplicitType(selfInfo, rootTarget = ctx.owner)
-                  if selfInfo1 eq selfInfo then NoType else selfInfo1
-                case _ =>
-                  NoType
-            if newSelfType.exists then
+            if (selfInfo eq NoType) || cls.is(ModuleClass) && !cls.isStatic then
+              // add capture set to self type of nested classes if no self type is given explicitly.
+              // It's unclear what the right level owner should be. A self type should
+              // be able to mention class parameters, which are owned by the class; that's
+              // why the class was picked as level owner. But self types should not be able
+              // to mention other fields.
+              val newSelfType = CapturingType(cinfo.selfType, CaptureSet.Var(cls))
               ccSetup.println(i"mapped self type for $cls: $newSelfType, was $selfInfo")
               val ps1 = ps.mapConserve(transformExplicitType(_, rootTarget = ctx.owner))
               val newInfo = ClassInfo(prefix, cls, ps1, decls, newSelfType)
@@ -621,11 +624,6 @@ class Setup extends PreRecheck, SymTransformer, SetupAPI:
                 updateInfo(modul, CapturingType(modul.info, newSelfType.captureSet))
                 modul.termRef.invalidateCaches()
           case _ =>
-            val info = tree.symbol.info
-            val newInfo = transformExplicitType(info, rootTarget = ctx.owner.unlessStatic)
-            updateInfo(tree.symbol, newInfo)
-            if newInfo ne info then
-              ccSetup.println(i"update info of ${tree.symbol} from $info to $newInfo")
       case _ =>
     end postProcess
   end setupTraverser

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -5120,10 +5120,8 @@ object Types {
           else if (clsd.is(Module)) givenSelf
           else if (ctx.erasedTypes) appliedRef
           else givenSelf.dealiasKeepAnnots match
-            case givenSelf1 @ CapturingType(tp, _) =>
-              givenSelf1.derivedAnnotatedType(tp & appliedRef, givenSelf1.annot)
-            case givenSelf1 @ RetainingType(tp, _) =>
-              givenSelf1.derivedAnnotatedType(tp & appliedRef, givenSelf1.annot)
+            case givenSelf1 @ AnnotatedType(tp, ann) if ann.symbol == defn.RetainsAnnot =>
+              givenSelf1.derivedAnnotatedType(tp & appliedRef, ann)
             case _ =>
               AndType(givenSelf, appliedRef)
         }
@@ -5969,7 +5967,7 @@ object Types {
       val prefix1 = this(tp.prefix)
       val parents1 = tp.declaredParents mapConserve this
       val selfInfo1: TypeOrSymbol = tp.selfInfo match {
-        case selfInfo: Type => this(selfInfo)
+        case selfInfo: Type => inContext(ctx.withOwner(tp.cls))(this(selfInfo))
         case selfInfo => selfInfo
       }
       tp.derivedClassInfo(prefix1, parents1, tp.decls, selfInfo1)