Replace sun.misc.Unsafe used in LazyVals with VarHandles

.github/workflows/ci.yaml

@@ -200,11 +200,11 @@ jobs:
          )"
 
     steps:
-      ######################################################################################
-      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA IS DISTRIBUTED USING JAVA 8. ##
-      ######################################################################################
-      - name: Set JDK 8 as default
-        run: echo "/usr/lib/jvm/java-8-openjdk-amd64/bin" >> $GITHUB_PATH
+      ############################################################################################################
+      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA 3.8.0+ IS DISTRIBUTED USING JAVA 17. ##
+      ############################################################################################################
+      - name: Set JDK 17 as default # some projects depend on javalib elements removed in newer versions
+        run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
       - name: Reset existing repo
         run: |
           git config --global --add safe.directory $GITHUB_WORKSPACE
@@ -257,11 +257,11 @@ jobs:
          )"
 
     steps:
-      ######################################################################################
-      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA IS DISTRIBUTED USING JAVA 8. ##
-      ######################################################################################
-      - name: Set JDK 8 as default
-        run: echo "/usr/lib/jvm/java-8-openjdk-amd64/bin" >> $GITHUB_PATH
+      ############################################################################################################
+      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA 3.8.0+ IS DISTRIBUTED USING JAVA 17. ##
+      ############################################################################################################
+      - name: Set JDK 17 as default # some projects depend on javalib elements removed in newer versions
+        run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
       - name: Reset existing repo
         run: |
           git config --global --add safe.directory $GITHUB_WORKSPACE
@@ -314,11 +314,11 @@ jobs:
          )"
 
     steps:
-      ######################################################################################
-      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA IS DISTRIBUTED USING JAVA 8. ##
-      ######################################################################################
-      - name: Set JDK 8 as default
-        run: echo "/usr/lib/jvm/java-8-openjdk-amd64/bin" >> $GITHUB_PATH
+      ############################################################################################################
+      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA 3.8.0+ IS DISTRIBUTED USING JAVA 17. ##
+      ############################################################################################################
+      - name: Set JDK 17 as default
+        run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
       - name: Reset existing repo
         run: |
           git config --global --add safe.directory $GITHUB_WORKSPACE
@@ -415,11 +415,11 @@ jobs:
       SONATYPE_USER: ${{ secrets.SONATYPE_USER_ORGSCALALANG }}
 
     steps:
-      ######################################################################################
-      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA IS DISTRIBUTED USING JAVA 8. ##
-      ######################################################################################
-      - name: Set JDK 8 as default
-        run: echo "/usr/lib/jvm/java-8-openjdk-amd64/bin" >> $GITHUB_PATH
+      ##############################################################################################
+      ## WARNING: DO NOT CHANGE THE JAVA VERSION HERE. SCALA 3.8.0+ IS DISTRIBUTED USING JAVA 17. ##
+      ##############################################################################################
+      - name: Set JDK 17 as default
+        run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
       - name: Reset existing repo
         run: |
           git config --global --add safe.directory $GITHUB_WORKSPACE

community-build/src/scala/dotty/communitybuild/projects.scala

@@ -127,6 +127,11 @@ object projects:
       s""";set $project/Compile/doc/sources ++= ($project/Compile/doc/dotty.tools.sbtplugin.DottyPlugin.autoImport.tastyFiles).value ;$project/doc"""
     ).mkString(" ")
 
+  private def removeRelease8(projects: String*): String =
+    projects.map(project =>
+      s"""set $project/Compile/scalacOptions := ($project/Compile/scalacOptions).value.filterNot(opt => opt == "-release" || opt == "8")"""
+    ).mkString("; ")
+
   private def aggregateDoc(in: String)(projects: String*) =
     val tastyFiles =
       (in +: projects).map(p => s"($p/Compile/doc/dotty.tools.sbtplugin.DottyPlugin.autoImport.tastyFiles).value").mkString(" ++ ")
@@ -445,7 +450,10 @@ object projects:
 
   lazy val discipline = SbtCommunityProject(
     project = "discipline",
-    sbtTestCommand = "coreJVM/test;coreJS/test",
+    sbtTestCommand = List(
+      removeRelease8("core.jvm", "core.js"),
+      "coreJVM/test;coreJS/test"
+    ).mkString("; "),
     sbtPublishCommand = "set every credentials := Nil;coreJVM/publishLocal;coreJS/publishLocal",
     scalacOptions = SbtCommunityProject.scalacOptions.filter(_ != "-Wsafe-init"),
   )
@@ -544,7 +552,12 @@ object projects:
 
   lazy val scissLucre = SbtCommunityProject(
     project           = "Lucre",
-    sbtTestCommand    = "adjunctJVM/test;baseJVM/test;confluentJVM/test;coreJVM/test;dataJVM/test;exprJVM/test;geomJVM/test;lucre-bdb/test;testsJVM/test",
+    sbtTestCommand    =
+      val subprojects = List("adjunct.jvm", "base.jvm", "confluent.jvm", "core.jvm", "data.jvm", "expr.jvm", "geom.jvm", "bdb", "tests.jvm")
+      List(
+        subprojects.map(name => s"""set ($name/Compile/compile/scalacOptions) := ($name/Compile/compile/scalacOptions).value.filterNot(opt => opt == "-release" || opt == "8")"""),
+        List("adjunctJVM/test;baseJVM/test;confluentJVM/test;coreJVM/test;dataJVM/test;exprJVM/test;geomJVM/test;lucre-bdb/test;testsJVM/test")
+      ).flatten.mkString("; "),
     extraSbtArgs      = List("-Dde.sciss.lucre.ShortTests=true"),
     sbtPublishCommand = "adjunctJVM/publishLocal;baseJVM/publishLocal;confluentJVM/publishLocal;coreJVM/publishLocal;dataJVM/publishLocal;exprJVM/publishLocal;geomJVM/publishLocal;lucre-bdb/publishLocal",
   )
@@ -615,7 +628,11 @@ object projects:
 
   lazy val fs2 = SbtCommunityProject(
     project = "fs2",
-    sbtTestCommand = "coreJVM/test; coreJS/test",  // io/test requires JDK9+
+    sbtTestCommand =
+      List(
+        removeRelease8("coreJVM", "coreJS"),
+        "coreJVM/test; coreJS/test;"
+      ).mkString("; "),
     sbtPublishCommand = "coreJVM/publishLocal; coreJS/publishLocal",
     scalacOptions = SbtCommunityProject.scalacOptions.filter(_ != "-Wsafe-init"),
   )

community-build/test/scala/dotty/communitybuild/CommunityBuildTest.scala

@@ -58,7 +58,8 @@ end CommunityBuildTestB
 @Category(Array(classOf[TestCategory]))
 class CommunityBuildTestC:
   @Test def akka = projects.akka.run()
-  @Test def betterfiles = projects.betterfiles.run()
+  // Disabled because `javax.xml.bind` is not available since java 11
+  // @Test def betterfiles = projects.betterfiles.run()
   @Test def cask = projects.cask.run()
   // Temporarily disabled until problem discovered in comments to #9449 is fixed
   // @Test def dottyCpsAsync = projects.dottyCpsAsync.run()
@@ -75,7 +76,8 @@ class CommunityBuildTestC:
   @Test def oslib = projects.oslib.run()
   // @Test def oslibWatch = projects.oslibWatch.run()
   @Test def parboiled2 = projects.parboiled2.run()
-  @Test def playJson = projects.playJson.run()
+  // Disabled because `javax.xml.bind` is not available since java 11
+  // @Test def playJson = projects.playJson.run()
   @Test def pprint = projects.pprint.run()
   //@Test def protoquill = projects.protoquill.run()
   @Test def requests = projects.requests.run()

compiler/src/dotty/tools/dotc/config/ScalaSettingsProperties.scala

@@ -10,15 +10,17 @@ object ScalaSettingsProperties:
   private lazy val minTargetVersion = classfileVersionMap.keysIterator.min
   private lazy val maxTargetVersion = classfileVersionMap.keysIterator.max
 
+  private val minReleaseVersion = 17
+
   def supportedTargetVersions: List[String] =
     (minTargetVersion to maxTargetVersion).toList.map(_.toString)
 
   def supportedReleaseVersions: List[String] =
     if scala.util.Properties.isJavaAtLeast("9") then
       val jdkVersion = JDK9Reflectors.runtimeVersionMajor(JDK9Reflectors.runtimeVersion()).intValue()
       val maxVersion = Math.min(jdkVersion, maxTargetVersion)
-      (minTargetVersion to maxVersion).toList.map(_.toString)
-    else List(minTargetVersion).map(_.toString)
+      (minReleaseVersion to maxVersion).toList.map(_.toString)
+    else List(minReleaseVersion).map(_.toString)
 
   def supportedScalaReleaseVersions: List[String] =
     ScalaRelease.values.toList.map(_.show)

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -756,7 +756,10 @@ class Definitions {
   def JavaEnumType = JavaEnumClass.typeRef
 
   @tu lazy val MethodHandleClass: ClassSymbol        = requiredClass("java.lang.invoke.MethodHandle")
+  @tu lazy val MethodHandlesClass: TermSymbol        = requiredModule("java.lang.invoke.MethodHandles")
+  @tu lazy val MethodHandles_lookup: Symbol          = MethodHandlesClass.requiredMethod("lookup")
   @tu lazy val MethodHandlesLookupClass: ClassSymbol = requiredClass("java.lang.invoke.MethodHandles.Lookup")
+    @tu lazy val MethodHandlesLookup_FindVarHandle: Symbol = MethodHandlesLookupClass.requiredMethod("findVarHandle")
   @tu lazy val VarHandleClass: ClassSymbol           = requiredClass("java.lang.invoke.VarHandle")
 
   @tu lazy val StringBuilderClass: ClassSymbol = requiredClass("scala.collection.mutable.StringBuilder")

compiler/src/dotty/tools/dotc/transform/LazyVals.scala

@@ -28,8 +28,10 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
    * The map contains the list of the offset trees.
    */
   class OffsetInfo(var defs: List[Tree], var ord: Int = 0)
+  class VarHandleInfo(var methodHandlesLookupDef: Tree, var varHandleDefs: List[Tree])
 
   private val appendOffsetDefs = mutable.Map.empty[Symbol, OffsetInfo]
+  private val appendVarHandleDefs = mutable.Map.empty[Symbol, VarHandleInfo]
 
   override def phaseName: String = LazyVals.name
 
@@ -109,12 +111,19 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
    */
   override def transformTemplate(template: Template)(using Context): Tree = {
     val cls = ctx.owner.asClass
-    appendOffsetDefs.get(cls) match {
-      case None => template
-      case Some(data) =>
-        data.defs.foreach(defin => defin.symbol.addAnnotation(Annotation(defn.ScalaStaticAnnot, defin.symbol.span)))
-        cpy.Template(template)(body = addInFront(data.defs, template.body))
-    }
+    if ctx.settings.YlegacyLazyVals.value then
+      appendOffsetDefs.get(cls) match {
+        case None => template
+        case Some(data) =>
+          data.defs.foreach(defin => defin.symbol.addAnnotation(Annotation(defn.ScalaStaticAnnot, defin.symbol.span)))
+          cpy.Template(template)(body = addInFront(data.defs, template.body))
+      }
+    else
+      appendVarHandleDefs.get(cls) match {
+        case None => template
+        case Some(data) =>
+          cpy.Template(template)(body = addInFront(data.methodHandlesLookupDef +: data.varHandleDefs, template.body))
+      }
   }
 
   private def addInFront(prefix: List[Tree], stats: List[Tree]) = stats match {
@@ -328,20 +337,24 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
    * @param memberDef     the transformed lazy field member definition
    * @param claz          the class containing this lazy val field
    * @param target        the target synthetic field
-   * @param offset        the offset of the field in the storage allocation of the class
+   * @param varHandle     the VarHandle of the field
    * @param thiz          a reference to the transformed class
    */
   def mkThreadSafeDef(memberDef: ValOrDefDef,
                       claz: ClassSymbol,
                       target: Symbol,
-                      offset: Tree,
+                      varHandle: Tree,
                       thiz: Tree)(using Context): (DefDef, DefDef) = {
     val tp = memberDef.tpe.widenDealias.resultType.widenDealias
     val waiting = ref(defn.LazyValsWaitingState)
     val controlState = ref(defn.LazyValsControlState)
     val evaluating = Select(ref(defn.LazyValsModule), lazyNme.RLazyVals.evaluating)
     val nullValue = Select(ref(defn.LazyValsModule), lazyNme.RLazyVals.nullValue)
-    val objCasFlag = Select(ref(defn.LazyValsModule), lazyNme.RLazyVals.objCas)
+    val casFlag =
+      typer.Applications.retypeSignaturePolymorphicFn( // must be retyped to avoid wrapping into Array[Object]
+        Select(varHandle, lazyNme.compareAndSet),
+        MethodType(List(defn.ObjectType,defn.ObjectType,defn.ObjectType), defn.BooleanType)
+      )
     val accessorMethodSymbol = memberDef.symbol.asTerm
     val lazyInitMethodName = LazyLocalInitName.fresh(memberDef.name.asTermName)
     val lazyInitMethodSymbol = newSymbol(claz, lazyInitMethodName, Synthetic | Method | Private, MethodType(Nil)(_ => Nil, _ => defn.ObjectType))
@@ -383,12 +396,12 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
       val lockRel = {
         val lockSymb = newSymbol(lazyInitMethodSymbol, lazyNme.lock, Synthetic, waiting.typeOpt)
         Block(ValDef(lockSymb, ref(target).cast(waiting.typeOpt))
-          :: objCasFlag.appliedTo(thiz, offset, ref(lockSymb), ref(resSymb)) :: Nil,
+          :: casFlag.appliedTo(thiz, ref(lockSymb), ref(resSymb)) :: Nil,
           ref(lockSymb).select(lazyNme.RLazyVals.waitingRelease).ensureApplied)
       }
       // finally block
       val fin = If(
-          objCasFlag.appliedTo(thiz, offset, evaluating, ref(resSymb)).select(nme.UNARY_!).appliedToNone,
+          casFlag.appliedTo(thiz, evaluating, ref(resSymb)).select(nme.UNARY_!).appliedToNone,
           lockRel,
           unitLiteral
         )
@@ -409,7 +422,7 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
       )
       // if CAS(_, null, Evaluating)
       If(
-        objCasFlag.appliedTo(thiz, offset, nullLiteral, evaluating),
+        casFlag.appliedTo(thiz, nullLiteral, evaluating),
         Block(ValDef(resSymb, nullLiteral) :: ValDef(resSymbNullable, nullLiteral) :: evaluate :: Nil, // var result: AnyRef = null
           Return(ref(resSymbNullable), lazyInitMethodSymbol)),
         unitLiteral
@@ -425,7 +438,7 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
             ref(current).select(defn.Object_eq).appliedTo(evaluating),
             // if is Evaluating then CAS(_, Evaluating, new Waiting)
             Block(
-              objCasFlag.appliedTo(thiz, offset, ref(current), Select(New(waiting), StdNames.nme.CONSTRUCTOR).ensureApplied) :: Nil,
+              casFlag.appliedTo(thiz, ref(current), Select(New(waiting), StdNames.nme.CONSTRUCTOR).ensureApplied) :: Nil,
               unitLiteral
             ),
             // if not Evaluating
@@ -461,33 +474,37 @@ class LazyVals extends MiniPhase with IdentityDenotTransformer {
     val claz = x.symbol.owner.asClass
     val thizClass = Literal(Constant(claz.info))
 
-    def offsetName(id: Int) = s"${StdNames.nme.LAZY_FIELD_OFFSET}${if (x.symbol.owner.is(Module)) "_m_" else ""}$id".toTermName
     val containerName = LazyLocalName.fresh(x.name.asTermName)
     val containerSymbol = newSymbol(claz, containerName, x.symbol.flags &~ containerFlagsMask | containerFlags | Private, defn.ObjectType, coord = x.symbol.coord).enteredAfter(this)
     containerSymbol.addAnnotation(Annotation(defn.VolatileAnnot, containerSymbol.span)) // private @volatile var _x: AnyRef
     containerSymbol.addAnnotations(x.symbol.annotations) // pass annotations from original definition
     containerSymbol.removeAnnotation(defn.ScalaStaticAnnot)
-    val getOffset =
-        Select(ref(defn.LazyValsModule), lazyNme.RLazyVals.getOffsetStatic)
     val containerTree = ValDef(containerSymbol, nullLiteral)
 
-    // create an offset for this lazy val
-    val offsetSymbol: TermSymbol = appendOffsetDefs.get(claz) match
-      case Some(info) =>
-        newSymbol(claz, offsetName(info.defs.size), Synthetic, defn.LongType).enteredAfter(this)
-      case None =>
-        newSymbol(claz, offsetName(0), Synthetic, defn.LongType).enteredAfter(this)
-    offsetSymbol.addAnnotation(Annotation(defn.ScalaStaticAnnot, offsetSymbol.span))
-    val fieldTree = thizClass.select(lazyNme.RLazyVals.getDeclaredField).appliedTo(Literal(Constant(containerName.mangledString)))
-    val offsetTree = ValDef(offsetSymbol, getOffset.appliedTo(fieldTree))
-    val offsetInfo = appendOffsetDefs.getOrElseUpdate(claz, new OffsetInfo(Nil))
-    offsetInfo.defs = offsetTree :: offsetInfo.defs
-    val offset = ref(offsetSymbol)
+    val varHandleInfo = appendVarHandleDefs.getOrElseUpdate(claz, new VarHandleInfo(EmptyTree, Nil))
+    varHandleInfo.methodHandlesLookupDef match
+      case EmptyTree =>
+        val lookupSym: TermSymbol = newSymbol(claz, (s"${claz.name}${lazyNme.methodHandleLookupSuffix}").toTermName, Synthetic, defn.MethodHandlesLookupClass.typeRef).enteredAfter(this)
+        lookupSym.addAnnotation(Annotation(defn.ScalaStaticAnnot, lookupSym.span))
+        varHandleInfo.methodHandlesLookupDef =
+          ValDef(lookupSym, Apply(Select(ref(defn.MethodHandlesClass), defn.MethodHandles_lookup.name), Nil))
+      case _ =>
+
+    // create a VarHandle for this lazy val
+    val varHandleSymbol: TermSymbol = newSymbol(claz, s"$containerName${lazyNme.lzyHandleSuffix}".toTermName, Synthetic, defn.VarHandleClass.typeRef).enteredAfter(this)
+    varHandleSymbol.addAnnotation(Annotation(defn.ScalaStaticAnnot, varHandleSymbol.span))
+    val getVarHandle = Apply(
+      Select(ref(varHandleInfo.methodHandlesLookupDef.symbol), defn.MethodHandlesLookup_FindVarHandle.name),
+      List(thizClass, Literal(Constant(containerName.mangledString)), Literal(Constant(defn.ObjectType)))
+    )
+    val varHandleTree = ValDef(varHandleSymbol, getVarHandle)
+    val varHandle = ref(varHandleSymbol)
 
+    varHandleInfo.varHandleDefs = varHandleTree :: varHandleInfo.varHandleDefs
     val swapOver =
         This(claz)
 
-    val (accessorDef, initMethodDef) = mkThreadSafeDef(x, claz, containerSymbol, offset, swapOver)
+    val (accessorDef, initMethodDef) = mkThreadSafeDef(x, claz, containerSymbol, varHandle, swapOver)
     Thicket(containerTree, accessorDef, initMethodDef)
   }
 
@@ -666,7 +683,6 @@ object LazyVals {
       val waitingRelease: TermName         = "countDown".toTermName
       val evaluating: TermName             = "Evaluating".toTermName
       val nullValue: TermName              = "NullValue".toTermName
-      val objCas: TermName                 = "objCAS".toTermName
       val get: TermName                    = N.get.toTermName
       val setFlag: TermName                = N.setFlag.toTermName
       val wait4Notification: TermName       = N.wait4Notification.toTermName
@@ -687,5 +703,13 @@ object LazyVals {
     val current: TermName     = "current".toTermName
     val lock: TermName        = "lock".toTermName
     val discard: TermName     = "discard".toTermName
+    val compareAndSet: TermName = "compareAndSet".toTermName
+    val lzyHandleSuffix: String = "$$lzyHandle"
+    val methodHandleLookupSuffix: String = "$$methodHandleLookup"
   }
+
+  extension (sym: Symbol) def isVarHandleForLazy(using Context) =
+    sym.name.endsWith(lazyNme.lzyHandleSuffix)
+  extension (sym: Symbol) def isMethodLookupForLazy(using Context) =
+    sym.name.endsWith(lazyNme.methodHandleLookupSuffix)
 }