Only nullify tasty if Ynullify-tasty is set; Refine FlexibleType nullification rules (#23938)

Given the nightly results, we think the nullification rules still need
some polish and is not ready for all compiled Scala code. Hence, we only
nullify tasty if `Ynullify-tasty` is set now.

Based on #23911

Fixes #23908

Could fix #23933, #23935, #23936, and #23937 (I tested locally due the
explicit nulls tests disabled temprarily)

Author: noti0na1

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -449,6 +449,7 @@ private sealed trait YSettings:
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-kind-polymorphism", "Disable kind polymorphism. (This flag has no effect)", deprecation = Deprecation.removed())
   val YexplicitNulls: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
   val YnoFlexibleTypes: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-flexible-types", "Disable turning nullable Java return types and parameter types into flexible types, which behave like abstract types with a nullable lower bound and non-nullable upper bound.")
+  val YflexifyTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Yflexify-tasty", "Apply flexification to Scala code compiled without -Yexplicit-nulls, when reading from tasty.")
   val YsafeInitGlobal: Setting[Boolean] = BooleanSetting(ForkSetting, "Ysafe-init-global", "Check safe initialization of global objects.")
   val YrequireTargetName: Setting[Boolean] = BooleanSetting(ForkSetting, "Yrequire-targetName", "Warn if an operator is defined without a @targetName annotation.")
   val YrecheckTest: Setting[Boolean] = BooleanSetting(ForkSetting, "Yrecheck-test", "Run basic rechecking (internal test only).")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -480,6 +480,9 @@ object Contexts {
     /** Is the flexible types option set? */
     def flexibleTypes: Boolean = base.settings.YexplicitNulls.value && !base.settings.YnoFlexibleTypes.value
 
+    /** Is the flexify tasty option set? */
+    def flexifyTasty: Boolean = base.settings.YexplicitNulls.value && base.settings.YflexifyTasty.value
+
     /** Is the best-effort option set? */
     def isBestEffort: Boolean = base.settings.YbestEffort.value
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1991,13 +1991,11 @@ class Definitions {
    *   - the upper bound of a TypeParamRef in the current constraint
    */
   def asContextFunctionType(tp: Type)(using Context): Type =
-    tp.stripTypeVar.dealias match
+    tp.stripNull().stripTypeVar.dealias match
       case tp1: TypeParamRef if ctx.typerState.constraint.contains(tp1) =>
         asContextFunctionType(TypeComparer.bounds(tp1).hiBound)
       case tp1 @ PolyFunctionOf(mt: MethodType) if mt.isContextualMethod =>
         tp1
-      case tp1: FlexibleType =>
-        asContextFunctionType(tp1.underlying)
       case tp1 =>
         if tp1.typeSymbol.name.isContextFunction && isFunctionNType(tp1) then tp1
         else NoType

compiler/src/dotty/tools/dotc/core/ImplicitNullInterop.scala

@@ -6253,6 +6253,8 @@ object Types extends TypeUtils {
       tp.derivedAndType(tp1, tp2)
     protected def derivedOrType(tp: OrType, tp1: Type, tp2: Type): Type =
       tp.derivedOrType(tp1, tp2)
+    protected def derivedAndOrType(tp: AndOrType, tp1: Type, tp2: Type): Type =
+      tp.derivedAndOrType(tp1, tp2)
     protected def derivedMatchType(tp: MatchType, bound: Type, scrutinee: Type, cases: List[Type]): Type =
       tp.derivedMatchType(bound, scrutinee, cases)
     protected def derivedAnnotatedType(tp: AnnotatedType, underlying: Type, annot: Annotation): Type =

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -921,10 +921,10 @@ class TreeUnpickler(reader: TastyReader,
 
       def ta = ctx.typeAssigner
 
-      // If explicit nulls is enabled, and the source file did not have explicit
+      // If explicit nulls and `Yflexify-tasty` is enabled, and the source file did not have explicit
       // nulls enabled, nullify the member to allow for compatibility.
       def nullify(sym: Symbol) =
-        if (ctx.explicitNulls && ctx.flexibleTypes && !explicitNulls) then
+        if (ctx.flexifyTasty && !explicitNulls) then
           sym.info = ImplicitNullInterop.nullifyMember(sym, sym.info, sym.is(Enum))
 
       val name = readName()

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -212,8 +212,18 @@ class CompilationTests {
       compileFilesInDir("tests/explicit-nulls/neg", explicitNullsOptions, FileFilter.exclude(TestSources.negExplicitNullsScala2LibraryTastyExcludelisted)),
       compileFilesInDir("tests/explicit-nulls/flexible-types-common", explicitNullsOptions and "-Yno-flexible-types"),
       compileFilesInDir("tests/explicit-nulls/unsafe-common", explicitNullsOptions and "-Yno-flexible-types", FileFilter.exclude(TestSources.negExplicitNullsScala2LibraryTastyExcludelisted)),
-    )
-  }.checkExpectedErrors()
+    ).checkExpectedErrors()
+
+    locally {
+      val unsafeFile = compileFile("tests/explicit-nulls/flexible-unpickle/neg/Unsafe_1.scala", explicitNullsOptions without "-Yexplicit-nulls")
+      val flexibleFile = compileFile("tests/explicit-nulls/flexible-unpickle/neg/Flexible_2.scala",
+          explicitNullsOptions.and("-Yflexify-tasty").withClasspath(defaultOutputDir + testGroup + "/Unsafe_1/neg/Unsafe_1"))
+
+      flexibleFile.keepOutput.checkExpectedErrors()
+
+      List(unsafeFile, flexibleFile).foreach(_.delete())
+    }
+  }
 
   @Ignore
   @Test def explicitNullsPos: Unit = {
@@ -226,9 +236,9 @@ class CompilationTests {
 
     locally {
       val tests = List(
-        compileFile("tests/explicit-nulls/flexible-unpickle/Unsafe_1.scala", explicitNullsOptions without "-Yexplicit-nulls"),
-        compileFile("tests/explicit-nulls/flexible-unpickle/Flexible_2.scala", explicitNullsOptions.withClasspath(
-          defaultOutputDir + testGroup + "/Unsafe_1/flexible-unpickle/Unsafe_1")),
+        compileFile("tests/explicit-nulls/flexible-unpickle/pos/Unsafe_1.scala", explicitNullsOptions without "-Yexplicit-nulls"),
+        compileFile("tests/explicit-nulls/flexible-unpickle/pos/Flexible_2.scala",
+        explicitNullsOptions.and("-Yflexify-tasty").withClasspath(defaultOutputDir + testGroup + "/Unsafe_1/pos/Unsafe_1")),
       ).map(_.keepOutput.checkCompile())
 
       tests.foreach(_.delete())

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -0,0 +1,24 @@
+import unsafe.*
+
+@A[String] class C
+
+def test =
+
+  val ii = Bar.f[Int](null) // error
+  val jj = Bar.g[String](null) // error
+  val jj2 = Bar.g[String | Null](null) // ok
+  val kk = Bar.g2[String](null) // error
+  val kk2 = Bar.g2[String | Null](null) // ok
+
+  val bar_x: Int = Bar.x
+  val bar_y: String | Null = Bar.y.replaceAll(" ","")
+
+  def testUsingFoo(using Foo[Option]) = Bar.h(null)
+
+  val ii2 = Bar2[String]().f(null) // error
+  val ii3 = Bar2[String | Null]().f(null) // ok
+
+  val a = Bar.ff(
+    (x: AnyRef) => x.toString,
+    42
+  )

tests/explicit-nulls/flexible-unpickle/neg/Flexible_2.scala

@@ -0,0 +1,22 @@
+package unsafe
+
+import scala.annotation.*
+
+type XtoY = [X] =>> [Y] =>> X => Y
+
+class Foo[T[_]]
+
+class A[T] extends Annotation
+
+object Bar:
+  def ff(f: AnyRef => String, g: AnyRef ?=> Int): (AnyRef => String) = ???
+  var x: Int = 0
+  var y: String = ""
+  def f[T <: Int](x: T): T = x
+  def g[T <: AnyRef](x: T): T = x
+  def g2[T >: Null <: AnyRef](x: T): T = x
+  def h(x: String)(using Foo[Option]): String = x
+  def h2(a: Foo[XtoY[String]]) = ???
+
+class Bar2[T]:
+  def f(x: T): T = x

tests/explicit-nulls/flexible-unpickle/neg/Unsafe_1.scala

@@ -1,6 +1,6 @@
 import unsafeNulls.Foo.*
 import unsafeNulls.Unsafe_1
-import unsafeNulls.{A, B, C, F, G, H, I, J, L, M, S, T, U, expects}
+import unsafeNulls.{A, B, C, F, G, H, I, J, L, M, N, S, T, U, expects}
 import scala.reflect.Selectable.reflectiveSelectable
 import scala.quoted.*
 
@@ -100,6 +100,11 @@ def Flexible_2() =
 
   val m: String = M.test(null)
 
+  // i23911
+  val n1: List[Map[String, Int]] = ???
+  val n2 = new N[List]()
+  val n3 = n2.accept[Any](n1)
+
   // i23845
   transparent inline def typeName[A]: String = ${typeNameMacro[A]}
 
@@ -109,3 +114,9 @@ def Flexible_2() =
     implicit val givenS: S[A] = ???
     expects(alphaTypeNameMacro[A])
   }
+
+// i23935
+opaque type ZArrow[-I, -R, +E, +O] = I => ZIO[R, E, O]
+object ZArrow:
+  def fromZIOAttempt[I, R, E, O](f: I => ZIO[R, E, O]): ZArrow[I, R, Throwable | E, O] =
+    (in: I) => ZIO.attempt(f(in)).flatten