Add experimental flexible types feature on top of explicit nulls

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -379,6 +379,7 @@ private sealed trait YSettings:
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting("-Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
+  val YflexibleTypes: Setting[Boolean] = BooleanSetting("-Yflexible-types", "Make Java return types and parameter types use flexible types. Flexible types essentially circumvent explicit nulls and force something resembling the old type system for Java interop.")
   val YcheckInit: Setting[Boolean] = BooleanSetting("-Ysafe-init", "Ensure safe initialization of objects")
   val YrequireTargetName: Setting[Boolean] = BooleanSetting("-Yrequire-targetName", "Warn if an operator is defined without a @targetName annotation")
   val YrecheckTest: Setting[Boolean] = BooleanSetting("-Yrecheck-test", "Run basic rechecking (internal test only)")

compiler/src/dotty/tools/dotc/core/CheckRealizable.scala

@@ -122,6 +122,7 @@ class CheckRealizable(using Context) {
         case tp: TypeProxy => isConcrete(tp.underlying)
         case tp: AndType => isConcrete(tp.tp1) && isConcrete(tp.tp2)
         case tp: OrType  => isConcrete(tp.tp1) && isConcrete(tp.tp2)
+        case tp: FlexibleType => isConcrete(tp.underlying)
         case _ => false
       }
       if (!isConcrete(tp)) NotConcrete

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -453,6 +453,9 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is the flexible types option set? */
+    def flexibleTypes: Boolean = base.settings.YflexibleTypes.value
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -63,10 +63,10 @@ object JavaNullInterop {
       // Don't nullify the return type of the `toString` method.
       // Don't nullify the return type of constructors.
       // Don't nullify the return type of methods with a not-null annotation.
-      nullifyExceptReturnType(tp)
+      nullifyExceptReturnType(tp, sym.owner.isClass)
     else
       // Otherwise, nullify everything
-      nullifyType(tp)
+      nullifyType(tp, sym.owner.isClass)
   }
 
   private def hasNotNullAnnot(sym: Symbol)(using Context): Boolean =
@@ -77,12 +77,12 @@ object JavaNullInterop {
    *  If tp is a type of a field, the inside of the type is nullified,
    *  but the result type is not nullable.
    */
-  private def nullifyExceptReturnType(tp: Type)(using Context): Type =
-    new JavaNullMap(true)(tp)
+  private def nullifyExceptReturnType(tp: Type, ownerIsClass: Boolean)(using Context): Type =
+    if ctx.flexibleTypes /*&& ownerIsClass*/ then new JavaFlexibleMap(true)(tp) else new JavaNullMap(true)(tp) // FLEX PARAMS
 
   /** Nullifies a Java type by adding `| Null` in the relevant places. */
-  private def nullifyType(tp: Type)(using Context): Type =
-    new JavaNullMap(false)(tp)
+  private def nullifyType(tp: Type, ownerIsClass: Boolean)(using Context): Type =
+    if ctx.flexibleTypes /*&& ownerIsClass*/ then new JavaFlexibleMap(false)(tp) else new JavaNullMap(false)(tp) // FLEX PARAMS
 
   /** A type map that implements the nullification function on types. Given a Java-sourced type, this adds `| Null`
    *  in the right places to make the nulls explicit in Scala.
@@ -146,4 +146,63 @@ object JavaNullInterop {
       case _ => tp
     }
   }
+
+  /**
+   *  Flexible types
+   */
+
+  private class JavaFlexibleMap(var outermostLevelAlreadyNullable: Boolean)(using Context) extends TypeMap {
+    /** Should we nullify `tp` at the outermost level? */
+    def needsFlexible(tp: Type): Boolean =
+      !outermostLevelAlreadyNullable && (tp match {
+        case tp: TypeRef =>
+          // We don't modify value types because they're non-nullable even in Java.
+          !tp.symbol.isValueClass &&
+          // We don't modify `Any` because it's already nullable.
+          !tp.isRef(defn.AnyClass) &&
+          // We don't nullify Java varargs at the top level.
+          // Example: if `setNames` is a Java method with signature `void setNames(String... names)`,
+          // then its Scala signature will be `def setNames(names: (String|Null)*): Unit`.
+          // This is because `setNames(null)` passes as argument a single-element array containing the value `null`,
+          // and not a `null` array.
+          !tp.isRef(defn.RepeatedParamClass)
+        case _ => true
+      })
+
+    override def apply(tp: Type): Type = tp match {
+      case tp: TypeRef if needsFlexible(tp) =>
+        //println(Thread.currentThread().getStackTrace()(3).getMethodName())
+        FlexibleType(tp)
+      case appTp @ AppliedType(tycon, targs) =>
+        val oldOutermostNullable = outermostLevelAlreadyNullable
+        // We don't make the outmost levels of type arguments nullable if tycon is Java-defined.
+        // This is because Java classes are _all_ nullified, so both `java.util.List[String]` and
+        // `java.util.List[String|Null]` contain nullable elements.
+        outermostLevelAlreadyNullable = tp.classSymbol.is(JavaDefined)
+        val targs2 = targs map this
+        outermostLevelAlreadyNullable = oldOutermostNullable
+        val appTp2 = derivedAppliedType(appTp, tycon, targs2)
+        if needsFlexible(tycon) then FlexibleType(appTp2) else appTp2
+      case ptp: PolyType =>
+        derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
+      case mtp: MethodType =>
+        val oldOutermostNullable = outermostLevelAlreadyNullable
+        outermostLevelAlreadyNullable = false
+        val paramInfos2 = mtp.paramInfos map this /*new JavaNullMap(outermostLevelAlreadyNullable)*/ // FLEX PARAMS
+        outermostLevelAlreadyNullable = oldOutermostNullable
+        derivedLambdaType(mtp)(paramInfos2, this(mtp.resType))
+      case tp: TypeAlias => mapOver(tp)
+      case tp: AndType =>
+        // nullify(A & B) = (nullify(A) & nullify(B)) | Null, but take care not to add
+        // duplicate `Null`s at the outermost level inside `A` and `B`.
+        outermostLevelAlreadyNullable = true
+        FlexibleType(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
+      case tp: TypeParamRef if needsFlexible(tp) =>
+        FlexibleType(tp)
+      // In all other cases, return the type unchanged.
+      // In particular, if the type is a ConstantType, then we don't nullify it because it is the
+      // type of a final non-nullable field.
+      case _ => tp
+    }
+  }
 }

compiler/src/dotty/tools/dotc/core/NullOpsDecorator.scala

@@ -8,6 +8,12 @@ import Types._
 object NullOpsDecorator:
 
   extension (self: Type)
+    def stripFlexible(using Context): Type = {
+      self match {
+        case FlexibleType(tp) => tp
+        case _ => self
+      }
+    }
     /** Syntactically strips the nullability from this type.
      *  If the type is `T1 | ... | Tn`, and `Ti` references to `Null`,
      *  then return `T1 | ... | Ti-1 | Ti+1 | ... | Tn`.
@@ -33,6 +39,7 @@ object NullOpsDecorator:
             if (tp1s ne tp1) && (tp2s ne tp2) then
               tp.derivedAndType(tp1s, tp2s)
             else tp
+          case tp @ FlexibleType(tp1) => strip(tp1)
           case tp @ TypeBounds(lo, hi) =>
             tp.derivedTypeBounds(strip(lo), strip(hi))
           case tp => tp

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -560,6 +560,9 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
       case CapturingType(parent, refs) =>
         val parent1 = recur(parent)
         if parent1 ne parent then tp.derivedCapturingType(parent1, refs) else tp
+      case tp: FlexibleType =>
+        val underlying = recur(tp.underlying)
+        if underlying ne tp.underlying then tp.derivedFlexibleType(underlying) else tp
       case tp: AnnotatedType =>
         val parent1 = recur(tp.parent)
         if parent1 ne tp.parent then tp.derivedAnnotatedType(parent1, tp.annot) else tp

compiler/src/dotty/tools/dotc/core/PatternTypeConstrainer.scala

@@ -10,6 +10,7 @@ import Contexts.ctx
 import dotty.tools.dotc.reporting.trace
 import config.Feature.migrateTo3
 import config.Printers._
+import dotty.tools.dotc.core.NullOpsDecorator.stripFlexible
 
 trait PatternTypeConstrainer { self: TypeComparer =>
 
@@ -175,7 +176,7 @@ trait PatternTypeConstrainer { self: TypeComparer =>
       case tp => tp
     }
 
-    dealiasDropNonmoduleRefs(scrut) match {
+    dealiasDropNonmoduleRefs(scrut.stripFlexible) match {
       case OrType(scrut1, scrut2) =>
         either(constrainPatternType(pat, scrut1), constrainPatternType(pat, scrut2))
       case AndType(scrut1, scrut2) =>

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -2243,6 +2243,9 @@ object SymDenotations {
           case CapturingType(parent, refs) =>
             tp.derivedCapturingType(recur(parent), refs)
 
+          case tp: FlexibleType =>
+            recur(tp.underlying)
+
           case tp: TypeProxy =>
             def computeTypeProxy = {
               val superTp = tp.superType

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -382,6 +382,10 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
       case OrType(tp21, tp22) =>
         if (tp21.stripTypeVar eq tp22.stripTypeVar) recur(tp1, tp21)
         else secondTry
+      // tp1 <: Flex(T) = T|N..T
+      // iff  tp1 <: T|N
+      case tp2: FlexibleType =>
+        recur(tp1, tp2.lo)
       case TypeErasure.ErasedValueType(tycon1, underlying2) =>
         def compareErasedValueType = tp1 match {
           case TypeErasure.ErasedValueType(tycon2, underlying1) =>
@@ -530,7 +534,10 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           hardenTypeVars(tp2)
 
         res
-
+      // invariant: tp2 is NOT a FlexibleType
+      // is Flex(T) <: tp2?
+      case tp1: FlexibleType =>
+        recur(tp1.underlying, tp2)
       case CapturingType(parent1, refs1) =>
         if tp2.isAny then true
         else if subCaptures(refs1, tp2.captureSet, frozenConstraint).isOK && sameBoxed(tp1, tp2, refs1)
@@ -2509,53 +2516,73 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
   /** Try to distribute `&` inside type, detect and handle conflicts
    *  @pre !(tp1 <: tp2) && !(tp2 <:< tp1) -- these cases were handled before
    */
-  private def distributeAnd(tp1: Type, tp2: Type): Type = tp1 match {
-    case tp1 @ AppliedType(tycon1, args1) =>
-      tp2 match {
-        case AppliedType(tycon2, args2)
-        if tycon1.typeSymbol == tycon2.typeSymbol && tycon1 =:= tycon2 =>
-          val jointArgs = glbArgs(args1, args2, tycon1.typeParams)
-          if (jointArgs.forall(_.exists)) (tycon1 & tycon2).appliedTo(jointArgs)
-          else NoType
-        case _ =>
-          NoType
-      }
-    case tp1: RefinedType =>
-      // opportunistically merge same-named refinements
-      // this does not change anything semantically (i.e. merging or not merging
-      // gives =:= types), but it keeps the type smaller.
-      tp2 match {
-        case tp2: RefinedType if tp1.refinedName == tp2.refinedName =>
-          val jointInfo = Denotations.infoMeet(tp1.refinedInfo, tp2.refinedInfo, safeIntersection = false)
-          if jointInfo.exists then
-            tp1.derivedRefinedType(tp1.parent & tp2.parent, tp1.refinedName, jointInfo)
-          else
+  private def distributeAnd(tp1: Type, tp2: Type): Type = {
+    var ft1 = false
+    var ft2 = false
+    def recur(tp1: Type, tp2: Type): Type = tp1 match {
+      case tp1 @ FlexibleType(tp) =>
+        // Hack -- doesn't generalise to other intersection/union types
+        // but covers a common special case for pattern matching
+        ft1 = true
+        recur(tp, tp2)
+      case tp1 @ AppliedType(tycon1, args1) =>
+        tp2 match {
+          case AppliedType(tycon2, args2)
+          if tycon1.typeSymbol == tycon2.typeSymbol && tycon1 =:= tycon2 =>
+            val jointArgs = glbArgs(args1, args2, tycon1.typeParams)
+            if (jointArgs.forall(_.exists)) (tycon1 & tycon2).appliedTo(jointArgs)
+            else {
+              NoType
+            }
+          case FlexibleType(tp) =>
+            // Hack from above
+            ft2 = true
+            recur(tp1, tp)
+          case _ =>
             NoType
-        case _ =>
-          NoType
-      }
-    case tp1: RecType =>
-      tp1.rebind(distributeAnd(tp1.parent, tp2))
-    case ExprType(rt1) =>
-      tp2 match {
-        case ExprType(rt2) =>
-          ExprType(rt1 & rt2)
-        case _ =>
-          NoType
-      }
-    case tp1: TypeVar if tp1.isInstantiated =>
-      tp1.underlying & tp2
-    case CapturingType(parent1, refs1) =>
-      if subCaptures(tp2.captureSet, refs1, frozen = true).isOK
-        && tp1.isBoxedCapturing == tp2.isBoxedCapturing
-      then
-        parent1 & tp2
-      else
-        tp1.derivedCapturingType(parent1 & tp2, refs1)
-    case tp1: AnnotatedType if !tp1.isRefining =>
-      tp1.underlying & tp2
-    case _ =>
-      NoType
+        }
+
+      // if result exists and is not notype, maybe wrap result in flex based on whether seen flex on both sides
+      case tp1: RefinedType =>
+        // opportunistically merge same-named refinements
+        // this does not change anything semantically (i.e. merging or not merging
+        // gives =:= types), but it keeps the type smaller.
+        tp2 match {
+          case tp2: RefinedType if tp1.refinedName == tp2.refinedName =>
+            val jointInfo = Denotations.infoMeet(tp1.refinedInfo, tp2.refinedInfo, safeIntersection = false)
+            if jointInfo.exists then
+              tp1.derivedRefinedType(tp1.parent & tp2.parent, tp1.refinedName, jointInfo)
+            else
+              NoType
+          case _ =>
+            NoType
+        }
+      case tp1: RecType =>
+        tp1.rebind(recur(tp1.parent, tp2))
+      case ExprType(rt1) =>
+        tp2 match {
+          case ExprType(rt2) =>
+            ExprType(rt1 & rt2)
+          case _ =>
+            NoType
+        }
+      case tp1: TypeVar if tp1.isInstantiated =>
+        tp1.underlying & tp2
+      case CapturingType(parent1, refs1) =>
+        if subCaptures(tp2.captureSet, refs1, frozen = true).isOK
+          && tp1.isBoxedCapturing == tp2.isBoxedCapturing
+        then
+          parent1 & tp2
+        else
+          tp1.derivedCapturingType(parent1 & tp2, refs1)
+      case tp1: AnnotatedType if !tp1.isRefining =>
+        tp1.underlying & tp2
+      case _ =>
+        NoType
+    }
+    // if flex on both sides, return flex type
+    val ret = recur(tp1, tp2)
+    if (ft1 && ft2) then FlexibleType(ret) else ret
   }
 
   /** Try to distribute `|` inside type, detect and handle conflicts

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -311,6 +311,8 @@ object TypeErasure {
           repr1.orElse(repr2)
         else
           NoSymbol
+      case tp: FlexibleType =>
+        arrayUpperBound(tp.underlying)
       case _ =>
         NoSymbol
 
@@ -337,6 +339,8 @@ object TypeErasure {
         isGenericArrayElement(tp.tp1, isScala2) && isGenericArrayElement(tp.tp2, isScala2)
       case tp: OrType =>
         isGenericArrayElement(tp.tp1, isScala2) || isGenericArrayElement(tp.tp2, isScala2)
+      case tp: FlexibleType =>
+        isGenericArrayElement(tp.underlying, isScala2)
       case _ => false
     }
   }
@@ -526,6 +530,7 @@ object TypeErasure {
     case tp: TypeProxy => hasStableErasure(tp.translucentSuperType)
     case tp: AndType => hasStableErasure(tp.tp1) && hasStableErasure(tp.tp2)
     case tp: OrType  => hasStableErasure(tp.tp1) && hasStableErasure(tp.tp2)
+    case _: FlexibleType => false
     case _ => false
   }
 
@@ -622,6 +627,7 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
       erasePolyFunctionApply(refinedInfo)
     case RefinedType(parent, nme.apply, refinedInfo: MethodType) if defn.isErasedFunctionType(parent) =>
       eraseErasedFunctionApply(refinedInfo)
+    case FlexibleType(tp) => this(tp)
     case tp: TypeProxy =>
       this(tp.underlying)
     case tp @ AndType(tp1, tp2) =>

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -52,7 +52,6 @@ object TypeOps:
         Stats.record("asSeenFrom skolem prefix required")
       case _ =>
     }
-
     new AsSeenFromMap(pre, cls).apply(tp)
   }
 
@@ -237,6 +236,7 @@ object TypeOps:
         if tp1.isBottomType && (tp1 frozen_<:< tp2) then orBaseClasses(tp2)
         else if tp2.isBottomType && (tp2 frozen_<:< tp1) then orBaseClasses(tp1)
         else intersect(orBaseClasses(tp1), orBaseClasses(tp2))
+      case FlexibleType(tp1) => orBaseClasses(tp1)
       case _ => tp.baseClasses
 
     /** The minimal set of classes in `cs` which derive all other classes in `cs` */