fix type intersection bug affecting Dolang.jl and SolverTools.jl (#32853)

Author: JeffBezanson

src/subtype.c

@@ -536,10 +536,13 @@ static void record_var_occurrence(jl_varbinding_t *vb, jl_stenv_t *e, int param)
 {
     if (vb != NULL && param) {
         // saturate counters at 2; we don't need values bigger than that
-        if (param == 2 && (vb->right ? e->Rinvdepth : e->invdepth) > vb->depth0 && vb->occurs_inv < 2)
-            vb->occurs_inv++;
-        else if (vb->occurs_cov < 2)
+        if (param == 2 && (vb->right ? e->Rinvdepth : e->invdepth) > vb->depth0) {
+            if (vb->occurs_inv < 2)
+                vb->occurs_inv++;
+        }
+        else if (vb->occurs_cov < 2) {
             vb->occurs_cov++;
+        }
     }
 }
 
@@ -1275,7 +1278,15 @@ static int subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int param)
             jl_value_t *tp0 = jl_tparam0(yd);
             if (!jl_is_typevar(tp0) || !jl_is_kind(x))
                 return 0;
-            return subtype((jl_value_t*)jl_type_type, y, e, param);
+            // DataType.super is special, so `DataType <: Type{T}` (T free) needs special handling.
+            // The answer is true iff `T` has full bounds (as in `Type`), but this needs to
+            // be checked at the same depth where `Type{T}` occurs --- the depth of the LHS
+            // doesn't matter because it (e.g. `DataType`) doesn't actually contain the variable.
+            int saved = e->invdepth;
+            e->invdepth = e->Rinvdepth;
+            int issub = subtype((jl_value_t*)jl_type_type, y, e, param);
+            e->invdepth = saved;
+            return issub;
         }
         while (xd != jl_any_type && xd->name != yd->name) {
             if (xd->super == NULL)

test/subtype.jl

@@ -1672,3 +1672,7 @@ c32703(::Type{<:Str{C}}, str::Str{C}) where {C<:CSE} = str
 @test_broken typeintersect(Tuple{Vector{Vector{Float32}},Matrix,Matrix},
                            Tuple{Vector{V},Matrix{Int},Matrix{S}} where {S, V<:AbstractVector{S}}) ==
              Tuple{Array{Array{Float32,1},1},Array{Int,2},Array{Float32,2}}
+
+@testintersect(Tuple{Pair{Int, DataType}, Any},
+               Tuple{Pair{A, B} where B<:Type, Int} where A,
+               Tuple{Pair{Int, DataType}, Int})