Fix gcdx and lcm with mixed signed/unsigned arguments (#59628)

Add `gcdx(a::Signed, b::Unsigned)` and `gcdx(a::Unsigned, b::Signed)`
methods to fix #58025:
```julia
julia> gcdx(UInt16(100), Int8(-101))  # pr
(0x0001, 0xffff, 0xffff)

julia> gcdx(UInt16(100), Int8(-101))  # master, incorrect result
(0x0005, 0xf855, 0x0003)
```

Also add the equivalent methods for `lcm` to fix the systematic
`InexactError` when one argument is a negative `Signed` and the other is
any `Unsigned`:
```julia
julia> lcm(UInt16(100), Int8(-101))  # pr
0x2774

julia> lcm(UInt16(100), Int8(-101))  # master, error
ERROR: InexactError: trunc(UInt16, -101)
Stacktrace:
 [1] throw_inexacterror(func::Symbol, to::Type, val::Int8)
   @ Core ./boot.jl:866
 [2] check_sign_bit
   @ ./boot.jl:872 [inlined]
 [3] toUInt16
   @ ./boot.jl:958 [inlined]
 [4] UInt16
   @ ./boot.jl:1011 [inlined]
 [5] convert
   @ ./number.jl:7 [inlined]
 [6] _promote
   @ ./promotion.jl:379 [inlined]
 [7] promote
   @ ./promotion.jl:404 [inlined]
 [8] lcm(a::UInt16, b::Int8)
   @ Base ./intfuncs.jl:152
 [9] top-level scope
   @ REPL[62]:1
```

Inspired by
#59487 (comment).
The difference is that the solution proposed in this PR keeps the
current correct result type for inputs such as `(::Int16, ::UInt8)`.

(cherry picked from commit 4f1e471)

Author: Liozou

base/intfuncs.jl

@@ -148,6 +148,8 @@ gcd(a::Rational) = checked_abs(a.num) // a.den
 lcm(a::Union{Integer,Rational}) = gcd(a)
 gcd(a::Unsigned, b::Signed) = gcd(promote(a, abs(b))...)
 gcd(a::Signed, b::Unsigned) = gcd(promote(abs(a), b)...)
+lcm(a::Unsigned, b::Signed) = lcm(promote(a, abs(b))...)
+lcm(a::Signed, b::Unsigned) = lcm(promote(abs(a), b)...)
 gcd(a::Real, b::Real) = gcd(promote(a,b)...)
 lcm(a::Real, b::Real) = lcm(promote(a,b)...)
 gcd(a::Real, b::Real, c::Real...) = gcd(a, gcd(b, c...))
@@ -252,6 +254,16 @@ function gcdx(a::Real, b::Real, cs::Real...)
     d′, x, ys... = gcdx(d, cs...)
     return d′, i*x, j*x, ys...
 end
+function gcdx(a::Signed, b::Unsigned)
+    R = promote_type(typeof(a), typeof(b))
+    _a = a % signed(R) # handle the case a == typemin(typeof(a)) if R != typeof(a)
+    d, u, v = gcdx(promote(abs(_a), b)...)
+    d, flipsign(u, a), v
+end
+function gcdx(a::Unsigned, b::Signed)
+    d, v, u = gcdx(b, a)
+    d, u, v
+end
 
 # multiplicative inverse of n mod m, error if none
 

test/intfuncs.jl

@@ -216,6 +216,35 @@ end
     x, y = Int8(-12), UInt(100)
     d, u, v = gcdx(x, y)
     @test x*u + y*v == d
+
+end
+
+# issue #58025
+@testset "Mixed signed/unsigned types" begin
+    cases = [ # adapted from https://github.com/JuliaLang/julia/pull/59487#issuecomment-3258209203
+        (UInt16(100), Int8(-101)),
+        (Int8(-50), UInt16(75)),
+        (UInt32(12), Int16(-18)),
+        (Int64(-24), UInt8(36)),
+        (UInt8(15), Int16(-25)),
+        (Int32(-42), UInt64(56)),
+        (UInt128(1000), Int32(-1500)),
+        (UInt64(0), Int32(-5)),
+        (Int16(-7), UInt8(0)),
+        (Int8(-14), UInt8(13)),
+    ]
+    for (a, b) in cases
+        g1 = gcd(a, b)
+        g2, s, t = gcdx(a, b)
+        @test g1 === g2
+        @test s*a + t*b == g2
+        @test g2 >= 0
+        @test lcm(a, b) === convert(typeof(g1), lcm(widen(a), widen(b)))
+    end
+
+    @test gcdx(Int16(-32768), Int8(-128)) === (Int16(128), Int16(0), Int16(-1))
+    @test gcdx(Int8(-128), UInt16(256)) === (0x0080, 0xffff, 0x0000)
+    @test_broken gcd(Int8(-128), UInt16(256)) === 0x0080
 end
 
 @testset "gcd/lcm/gcdx for custom types" begin