Fix overflow and type issues with generic det of a triangular matrix (#1418)

Fixes #1415

---------

Co-authored-by: Steven G. Johnson <[email protected]>

Author: lxvm

src/generic.jl

@@ -1846,7 +1846,7 @@ julia> det(BigInt[1 0; 2 2]) # exact integer determinant
 function det(A::AbstractMatrix{T}) where {T}
     if istriu(A) || istril(A)
         S = promote_type(T, typeof((one(T)*zero(T) + zero(T))/one(T)))
-        return convert(S, det(UpperTriangular(A)))
+        return prod(Base.Fix1(convert, S), @view A[diagind(A)]; init=one(S))
     end
     return det(lu(A; check = false))
 end

test/generic.jl

@@ -17,6 +17,7 @@ using Main.LinearAlgebraTestHelpers.OffsetArrays
 using Main.LinearAlgebraTestHelpers.DualNumbers
 using Main.LinearAlgebraTestHelpers.FillArrays
 using Main.LinearAlgebraTestHelpers.SizedArrays
+using Main.LinearAlgebraTestHelpers.Furlongs
 
 Random.seed!(123)
 
@@ -96,6 +97,18 @@ n = 5 # should be odd
     @testset "det with nonstandard Number type" begin
         elty <: Real && @test det(Dual.(triu(A), zero(A))) isa Dual
     end
+    if elty <: Int
+        @testset "det no overflow - triangular" begin
+            A = diagm([typemax(elty), typemax(elty)])
+            @test det(A) == det(float(A))
+        end
+    end
+    @testset "det with units - triangular" begin
+        for dim in 0:4
+            A = diagm(Furlong.(ones(elty, dim)))
+            @test det(A) == Furlong{dim}(one(elty))
+        end
+    end
 end
 
 @testset "diag" begin