hashing interface

Project.toml

@@ -7,11 +7,15 @@ version = "0.6.1"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 nauty_jll = "55c6dc9b-343a-50ca-8ff2-b71adb3733d5"
+xxHash_jll = "5fdcd639-92d1-5a06-bf6b-28f2061df1a9"
 
 [compat]
 Graphs = "1.9"
 LinearAlgebra = "1"
 SHA = "0.7, 1"
+Serialization = "1"
 julia = "1.6"
 nauty_jll = "2.8.9"
+xxHash_jll = "0.8"

src/NautyGraphs.jl

@@ -1,16 +1,16 @@
 module NautyGraphs
 
-using Graphs, LinearAlgebra, SHA
+using Graphs, LinearAlgebra
 using Graphs.SimpleGraphs: SimpleEdgeIter
 import nauty_jll
+import xxHash_jll, SHA, Serialization
 
 const Cbool = Cint
-const HashType = UInt
-
 abstract type AbstractNautyGraph{T} <: AbstractGraph{T} end
 
 include("utils.jl")
 include("graphset.jl")
+include("hashing.jl")
 include("densenautygraph.jl")
 include("nauty.jl")
 
@@ -32,10 +32,12 @@ export
     DenseNautyGraph,
     AutomorphismGroup,
     labels,
+    iscanon,
     nauty,
     canonize!,
     canonical_permutation,
     is_isomorphic,
     ≃,
-    ghash
+    ghash,
+    AbstractHashAlg, XXHash64Alg, XXHash128Alg, SHA64Alg, SHA128Alg, Base64Alg
 end

src/densenautygraph.jl

@@ -9,7 +9,7 @@ mutable struct DenseNautyGraph{D,W<:Unsigned} <: AbstractNautyGraph{Int}
     graphset::Graphset{W}
     labels::Vector{Int}
     ne::Int
-    hashval::Union{HashType,Nothing}
+    iscanon::Bool
 end
 function DenseNautyGraph{D}(graphset::Graphset{W}; vertex_labels=nothing) where {D,W}
     if !isnothing(vertex_labels) && graphset.n != length(vertex_labels)
@@ -21,7 +21,7 @@ function DenseNautyGraph{D}(graphset::Graphset{W}; vertex_labels=nothing) where
     if isnothing(vertex_labels)
         vertex_labels = zeros(Int, graphset.n)
     end
-    return DenseNautyGraph{D,W}(graphset, vertex_labels, ne, nothing)
+    return DenseNautyGraph{D,W}(graphset, vertex_labels, ne, false)
 end
 
 
@@ -64,6 +64,7 @@ end
 function (::Type{G})(g::AbstractNautyGraph) where {G<:AbstractNautyGraph}
     h = invoke(G, Tuple{AbstractGraph}, g)
     @views h.labels .= g.labels
+    h.iscanon = g.iscanon
     return h
 end
 
@@ -90,12 +91,12 @@ end
 DenseNautyGraph{D}(edge_list::Vector{<:AbstractEdge}; vertex_labels=nothing) where {D} = DenseNautyGraph{D,UInt}(edge_list; vertex_labels)
 
 
-Base.copy(g::G) where {G<:DenseNautyGraph} = G(copy(g.graphset), copy(g.labels), g.ne, g.hashval)
+Base.copy(g::G) where {G<:DenseNautyGraph} = G(copy(g.graphset), copy(g.labels), g.ne, g.iscanon)
 function Base.copy!(dest::G, src::G) where {G<:DenseNautyGraph}
     copy!(dest.graphset, src.graphset)
     copy!(dest.labels, src.labels)
     dest.ne = src.ne
-    dest.hashval = src.hashval
+    dest.iscanon = src.iscanon
     return dest
 end
 
@@ -107,6 +108,7 @@ Base.:(==)(g::DenseNautyGraph, h::DenseNautyGraph) = (g.graphset == h.graphset)
 
 # BASIC GRAPH API
 labels(g::AbstractNautyGraph) = g.labels
+iscanon(g::AbstractNautyGraph) = g.iscanon
 Graphs.nv(g::DenseNautyGraph) = g.graphset.n
 Graphs.ne(g::DenseNautyGraph) = g.ne
 Graphs.vertices(g::DenseNautyGraph) = Base.OneTo(nv(g))
@@ -222,7 +224,7 @@ function Graphs.add_edge!(g::DenseNautyGraph, e::Edge)
     g.graphset[e.src, e.dst] = 1
     is_directed(g) || (g.graphset[e.dst, e.src] = 1)
     g.ne += 1
-    g.hashval = nothing
+    g.iscanon = false
     return true
 end
 Graphs.add_edge!(g::AbstractNautyGraph, i::Integer, j::Integer) = Graphs.add_edge!(g, edgetype(g)(i, j))
@@ -235,7 +237,7 @@ function Graphs.rem_edge!(g::DenseNautyGraph, e::Edge)
     g.graphset[e.src, e.dst] = 0
     is_directed(g) || (g.graphset[e.dst, e.src] = 0)
     g.ne -= 1
-    g.hashval = nothing
+    g.iscanon = false
     return true
 end
 Graphs.rem_edge!(g::AbstractNautyGraph, i::Integer, j::Integer) = Graphs.rem_edge!(g, edgetype(g)(i, j))
@@ -246,7 +248,7 @@ function Graphs.add_vertices!(g::DenseNautyGraph, n::Integer; vertex_labels=0)
     _add_vertices!(g.graphset, n)
     resize!(g.labels, ng + n)
     g.labels[ng+1:end] .= vertex_labels
-    g.hashval = nothing
+    g.iscanon = false
     return n
 end
 Graphs.add_vertex!(g::DenseNautyGraph; vertex_label::Integer=0) = Graphs.add_vertices!(g, 1; vertex_labels=vertex_label) > 0
@@ -261,7 +263,7 @@ function Graphs.rem_vertices!(g::DenseNautyGraph, inds)
     is_directed(g) || (ne ÷= 2)
     g.ne = ne
 
-    g.hashval = nothing
+    g.iscanon = false
     return true
 end
 Graphs.rem_vertex!(g::DenseNautyGraph, i::Integer) = rem_vertices!(g, (i,))

src/graphset.jl

@@ -103,16 +103,26 @@ end
     return gset
 end
 
-function _increase_padding!(gset::Graphset{W}, m::Integer=1) where {W}
-    # TODO: optimize this
-    for _ in Base.OneTo(m)
+@inline function active_words(gset::Graphset{W}) where {W}
+    # Return the words actually used for representing the graph, without any unnecessary padding
+    m_eff = cld(gset.n, wordsize(W))
+    return (gset.words[(i - 1) * gset.m + j] for j in 1:m_eff for i in 1:gset.n)
+end
+
+function increase_padding!(gset::Graphset{W}, Δm::Integer=1) where {W}
+    for _ in Base.OneTo(Δm)
         gset.m += 1
         for i in Base.OneTo(gset.n)
             insert!(gset.words, i * gset.m, zero(W))
         end
     end
     return gset
 end
+# function decrease_padding!(gset::Graphset{W}, Δm::Integer=1) where {W}
+#     return gset
+# end
+# function minimize_padding!(gset::Graphset{W}) where {W}
+# end
 
 @inline function partial_leftshift(word::Unsigned, n::Integer, start::Integer, fillword::Unsigned=zero(word))
     # Starting from the `start`th bit from the left of `word`, shift all bits to the left `n` times,
@@ -130,7 +140,7 @@ end
 end
 
 function _add_vertices!(gset::Graphset{W}, n::Integer) where {W} # TODO think of a better name
-    _increase_padding!(gset, cld(gset.n + n, wordsize(gset)) - gset.m)
+    increase_padding!(gset, cld(gset.n + n, wordsize(gset)) - gset.m)
     append!(gset.words, fill(zero(W), n*gset.m))
     gset.n += n
     return gset

src/hashing.jl

@@ -0,0 +1,66 @@
+abstract type AbstractHashAlg end
+struct Base64Alg <: AbstractHashAlg end
+struct XXHash64Alg <: AbstractHashAlg end
+struct XXHash128Alg <: AbstractHashAlg end
+struct SHA64Alg <: AbstractHashAlg end
+struct SHA128Alg <: AbstractHashAlg end
+
+function _ghash_base64(gset::Graphset, labels)
+    if length(gset) > 8192
+        throw(ArgumentError("Graph is too large (`nv(g) > 90`) and cannot be hashed using `Base64Alg`. Use a different hash algorithm instead."))
+    end
+    return Base.hash(labels, Base.hash(collect(active_words(gset))))
+end
+
+__xxhash64(x::AbstractArray) = @ccall xxHash_jll.libxxhash.XXH3_64bits(Ref(x, 1)::Ptr{Cvoid}, sizeof(x)::Csize_t)::UInt64
+__xxhash64seed(x::AbstractArray, seed::UInt64) = @ccall xxHash_jll.libxxhash.XXH3_64bits_withSeed(Ref(x, 1)::Ptr{Cvoid}, sizeof(x)::Csize_t, seed::UInt64)::UInt64
+function _ghash_xxhash64(gset::Graphset, labels)
+    return __xxhash64seed(labels, __xxhash64(collect(active_words(gset))))
+end
+
+__xxhash128(x::AbstractArray) = @ccall xxHash_jll.libxxhash.XXH3_128bits(Ref(x, 1)::Ptr{Cvoid}, sizeof(x)::Csize_t)::UInt128
+__xxhash128seed(x::AbstractArray, seed::UInt128) = @ccall xxHash_jll.libxxhash.XXH3_128bits_withSeed(Ref(x, 1)::Ptr{Cvoid}, sizeof(x)::Csize_t, seed::UInt128)::UInt128
+function _ghash_xxhash128(gset::Graphset, labels)
+    return __xxhash128seed(labels, __xxhash128(collect(active_words(gset))))
+end
+
+# as suggested by stevengj here: https://discourse.julialang.org/t/hash-collision-with-small-vectors/131702/10
+function __SHAhash(x)
+    io = IOBuffer()
+    Serialization.serialize(io, x)
+    return SHA.sha256(take!(io))
+end
+__SHAhash64(x) = reinterpret(UInt64, __SHAhash(x))[1]
+function _ghash_SHA64(gset::Graphset, labels)
+    return __SHAhash64((labels, collect(active_words(gset))))
+end
+__SHAhash128(x) = reinterpret(UInt128, __SHAhash(x))[1]
+function _ghash_SHA128(gset::Graphset, labels)
+    return __SHAhash128((labels, collect(active_words(gset))))
+end
+
+function _ghash(gset, labels; alg::AbstractHashAlg)
+    if alg isa XXHash64Alg
+        # We need to allocate any views before we pass them to xxHash
+        if labels isa SubArray
+            h = _ghash_xxhash64(gset, collect(labels))
+        else
+            h = _ghash_xxhash64(gset, labels)
+        end
+    elseif alg isa XXHash128Alg
+        if labels isa SubArray
+            h = _ghash_xxhash128(gset, collect(labels))
+        else
+            h = _ghash_xxhash128(gset, labels)
+        end
+    elseif alg isa SHA64Alg
+        h = _ghash_SHA64(gset, labels)
+    elseif alg isa SHA128Alg
+        h = _ghash_SHA128(gset, labels)
+    elseif alg isa Base64Alg
+        h = _ghash_base64(gset, labels)
+    else
+        throw(ArgumentError("$alg is not a valid hashing algorithm."))
+    end
+    return h
+end

src/nauty.jl

@@ -103,31 +103,14 @@ end
         canong.words::Ref{W})::Cvoid end
 end
 
-function _sethash!(g::DenseNautyGraph, canong::Graphset, canonperm)
-    # Base.hash skips elements in arrays of length >= 8192
-    # Use SHA in these cases
-    canong_hash = length(canong) >= 8192 ? hash_sha(canong) : hash(canong)
-    labels_hash = @views length(g.labels) >= 8192 ? hash_sha(g.labels[canonperm]) : hash(g.labels[canonperm])
-
-    hashval = hash(labels_hash, canong_hash)
-    g.hashval = hashval
-    return
-end
-function _canonize!(g::DenseNautyGraph, canong::Graphset, canonperm)
-    copy!(g.graphset, canong)
-    permute!(g.labels, canonperm)
-    return
-end
-
-
 """
-    nauty(g::AbstractNautyGraph, [options::NautyOptions]; [canonize=false])
+    nauty(g::AbstractNautyGraph, [options::NautyOptions; canonize=false])
 
 Compute a graph `g`'s canonical form and automorphism group.
 """
 function nauty(::AbstractNautyGraph, ::NautyOptions; kwargs...) end
 
-function nauty(g::DenseNautyGraph, options::NautyOptions=default_options(g); canonize=false, compute_hash=true)
+function nauty(g::DenseNautyGraph, options::NautyOptions=default_options(g); canonize=false)
     if is_directed(g) && !isone(options.digraph)
         error("Nauty options need to match the directedness of the input graph. Make sure to instantiate options with `digraph=true` if the input graph is directed.")
     end
@@ -140,8 +123,7 @@ function nauty(g::DenseNautyGraph, options::NautyOptions=default_options(g); can
     # generators = Vector{Cint}[] # TODO: extract generators from nauty call
     autg = AutomorphismGroup(statistics.grpsize1 * 10^statistics.grpsize2, orbits)
 
-    compute_hash && _sethash!(g, canong, canonperm)
-    canonize && _canonize!(g, canong, canonperm)
+    canonize && _copycanon!(g, canong, canonperm)
     return canonperm, autg
 end
 
@@ -154,10 +136,15 @@ function canonize!(::AbstractNautyGraph) end
 
 function canonize!(g::DenseNautyGraph)
     canong, canonperm, _ = _densenauty(g)
-    _sethash!(g, canong, canonperm)
-    _canonize!(g, canong, canonperm)
+    _copycanon!(g, canong, canonperm)
     return canonperm
 end
+function _copycanon!(g, canong, canonperm)
+    copy!(g.graphset, canong)
+    permute!(g.labels, canonperm)
+    g.iscanon = true
+    return
+end
 
 """
     canonical_permutation(g::AbstractNautyGraph)
@@ -179,27 +166,41 @@ Check whether two graphs `g` and `h` are isomorphic to each other by comparing t
 function is_isomorphic(::AbstractNautyGraph, ::AbstractNautyGraph) end
 
 function is_isomorphic(g::DenseNautyGraph, h::DenseNautyGraph)
+    iscanon(g) && iscanon(h) && return g == h
     canong, permg, _ = _densenauty(g)
     canonh, permh, _ = _densenauty(h)
     return canong == canonh && view(g.labels, permg) == view(h.labels, permh)
 end
 ≃(g::AbstractNautyGraph, h::AbstractNautyGraph) = is_isomorphic(g, h)
 
-
 """
-    ghash(g::AbstractNautyGraph)
-
-Hash the canonical version of `g`, so that (up to hash collisions) `ghash(g1) == ghash(g2)` implies `is_isomorphic(g1, g2) == true`.
-Hashes are computed using `Base.hash` for small graphs (nv < 8192), or using the first 64 bits of `sha256` for larger graphs.
+    ghash(g::AbstractNautyGraph; [alg=XXHash64Alg()])
+
+Compute a hash of the canonical version of `g`, meaning that `is_isomorphic(g1, g2) == true` implies `ghash(g1) == ghash(g2)`. The converse usually holds as well, 
+but in rare cases, hash collisions may cause non-isomorphic graphs to have the same hash. The likelihood of a hash collision occuring depends on the 
+chosen hashing algorithm, which can be specified via the `alg` keyword. Valid algorithm choices are:
+
+- `XXHash64Alg()`: The 64bit version of the xxHash algorithm (`XXH3_64bits`). Fast and resistant against collisions, but not cryptographically secure. 
+See (xxHash)[https://xxhash.com] for more details on collision resistance. This is the default option.
+- `XXHash128Alg()`: The 128bit version of the xxHash algorithm (`XXH3_128bits`). Slightly slower than the 64bit xxHash, resistant against collisions, but not cryptographically secure.
+See the (xxHash)[https://xxhash.com] for more details on collision resistance.
+- `SHA64Alg()`: The first 64bits of the SHA256 hash. Slow but cryptographically secure. Consider using `SHA128Alg()` instead, since the 128bit version runs at the same speed.
+- `SHA128Alg()`: The first 128bits of the SHA256 hash. Slow but cryptographically secure.
+- `Base64Alg()`: The built-in Julia hash function `Base.hash`. Fast, but not secure against collisions, so __use with caution__! 
+It is strongly recommended to use `XXHash64Alg()` instead. Cannot hash graphs with more than `√8192 ≈ 90` vertices.
+
+!!! warning "Warning"
+__Using different hashing algorithms will result in different hash values__. Before you compare different graph hashes, you have to 
+ensure that the hashes were computed with the same algorithm, or you will get meaningless results.
 """
-function ghash(::AbstractNautyGraph) end
-
-function ghash(g::DenseNautyGraph)
-    if !isnothing(g.hashval)
-        return g.hashval
+function ghash(::AbstractNautyGraph; alg=XXHash64Alg()::AbstractHashAlg) end
+
+function ghash(g::DenseNautyGraph; alg=XXHash64Alg()::AbstractHashAlg)
+    if iscanon(g)
+        return _ghash(g.graphset, g.labels; alg)
+    else
+        canong, canonperm, _ = _densenauty(g)
+        return _ghash(canong, @view g.labels[canonperm]; alg)
     end
-
-    canong, canonperm, _ = _densenauty(g)
-    _sethash!(g, canong, canonperm)
-    return g.hashval
+    return h
 end

src/utils.jl

@@ -13,20 +13,4 @@ function vertexlabels2labptn!(lab::Vector{<:Integer}, ptn::Vector{<:Integer}, la
         ptn[i] = ifelse(labels[lab[i+1]+1] == labels[lab[i]+1], 1, 0)
     end
     return lab, ptn
-end
-
-function hash_sha(x)
-    io = IOBuffer()
-    write(io, x)
-    return concatbytes(@view sha256(take!(io))[1:8])
-end
-
-function concatbytes(W::Type{<:Unsigned}, bytes)
-    w = zero(W)
-    for b in bytes
-        w |= b
-        w <<= 8
-    end
-    return w
-end
-concatbytes(bytes) = concatbytes(UInt64, bytes)
+end

test/densenautygraph.jl

@@ -173,8 +173,7 @@ symmetrize_adjmx(A) = (A = convert(typeof(A), (A + A') .> 0); for i in axes(A, 1
     @test h.ne == g.ne
     @test h.graphset.m == g.graphset.m
     @test h.labels == g.labels
-    @test h.hashval == g.hashval
-
+    @test h.iscanon == g.iscanon
 
     glab = NautyGraph(5; vertex_labels=1:5)
     add_edge!(glab, 1, 2)