[test] Replace Pkg.PlatformEngine --> Custom download & verify

Project.toml

@@ -23,9 +23,11 @@ julia = "1.3"
 
 [extras]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Dates", "Pkg", "Statistics", "Test"]
+test = ["Dates", "Downloads", "Pkg", "Statistics", "SHA", "Test"]

test/remotefiles.jl

@@ -1,10 +1,5 @@
 # this file downloads files which are used during testing the package
 # if they are already present and their checksum matches, they are not downloaded again
-
-using Pkg.PlatformEngines
-
-probe_platform_engines!()  # for download
-
 const testdatadir = @__DIR__
 
 REPO_URL = "https://github.com/yeesian/ArchGDALDatasets/blob/master/"
@@ -45,12 +40,66 @@ remotefiles = [
     ("ospy/data5/doq2.rrd", "8274dad00b27e008e5ada62afb1025b0e6e2ef2d2ff2642487ecaee64befd914"),
 ]
 
+function verify(path::AbstractString, hash::AbstractString)
+    @assert occursin(r"^[0-9a-f]{64}$", hash)
+    hash = lowercase(hash)
+    if isfile(path)
+        calc_hash = open(path) do file
+            bytes2hex(sha256(file))
+        end
+        @assert occursin(r"^[0-9a-f]{64}$", calc_hash)
+        if calc_hash != hash
+            @error "Hash Mismatch! Expected: $hash, Calculated: $calc_hash\n"
+            return false
+        else
+            return true
+        end
+    else
+        error("File read error: $path")
+    end
+end
+
+function download_verify(url::AbstractString, hash::Union{AbstractString, Nothing}, dest::AbstractString)
+    file_existed = false
+    # verify if file exists
+    if isfile(dest)
+        file_existed = true
+        if hash !== nothing && verify(dest, hash)
+            # hash verified
+            return true
+        else
+            # either hash is nothing or couldn't pass the SHA test
+            @error("Failed to verify file: $dest with hash: $hash. Re-downloading file...")
+        end
+    end
+    # if the file exists but some problem exists, we delete it to start from scratch
+    file_existed && Base.rm(dest; force=true)
+    # Make sure the containing folder exists
+    mkpath(dirname(dest))
+    # downloads the file at dest
+    Downloads.download(url, dest)
+    # hash exists and verification fails
+    if hash !== nothing && !verify(dest, hash)
+        if file_existed
+            # the file might be corrupted so we start from scracth
+            Base.rm(dest; force=true)
+            Downloads.download(url, dest)
+            if hash !== nothing && !verify(dest, hash)
+                error("Verification failed")
+            end
+        else
+            error("Verification failed. File not created after download.")
+        end
+    end
+    return !file_existed
+end
+
 for (f, sha) in remotefiles
     # create the directories if they don't exist
     currdir = dirname(f)
     isdir(currdir) || mkpath(currdir)
     # download the file if it is not there or if it has a different checksum
     currfile = normpath(joinpath(testdatadir, f))
     url = REPO_URL * f * "?raw=true"
-    PlatformEngines.download_verify(url, sha, currfile; force=true)
+    download_verify(url, sha, currfile)
 end

test/runtests.jl

@@ -1,6 +1,7 @@
 using Test
 using Dates
-using Pkg.PlatformEngines
+using Downloads
+using SHA
 
 # ensure all testing files are present
 include("remotefiles.jl")