Update state serialization (#962)

* Update state serialization

* Use `Scratch` instead of `MutableArtifacts.toml`

* Add tests to ensure that the state is serialized faithfully

* Update wizard.jl

* Only compare serializeable fields in a WizardState

* Call `Wizard.load_wizard_state` non-interactively in tests

* Allow JLD2 v0.2

Co-authored-by: Mosè Giordano <[email protected]>

Author: staticfloat

Manifest.toml

@@ -28,7 +28,7 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BinaryBuilderBase]]
 deps = ["CodecZlib", "Downloads", "InteractiveUtils", "JSON", "LibGit2", "Libdl", "Logging", "OutputCollectors", "Pkg", "Random", "SHA", "Scratch", "SimpleBufferStream", "TOML", "Tar", "UUIDs", "p7zip_jll", "pigz_jll"]
-git-tree-sha1 = "ba037f06ff8be240fcbd33c4ec4a397ebb9a9329"
+git-tree-sha1 = "0642b01b5f9a13170beb12f1dc5f35153029b104"
 repo-rev = "master"
 repo-url = "https://github.com/JuliaPackaging/BinaryBuilderBase.jl.git"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
@@ -40,16 +40,22 @@ git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
 uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
 version = "0.7.0"
 
+[[Compat]]
+deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
+git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "3.25.0"
+
 [[DataAPI]]
-git-tree-sha1 = "6d64b28d291cb94a0d84e6e41081fb081e7f717f"
+git-tree-sha1 = "8ab70b4de35bb3b8cc19654f6b893cf5164f8ee8"
 uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
-version = "1.5.0"
+version = "1.5.1"
 
 [[DataStructures]]
-deps = ["InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "88d48e133e6d3dd68183309877eac74393daa7eb"
+deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
+git-tree-sha1 = "4437b64df1e0adccc3e5d1adbc3ac741095e4677"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.17.20"
+version = "0.18.9"
 
 [[DataValueInterfaces]]
 git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
@@ -60,6 +66,10 @@ version = "1.0.0"
 deps = ["Printf"]
 uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 
+[[DelimitedFiles]]
+deps = ["Mmap"]
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+
 [[Distributed]]
 deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
@@ -68,12 +78,6 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 deps = ["ArgTools", "LibCURL", "NetworkOptions"]
 uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 
-[[FileIO]]
-deps = ["Pkg"]
-git-tree-sha1 = "fee8955b9dfa7bec67117ef48085fb2b559b9c22"
-uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
-version = "1.4.5"
-
 [[FileWatching]]
 uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
 
@@ -117,10 +121,10 @@ uuid = "82899510-4779-5014-852e-03e436cf321d"
 version = "1.0.0"
 
 [[JLD2]]
-deps = ["CodecZlib", "DataStructures", "FileIO", "Mmap", "Pkg", "Printf", "UUIDs"]
-git-tree-sha1 = "9353b717ee4e27beab4e902c92a06bb5f160d2cf"
+deps = ["CodecZlib", "DataStructures", "MacroTools", "Mmap", "Pkg", "Printf", "Requires", "UUIDs"]
+git-tree-sha1 = "1b8168c14939e43c7c4d2c9e8f0ddd8718965430"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.1.14"
+version = "0.2.4"
 
 [[JLLWrappers]]
 git-tree-sha1 = "a431f5f2ca3f4feef3bd7a5e94b8b8d4f2f647a0"
@@ -167,9 +171,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[LoggingExtras]]
 deps = ["Dates"]
-git-tree-sha1 = "70518b2cce1fea30c5e9dd0c44407a90f394de47"
+git-tree-sha1 = "95e675246b58a7cb1199a90cd50a25850ff215c7"
 uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
-version = "0.4.4"
+version = "0.4.5"
 
 [[MacroTools]]
 deps = ["Markdown", "Random"]
@@ -286,6 +290,12 @@ git-tree-sha1 = "60522125b10a3b71b07677302c3aedd0b4363835"
 uuid = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4"
 version = "1.5.3"
 
+[[Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "cfbac6c1ed70c002ec6361e7fd334f02820d6419"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.1.2"
+
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
@@ -298,6 +308,10 @@ version = "1.0.3"
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
+[[SharedArrays]]
+deps = ["Distributed", "Mmap", "Random", "Serialization"]
+uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
+
 [[SimpleBufferStream]]
 git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1"
 uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7"
@@ -312,6 +326,14 @@ git-tree-sha1 = "80cef67d2953e33935b41c6ab0a178b9987b1c99"
 uuid = "2133526b-2bfb-4018-ac12-889fb3908a75"
 version = "0.1.1"
 
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
 [[StructIO]]
 deps = ["Test"]
 git-tree-sha1 = "010dc73c7146869c042b49adcdb6bf528c12e859"
@@ -330,9 +352,9 @@ version = "1.0.0"
 
 [[Tables]]
 deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"]
-git-tree-sha1 = "8dc2bb7d3548e315d890706547b24502ed79504f"
+git-tree-sha1 = "a716dde43d57fa537a19058d044b495301ba6565"
 uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
-version = "1.3.1"
+version = "1.3.2"
 
 [[Tar]]
 deps = ["ArgTools", "SHA"]

Project.toml

@@ -26,6 +26,7 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Registrator = "4418983a-e44d-11e8-3aec-9789530b3b3e"
 RegistryTools = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 ghr_jll = "07c12ed4-43bc-5495-8a2a-d5838ef8d533"
@@ -35,7 +36,7 @@ ArgParse = "1.1"
 BinaryBuilderBase = "0.5"
 GitHub = "5.1"
 HTTP = "0.8"
-JLD2 = "0.1.6"
+JLD2 = "0.1.6, 0.2"
 JLLWrappers = "1.2.0"
 JSON = "0.21"
 LoggingExtras = "0.4"

src/Wizard.jl

@@ -15,6 +15,7 @@ import PkgLicenses
 using JLD2
 using Base.BinaryPlatforms
 using Dates
+using Scratch
 
 # It's Magic (TM)!
 export run_wizard, deploy
@@ -27,17 +28,16 @@ include("wizard/obtain_source.jl")
 include("wizard/interactive_build.jl")
 include("wizard/deploy.jl")
 
-function save_last_wizard_state(state::WizardState)
-    BinaryBuilderBase.create_and_bind_mutable_artifact!("wizard_state") do dir
-        jldopen(joinpath(dir, "wizard.state"), "w") do f
-            serialize(f, state)
-        end
+function save_wizard_state(state::WizardState,
+                           wizard_state_dir::String = @get_scratch!("wizard_state"))
+    jldopen(joinpath(wizard_state_dir, "wizard.state"), "w") do f
+        serialize(f, state)
     end
     return state
 end
 
-function load_last_wizard_state(; as_is::Bool = false)
-    wizard_state_dir = BinaryBuilderBase.get_mutable_artifact_path("wizard_state")
+function load_wizard_state(wizard_state_dir::String = @get_scratch!("wizard_state");
+                           as_is::Bool = false)
 
     # If no state dir exists, early-exit
     if wizard_state_dir === nothing
@@ -89,7 +89,7 @@ function run_wizard(state::Union{Nothing,WizardState} = nothing)
     if state === nothing
         # If we weren't given a state, check to see if we'd like to resume a
         # previous run or start from scratch again.
-        state = load_last_wizard_state()
+        state = load_wizard_state()
     end
 
     try
@@ -119,11 +119,11 @@ function run_wizard(state::Union{Nothing,WizardState} = nothing)
             end
 
             # Save it every step along the way
-            save_last_wizard_state(state)
+            save_wizard_state(state)
         end
     catch err
         # If anything goes wrong, immediately save the current wizard state
-        save_last_wizard_state(state)
+        save_wizard_state(state)
         if isa(err, InterruptException)
             msg = "\n\nWizard stopped, use BinaryBuilder.run_wizard() to resume"
             if state.step in (:step3, :step3_retry, :step5a, :step5b, :step5c, :step6)

src/wizard/state.jl

@@ -93,6 +93,16 @@ function unserialize(io)
     return x
 end
 
+# Compare only serializeable fields when checking ==
+function Base.:(==)(x::WizardState, y::WizardState)
+    for field in serializeable_fields(x)
+        if getproperty(x, field) != getproperty(y, field)
+            return false
+        end
+    end
+    return true
+end
+
 function show(io::IO, x::WizardState)
     print(io, "WizardState [$(x.step)]")
 end

test/wizard.jl

@@ -414,6 +414,18 @@ end
     end
 end
 
+@testset "Wizard - state serialization" begin
+    for state_generator in (Wizard.WizardState, step2_state, step3_state, step7_state)
+        mktempdir() do dir
+            state = state_generator()
+
+            Wizard.save_wizard_state(state, dir)
+            @test Wizard.load_wizard_state(dir; as_is=true) == state
+        end
+    end
+end
+
+
 @testset "GitHub - authentication" begin
     withenv("GITHUB_TOKEN" => "") do
         @test Wizard.github_auth(allow_anonymous=true) isa GitHub.AnonymousAuth