Add Mittelmann and clean decompression

Author: gdalle

Project.toml

@@ -12,11 +12,13 @@ DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
 GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 QPSReader = "10f199a5-22af-520b-b891-7ce84a7b1bd0"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 
 [compat]
 CodecBzip2 = "0.8.5"
 DataDeps = "0.7.13"
 GZip = "0.6.2"
 Logging = "1.10.0"
 QPSReader = "0.2.1"
+Scratch = "1.3.0"
 julia = "1.10"

README.md

@@ -2,21 +2,26 @@
 
 A Julia package for automatic download and parsing of linear, quadratic and integer programming instances.
 
-## Principle
-
 Supported datasets:
 
 - [x] [Netlib](https://www.netlib.org/lp/data/index.html)
 - [x] [MIPLIB 2017](https://miplib.zib.de/index.html)
-- [ ] [Mittelmann benchmark](https://plato.asu.edu/ftp/lptestset/)
-
-The source files are downloaded automatically thanks to [DataDeps.jl](https://github.com/oxinabox/DataDeps.jl).
-The returned problem format is `QPSData` from [QPSReader.jl](https://github.com/JuliaSmoothOptimizers/QPSReader.jl).
+- [x] [Mittelmann LP benchmark](https://plato.asu.edu/ftp/lptestset/)
 
 ## Getting started
 
-To see which instances are available, call `list_instances(dataset, name)`.
+1. To see which instances are available, call `list_instances(dataset, name)`.
+2. To read a specific instance, call `read_instance(dataset, name)`.
+3. The returned problem format is `QPSData` from [QPSReader.jl](https://github.com/JuliaSmoothOptimizers/QPSReader.jl).
+
+More details are available in the docstrings.
+
+## Tips
 
-To read a specific instance, call `read_instance(dataset, name)`.
+The problem source files are downloaded automatically thanks to [DataDeps.jl](https://github.com/oxinabox/DataDeps.jl).
+Note that each download has to be validated manually from the REPL.
+This doesn't work well when the triggering line of code is executed with VSCode's Julia extension, better run it in the REPL directly.
+An alternative is to set `ENV["DATADEPS_ALWAYS_ACCEPT"] = true`.
 
-More details are available in the docstrings of these functions.
+The decompressed instances can be rather large (over 80 GB for the complete MIPLIB 2017 collection).
+If you need to clean up some space, you can delete unneeded files inside the folder located at `MathProgBenchmarks.MPS_SCRATCH`.

src/MathProgBenchmarks.jl

@@ -5,6 +5,7 @@ using DataDeps
 import GZip
 using Logging
 using QPSReader
+using Scratch
 
 include("datasets.jl")
 include("instances.jl")

src/init.jl

@@ -1,11 +1,16 @@
+MPS_SCRATCH = ""
+
 function __init__()
+    global MPS_SCRATCH = @get_scratch!("mps_scratch")
     # MIPLIB 2017
     register(
         DataDep(
             "miplib2017-collection",
             """
-            All instances in the MIPLIB 2017 collection set (size: 3.5 GB).
-            Source: https://miplib.zib.de/index.html.""",
+            All instances in the MIPLIB 2017 collection set.
+            Source: https://miplib.zib.de/index.html.
+            Compressed download size: 3.5 GB
+            Decompressed size: ?? GB""",
             "https://miplib.zib.de/downloads/collection.zip",
             post_fetch_method = unpack,
         ),
@@ -14,11 +19,13 @@ function __init__()
         DataDep(
             "miplib2017-benchmark",
             """
-            All instances in the MIPLIB 2017 benchmark set (size: 317 MB).
-            Source: https://miplib.zib.de/index.html.""",
+            All instances in the MIPLIB 2017 benchmark set.
+            Source: https://miplib.zib.de/index.html.
+            Compressed download size: 317 MB
+            Decompressed size: 639 MB""",
             "https://miplib.zib.de/downloads/benchmark.zip",
+            "c756eefd544d83b31809306b45d3549a1a5b9378e6aa78b68738b1a3b6a418fa",
             post_fetch_method = unpack,
-            "c756eefd544d83b31809306b45d3549a1a5b9378e6aa78b68738b1a3b6a418fa"
         ),
     )
     register(
@@ -42,26 +49,19 @@ function __init__()
         ),
     )
     # Mittelmann
-    for (folder, names) in pairs(MITTELMANN_LP_INSTANCES)
-        for name in names
-            @info "$folder / $name"
-            datadep_name = "mittelmann-lp-$name"
-            # TODO: some paths are just .bz2, following no visible pattern
-            if isnothing(folder)
-                datadep_msg = """
-                Instance $name from the Mittelmann LP benchmark set.
-                Source: https://plato.asu.edu/ftp/lptestset/."""
-                datadep_url = "https://plato.asu.edu/ftp/lptestset/$name.mps.bz2"
-            else
-                datadep_msg = """
-                Instance $name from the Mittelmann LP benchmark set (folder $folder).
-                Source: https://plato.asu.edu/ftp/lptestset/."""
-                datadep_url = "https://plato.asu.edu/ftp/lptestset/$folder/$name.mps.bz2"
-            end
-            register(
-                DataDep(datadep_name, datadep_msg, datadep_url)
-            )
-        end
-    end
+    register(
+        DataDep(
+            "mittelmann-lp",
+            """
+            All instances in the Mittelmann LP benchmark set.
+            Source: https://plato.asu.edu/ftp/lptestset/.
+            Compressed download size: ??
+            Decompressed size: 2.21 GB""",
+            map(
+                path -> "https://plato.asu.edu/ftp/lptestset/$path.bz2",
+                MITTELMANN_LP_INSTANCES
+            ),
+        )
+    )
     return nothing
 end

src/instances.jl

@@ -1,7 +1,9 @@
 """
     read_instance(dataset::Dataset, name::String)
 
-Read the instance identified by `name` from a given `dataset` and return a `QPSReader.QPSData` object.
+Read the instance identified by `name` from a given `dataset`.
+
+Return a tuple `(data, path)` where `data isa QPSReader.QPSData` is an object created by [QPSReader.jl](https://github.com/JuliaSmoothOptimizers/QPSReader.jl) and `path` is the path to the MPS file.
 
 # See also
 
@@ -25,7 +27,7 @@ function read_miplib2017_instance(name::String)
     end
     name = lowercase(name)
     mps_gz_path = joinpath(folder, "$name.mps.gz")
-    return read_mps(mps_gz_path)
+    return read_mps(mps_gz_path; scratch_subfolder = "miplib2017")
 end
 
 function read_netlib_instance(name::String)
@@ -38,35 +40,41 @@ function read_netlib_instance(name::String)
     end
     netlib_path = fetch_netlib()
     sif_path = joinpath(netlib_path, "$name.SIF")
-    return read_mps(sif_path; mpsformat)
+    return read_mps(sif_path; scratch_subfolder = "netlib", mpsformat)
 end
 
 function read_mittelmann_lp_instance(name::String)
-    folder_path = @datadep_str("mittelmann-lp-$name")
-    path1 = joinpath(folder_path, "$name.mps.bz2")
-    path2 = joinpath(folder_path, "$name.bz2")
-    if ispath(path1)
-        return read_mps(path1)
+    folder_path = @datadep_str("mittelmann-lp")
+    mps_bz2_path1 = joinpath(folder_path, "$name.mps.bz2")
+    mps_bz2_path2 = joinpath(folder_path, "$name.bz2")
+    if ispath(mps_bz2_path1)
+        return read_mps(mps_bz2_path1; scratch_subfolder = "mittelman-lp")
     else
-        return read_mps(path2)
+        return read_mps(mps_bz2_path2; scratch_subfolder = "mittelman-lp")
     end
 end
 
-function read_mps(path::String; mpsformat::Symbol = :free)
-    if endswith(path, ".mps.gz")
-        contents = GZip.open(path, "r") do f
-            read(f, String)
-        end
-        mps_path = string(tempname(), ".mps")
-        open(mps_path, "w") do f
-            write(f, contents)
-        end
-    elseif endswith(path, ".bz2") || endswith(path, ".mps.bz2")
-        compressed = CodecBzip2.read(path)
-        contents = String(CodecBzip2.transcode(CodecBzip2.Bzip2Decompressor, compressed))
-        mps_path = string(tempname(), ".mps")
-        open(mps_path, "w") do f
-            write(f, contents)
+function read_mps(path::String; scratch_subfolder::String, mpsformat::Symbol = :free)
+    name = splitext(splitext(splitpath(path)[end])[1])[1]
+    if !isdir(joinpath(MPS_SCRATCH, scratch_subfolder))
+        mkdir(joinpath(MPS_SCRATCH, scratch_subfolder))
+    end
+    if endswith(path, ".gz") || endswith(path, ".bz2")
+        mps_path = joinpath(MPS_SCRATCH, scratch_subfolder, "$name.mps")
+        if !ispath(mps_path)
+            if endswith(path, ".mps.gz")
+                contents = GZip.open(path, "r") do f
+                    read(f, String)
+                end
+            elseif endswith(path, ".mps.bz2")
+                compressed = CodecBzip2.read(path)
+                contents = String(CodecBzip2.transcode(CodecBzip2.Bzip2Decompressor, compressed))
+            elseif endswith(path, ".bz2")
+                throw(ArgumentError("File at $path not supported"))
+            end
+            open(mps_path, "w") do f
+                write(f, contents)
+            end
         end
     else
         @assert endswith(path, ".mps") || endswith(path, ".SIF")
@@ -76,5 +84,5 @@ function read_mps(path::String; mpsformat::Symbol = :free)
     qps_data = with_logger(NullLogger()) do
         readqps(mps_path; mpsformat)
     end
-    return qps_data
+    return qps_data, mps_path
 end

src/lists.jl

@@ -36,5 +36,6 @@ function list_netlib_instances()
 end
 
 function list_mittelmann_lp_instances()
-    return reduce(vcat, values(MITTELMANN_LP_INSTANCES))
+    all_names = reduce(vcat, MITTELMANN_LP_INSTANCES)
+    return map(n -> string(chopsuffix(split(n, "/")[end], ".mps")), all_names)
 end