Interface overhaul

Project.toml

@@ -5,6 +5,7 @@ version = "0.2.4"
 
 [deps]
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
+Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
 ConstrainedShortestPaths = "b3798467-87dc-4d99-943d-35a1bd39e395"
 DataDeps = "124859b0-ceae-595e-8997-d05f6a7a8dfe"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
@@ -13,7 +14,10 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
 Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
+InferOpt = "4846b161-c94e-4150-8dac-c7ae193c601f"
 Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
+IterTools = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -31,6 +35,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [compat]
 Colors = "0.13.1"
+Combinatorics = "1.0.3"
 ConstrainedShortestPaths = "0.6.0"
 DataDeps = "0.7"
 Distributions = "0.25"
@@ -39,7 +44,10 @@ Flux = "0.14, 0.15, 0.16"
 Graphs = "1.11"
 HiGHS = "1.9"
 Images = "0.26.1"
+InferOpt = "0.7.0"
 Ipopt = "1.6"
+IterTools = "1.10.0"
+JSON = "0.21.4"
 JuMP = "1.22"
 LaTeXStrings = "1.4.0"
 LinearAlgebra = "1"

docs/make.jl

@@ -12,8 +12,8 @@ api_dir = joinpath(@__DIR__, "src", "api")
 api_files = map(x -> joinpath("api", x), readdir(api_dir))
 tutorial_files = readdir(tutorial_dir)
 md_tutorial_files = [split(file, ".")[1] * ".md" for file in tutorial_files]
-benchmark_files = readdir(benchmarks_dir)
-md_benchmark_files = [split(file, ".")[1] * ".md" for file in benchmark_files]
+benchmark_files = [joinpath("benchmarks", e) for e in readdir(benchmarks_dir)]
+# md_benchmark_files = [split(file, ".")[1] * ".md" for file in benchmark_files]
 
 include_tutorial = true
 
@@ -25,20 +25,14 @@ if include_tutorial
 end
 
 makedocs(;
-    modules=[DecisionFocusedLearningBenchmarks, DecisionFocusedLearningBenchmarks.Warcraft],
+    modules=[DecisionFocusedLearningBenchmarks],
     authors="Members of JuliaDecisionFocusedLearning",
     sitename="DecisionFocusedLearningBenchmarks.jl",
-    format=Documenter.HTML(),
+    format=Documenter.HTML(; size_threshold=typemax(Int)),
     pages=[
         "Home" => "index.md",
         "Tutorials" => include_tutorial ? md_tutorial_files : [],
-        "Benchmark problems list" => [
-            "benchmarks/subset_selection.md",
-            "benchmarks/fixed_size_shortest_path.md",
-            "benchmarks/warcraft.md",
-            "benchmarks/portfolio_optimization.md",
-            "benchmarks/vsp.md",
-        ],
+        "Benchmark problems list" => benchmark_files,
         "API reference" => api_files,
     ],
 )

docs/src/api/0_interface.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Interface
 
 ## Public

docs/src/api/argmax.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Argmax
 
 ## Public

docs/src/api/argmax_2d.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Argmax2D
 
 ## Public

docs/src/api/dvsp.md

@@ -0,0 +1,19 @@
+```@meta
+CollapsedDocStrings = true
+```
+
+# Dynamic Vehicle Scheduling
+
+## Public
+
+```@autodocs
+Modules = [DecisionFocusedLearningBenchmarks.DynamicVehicleScheduling]
+Private = false
+```
+
+## Private
+
+```@autodocs
+Modules = [DecisionFocusedLearningBenchmarks.DynamicVehicleScheduling]
+Public = false
+```

docs/src/api/dynamic_assorment.md

@@ -0,0 +1,19 @@
+```@meta
+CollapsedDocStrings = true
+```
+
+# Dynamic Assortment
+
+## Public
+
+```@autodocs
+Modules = [DecisionFocusedLearningBenchmarks.DynamicAssortment]
+Private = false
+```
+
+## Private
+
+```@autodocs
+Modules = [DecisionFocusedLearningBenchmarks.DynamicAssortment]
+Public = false
+```

docs/src/api/fixed_shortest_path.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Fixed-size shortest path
 
 ## Public

docs/src/api/portfolio_optimization.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Subset selection
 
 ## Public

docs/src/api/ranking.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Ranking
 
 ## Public

docs/src/api/subset_selection.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Subset selection
 
 ## Public

docs/src/api/vsp.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Stochastic Vehicle Scheduling
 
 ## Public

docs/src/api/warcraft.md

@@ -1,3 +1,7 @@
+```@meta
+CollapsedDocStrings = true
+```
+
 # Warcraft
 
 ## Public

docs/src/benchmarks/argmax.md

@@ -0,0 +1 @@
+# Argmax

docs/src/benchmarks/dvsp.md

@@ -0,0 +1,3 @@
+# Dynamic Vehicle Scheduling
+
+[`DynamicVehicleSchedulingBenchmark`](@ref).

docs/src/benchmarks/dynamic_assorment.md

@@ -0,0 +1,3 @@
+# Dynamic Assortment
+
+[`DynamicAssortmentBenchmark`](@ref).

docs/src/benchmarks/ranking.md

@@ -0,0 +1 @@
+# Ranking

docs/src/tutorials/warcraft.jl

@@ -86,3 +86,6 @@ final_gap = compute_gap(b, test_dataset, model, maximizer)
 θ = model(x)
 y = maximizer(θ)
 plot_data(b, DataSample(; x, θ_true=θ, y_true=y))
+
+using Test #src
+@test final_gap < starting_gap #src

docs/src/warcraft.md

@@ -0,0 +1,155 @@
+```@meta
+EditURL = "tutorials/warcraft.jl"
+```
+
+# Path-finding on image maps
+
+In this tutorial, we showcase DecisionFocusedLearningBenchmarks.jl capabilities on one of its main benchmarks: the Warcraft benchmark.
+This benchmark problem is a simple path-finding problem where the goal is to find the shortest path between the top left and bottom right corners of a given image map.
+The map is represented as a 2D image representing a 12x12 grid, each cell having an unknown travel cost depending on the terrain type.
+
+First, let's load the package and create a benchmark object as follows:
+
+````@example warcraft
+using DecisionFocusedLearningBenchmarks
+b = WarcraftBenchmark()
+````
+
+## Dataset generation
+
+These benchmark objects behave as generators that can generate various needed elements in order to build an algorithm to tackle the problem.
+First of all, all benchmarks are capable of generating datasets as needed, using the [`generate_dataset`](@ref) method.
+This method takes as input the benchmark object for which the dataset is to be generated, and a second argument specifying the number of samples to generate:
+
+````@example warcraft
+dataset = generate_dataset(b, 50);
+nothing #hide
+````
+
+We obtain a vector of [`DataSample`](@ref) objects, containing all needed data for the problem.
+Subdatasets can be created through regular slicing:
+
+````@example warcraft
+train_dataset, test_dataset = dataset[1:45], dataset[46:50]
+````
+
+And getting an individual sample will return a [`DataSample`](@ref) with four fields: `x`, `instance`, `θ`, and `y`:
+
+````@example warcraft
+sample = test_dataset[1]
+````
+
+`x` correspond to the input features, i.e. the input image (3D array) in the Warcraft benchmark case:
+
+````@example warcraft
+x = sample.x
+````
+
+`θ_true` correspond to the true unknown terrain weights. We use the opposite of the true weights in order to formulate the optimization problem as a maximization problem:
+
+````@example warcraft
+θ_true = sample.θ_true
+````
+
+`y_true` correspond to the optimal shortest path, encoded as a binary matrix:
+
+````@example warcraft
+y_true = sample.y_true
+````
+
+`instance` is not used in this benchmark, therefore set to nothing:
+
+````@example warcraft
+isnothing(sample.instance)
+````
+
+For some benchmarks, we provide the following plotting method [`plot_data`](@ref) to visualize the data:
+
+````@example warcraft
+plot_data(b, sample)
+````
+
+We can see here the terrain image, the true terrain weights, and the true shortest path avoiding the high cost cells.
+
+## Building a pipeline
+
+DecisionFocusedLearningBenchmarks also provides methods to build an hybrid machine learning and combinatorial optimization pipeline for the benchmark.
+First, the [`generate_statistical_model`](@ref) method generates a machine learning predictor to predict cell weights from the input image:
+
+````@example warcraft
+model = generate_statistical_model(b)
+````
+
+In the case of the Warcraft benchmark, the model is a convolutional neural network built using the Flux.jl package.
+
+````@example warcraft
+θ = model(x)
+````
+
+Note that the model is not trained yet, and its parameters are randomly initialized.
+
+Finally, the [`generate_maximizer`](@ref) method can be used to generate a combinatorial optimization algorithm that takes the predicted cell weights as input and returns the corresponding shortest path:
+
+````@example warcraft
+maximizer = generate_maximizer(b; dijkstra=true)
+````
+
+In the case o fthe Warcraft benchmark, the method has an additional keyword argument to chose the algorithm to use: Dijkstra's algorithm or Bellman-Ford algorithm.
+
+````@example warcraft
+y = maximizer(θ)
+````
+
+As we can see, currently the pipeline predicts random noise as cell weights, and therefore the maximizer returns a straight line path.
+
+````@example warcraft
+plot_data(b, DataSample(; x, θ_true=θ, y_true=y))
+````
+
+We can evaluate the current pipeline performance using the optimality gap metric:
+
+````@example warcraft
+starting_gap = compute_gap(b, test_dataset, model, maximizer)
+````
+
+## Using a learning algorithm
+
+We can now train the model using the InferOpt.jl package:
+
+````@example warcraft
+using InferOpt
+using Flux
+using Plots
+
+perturbed_maximizer = PerturbedMultiplicative(maximizer; ε=0.2, nb_samples=100)
+loss = FenchelYoungLoss(perturbed_maximizer)
+
+starting_gap = compute_gap(b, test_dataset, model, maximizer)
+
+opt_state = Flux.setup(Adam(1e-3), model)
+loss_history = Float64[]
+for epoch in 1:50
+    val, grads = Flux.withgradient(model) do m
+        sum(loss(m(x), y_true) for (; x, y_true) in train_dataset) / length(train_dataset)
+    end
+    Flux.update!(opt_state, model, grads[1])
+    push!(loss_history, val)
+end
+
+plot(loss_history; xlabel="Epoch", ylabel="Loss", title="Training loss")
+````
+
+````@example warcraft
+final_gap = compute_gap(b, test_dataset, model, maximizer)
+````
+
+````@example warcraft
+θ = model(x)
+y = maximizer(θ)
+plot_data(b, DataSample(; x, θ_true=θ, y_true=y))
+````
+
+---
+
+*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*
+

src/Argmax/Argmax.jl

@@ -67,22 +67,18 @@ end
 """
 $TYPEDSIGNATURES
 
-Generate a dataset of labeled instances for the argmax problem.
+Generate a data sample for the argmax benchmark.
+This function generates a random feature matrix, computes the costs using the encoder,
+and adds noise to the costs before computing a target solution.
 """
-function Utils.generate_dataset(
-    bench::ArgmaxBenchmark, dataset_size::Int=10; seed::Int=0, noise_std=0.0
+function Utils.generate_sample(
+    bench::ArgmaxBenchmark, rng::AbstractRNG; noise_std::Float32=0.0f0
 )
     (; instance_dim, nb_features, encoder) = bench
-    rng = MersenneTwister(seed)
-    features = [randn(rng, Float32, nb_features, instance_dim) for _ in 1:dataset_size]
-    costs = encoder.(features)
-    noisy_solutions = [
-        one_hot_argmax(θ + noise_std * randn(rng, Float32, instance_dim)) for θ in costs
-    ]
-    return [
-        DataSample(; x, θ_true, y_true) for
-        (x, θ_true, y_true) in zip(features, costs, noisy_solutions)
-    ]
+    features = randn(rng, Float32, nb_features, instance_dim)
+    costs = encoder(features)
+    noisy_solution = one_hot_argmax(costs + noise_std * randn(rng, Float32, instance_dim))
+    return DataSample(; x=features, θ_true=costs, y_true=noisy_solution)
 end
 
 """