Merge pull request #9 from JuliaDecisionFocusedLearning/better-doc

Document existing benchmarks

Author: BatyLeo

docs/make.jl

@@ -6,12 +6,20 @@ cp(joinpath(@__DIR__, "..", "README.md"), joinpath(@__DIR__, "src", "index.md");
 
 md_dir = joinpath(@__DIR__, "src")
 tutorial_dir = joinpath(@__DIR__, "src", "tutorials")
+benchmarks_dir = joinpath(@__DIR__, "src", "benchmarks")
+
 tutorial_files = readdir(tutorial_dir)
 md_tutorial_files = [split(file, ".")[1] * ".md" for file in tutorial_files]
+benchmark_files = readdir(benchmarks_dir)
+md_benchmark_files = [split(file, ".")[1] * ".md" for file in benchmark_files]
+
+include_tutorial = true
 
-for file in tutorial_files
-    filepath = joinpath(tutorial_dir, file)
-    Literate.markdown(filepath, md_dir; documenter=true, execute=false)
+if include_tutorial
+    for file in tutorial_files
+        filepath = joinpath(tutorial_dir, file)
+        Literate.markdown(filepath, md_dir; documenter=true, execute=false)
+    end
 end
 
 makedocs(;
@@ -21,15 +29,23 @@ makedocs(;
     format=Documenter.HTML(),
     pages=[
         "Home" => "index.md",
-        "Tutorials" => md_tutorial_files,
+        "Tutorials" => include_tutorial ? md_tutorial_files : [],
+        "Benchmark problems list" => [
+            "benchmarks/subset_selection.md",
+            "benchmarks/fixed_size_shortest_path.md",
+            "benchmarks/warcraft.md",
+            "benchmarks/portfolio_optimization.md",
+        ],
         "API reference" =>
             ["api/interface.md", "api/decision_focused.md", "api/warcraft.md"],
     ],
 )
 
-for file in md_tutorial_files
-    filepath = joinpath(md_dir, file)
-    rm(filepath)
+if include_tutorial
+    for file in md_tutorial_files
+        filepath = joinpath(md_dir, file)
+        rm(filepath)
+    end
 end
 
 deploydocs(;

docs/src/benchmarks/fixed_size_shortest_path.md

@@ -0,0 +1,4 @@
+# Shortest paths
+
+[`FixedSizeShortestPathBenchmark`](@ref) is a benchmark problem that consists of finding the shortest path in a grid graph between the top left and bottom right corners.
+In this benchmark, the grid size is the same for all instances.

docs/src/benchmarks/portfolio_optimization.md

@@ -0,0 +1,12 @@
+# Portfolio Optimization
+
+[`PortfolioOptimizationBenchmark`](@ref) is a Markovitz portfolio optimization problem, where asset prices are unknown, and only contextual data is available to predict these prices.
+The goal is to predict asset prices $c$ and maximize the expected return of a portfolio, subject to a risk constraint using this maximization program:
+```math
+\begin{aligned}
+\max\quad & c^\top x\\
+\text{s.t.}\quad & x^\top \Sigma x \leq \gamma\\
+& 1^\top x \leq 1\\
+& x \geq 0
+\end{aligned}
+```

docs/src/benchmarks/subset_selection.md

@@ -0,0 +1,13 @@
+# Subset Selection
+
+[`SubsetSelectionBenchmark`](@ref) is the most trivial benchmark problem in this package.
+It is minimalistic and serves as a simple example for debugging and testing purposes.
+
+## Description
+We have a set of ``n`` items, each item having an unknown value.
+We want to select a subset of ``k`` items that maximizes the sum of the values of the selected items.
+
+As input, instead of the items costs, we are given a feature vector, such that an unknown linear mapping between the feature vector and the value of the items exists.
+
+By default, this linear mapping is the identity mapping, i.e., the value of each item is equal to the value of the corresponding feature vector element.
+However, this mapping can be changed by setting the `identity_mapping` parameter to `false`.

docs/src/benchmarks/warcraft.md

@@ -0,0 +1,3 @@
+# Warcraft
+
+See the tutorial for a full demo of [`WarcraftBenchmark`](@ref).

src/SubsetSelection/SubsetSelection.jl

@@ -58,13 +58,21 @@ Generate a dataset of labeled instances for the subset selection problem.
 The mapping between features and cost is identity.
 """
 function Utils.generate_dataset(
-    bench::SubsetSelectionBenchmark, dataset_size::Int=10; seed::Int=0
+    bench::SubsetSelectionBenchmark,
+    dataset_size::Int=10;
+    seed::Int=0,
+    identity_mapping=true,
 )
     (; n, k) = bench
     rng = MersenneTwister(seed)
     features = [randn(rng, Float32, n) for _ in 1:dataset_size]
-    costs = copy(features)  # we assume that the cost is the same as the feature
-    solutions = top_k.(features, k)
+    costs = if identity_mapping
+        copy(features)  # we assume that the cost is the same as the feature
+    else
+        mapping = Dense(n => n; bias=false)
+        mapping.(features)
+    end
+    solutions = top_k.(costs, k)
     return [DataSample(; x=x, θ=θ, y=y) for (x, θ, y) in zip(features, costs, solutions)]
 end
 

test/subset_selection.jl

@@ -11,6 +11,7 @@
     @test String(take!(io)) == "SubsetSelectionBenchmark(n=25, k=5)"
 
     dataset = generate_dataset(b, 50)
+    dataset2 = generate_dataset(b, 50; identity_mapping=false)
     model = generate_statistical_model(b)
     maximizer = generate_maximizer(b)