Flux constraints JuMP (#14)

* overloads

* update

* update

* update

* update

* update

* update

* update

* update

* update

* running code

* code working

* update working

* update code

* rm unwanted code

* working code finally

* relax solutions working

* update best set-up

* update code

* update code

* fix bug

* update code

* update to save data

* add to git ignore

* update

* update

* update

* update

* try fix

* update code

* add arrow compresser

* update

* update

* update

* fix

* fix

* working code

* update

* update

* update

* fix error

* add comments

* fix typo

* update visuals

* update

* update code

* fix

* update code train

* update

* update

* update

* faster train float32

* update

* update

* update generation

* update

* update gitignore

* update objective

* fix update

* update lambda

* update fix

* update

* update

* fix tests

* update penalty

* update with model save

* update save train

* update code

* update train

* update

* update

* update

* update code

* update vis

---------

Co-authored-by: Andrew David Werner Rosemberg <[email protected]>
Co-authored-by: Andrew David Werner Rosemberg <[email protected]>
Co-authored-by: Andrew David Werner Rosemberg <[email protected]>
Co-authored-by: Andrew David Werner Rosemberg <[email protected]>

Author: 5 people

.gitignore

@@ -8,3 +8,13 @@ Manifest.toml
 *.arrow
 *.m
 *.csv
+.DS_Store
+*.out
+*.sbatch
+examples/unitcommitment/app/*
+*.edu
+examples/unitcommitment/wandb/*
+*.png
+*.jls
+*.jlso
+*.jld2

LocalPreferences.toml

@@ -0,0 +1,3 @@
+[CUDA_Runtime_jll]
+__clear__ = ["local"]
+version = "12.1"

Project.toml

@@ -11,9 +11,12 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MLJFlux = "094fc8d1-fd35-5302-93ea-dabda2abf845"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
 Nonconvex = "01bcebdf-4d21-426d-b5c4-6132c1619978"
+Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
 ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
@@ -22,21 +25,23 @@ Arrow = "2"
 CSV = "0.10"
 Dualization = "0.5"
 JuMP = "1"
-ParametricOptInterface = "0.5"
+ParametricOptInterface = "0.7"
+Zygote = "^0.6.68"
 julia = "1.6"
 
 [extras]
 AbstractGPs = "99985d1d-32ba-4be9-9821-2ec096f28918"
+CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
 Clarabel = "61c947e1-3e6d-4ee4-985a-eec8c727bd6e"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b"
 Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9"
+MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
 NonconvexNLopt = "b43a31b8-ff9b-442d-8e31-c163daa8ab75"
 PGLib = "07a8691f-3d11-4330-951b-3c50f98338be"
 PowerModels = "c36e90e8-916a-50a6-bd94-075b64ef4655"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
 
 [targets]
 test = ["Test", "DelimitedFiles", "PGLib", "HiGHS", "PowerModels", "DataFrames", "Clarabel", "Ipopt", "NonconvexNLopt", "MLJ"]

README.md

@@ -28,7 +28,7 @@ The user needs to first define a problem iterator:
 # The problem to iterate over
 model = Model(() -> POI.Optimizer(HiGHS.Optimizer()))
 @variable(model, x)
-p = @variable(model, p in POI.Parameter(1.0)) # The parameter (defined using POI)
+p = @variable(model, p in MOI.Parameter(1.0)) # The parameter (defined using POI)
 @constraint(model, cons, x + p >= 3)
 @objective(model, Min, 2x)
 
@@ -42,7 +42,7 @@ problem_iterator = ProblemIterator(parameter_values)
 The parameter values of the problem iterator can be saved by simply:
 
 ```julia
-save(problem_iterator, "input_file.csv", CSVFile)
+save(problem_iterator, "input_file", CSVFile)
 ```
 
 Which creates the following CSV:
@@ -71,7 +71,7 @@ Then chose what values to record:
 recorder = Recorder{CSVFile}("output_file.csv", primal_variables=[x], dual_variables=[cons])
 
 # Finally solve all problems described by the iterator
-solve_batch(model, problem_iterator, recorder)
+solve_batch(problem_iterator, recorder)
 ```
 
 Which creates the following CSV:

examples/flux/HuggingFaceDatasets.jl

@@ -0,0 +1,39 @@
+using PyCall
+using Conda
+# Conda.add("huggingface_hub")
+
+huggingface_hub = pyimport("huggingface_hub")
+
+huggingface_hub.login(token=ENV["HUGGINGFACE_TOKEN"])
+
+function download_dataset(organization, dataset, case_name, io_type; formulation="", cache_dir="./data/")
+    dataset_url = joinpath(organization, dataset)
+    if io_type ∉ ["input", "output"]
+        throw(ArgumentError("io_type must be 'input' or 'output'."))
+    end
+
+    if io_type == "input"
+        data_path = joinpath(case_name, "input")
+    else
+        if formulation == ""
+            throw(ArgumentError("Formulation must be specified for 'output' data."))
+        end
+        data_path = joinpath(case_name, "output", formulation)
+    end
+
+    # Fetch the dataset from the provided URL
+    huggingface_hub.snapshot_download(dataset_url, allow_patterns=["$data_path/*.arrow"], local_dir=cache_dir, repo_type="dataset", local_dir_use_symlinks=false)
+    
+    return nothing
+end
+
+cache_dir="./examples/powermodels/data/"
+organization = "L2O"
+dataset = "pglib_opf_solves"
+case_name = "pglib_opf_case300_ieee"
+formulation = "DCPPowerModel"
+io_type = "input"
+download_dataset(organization, dataset, case_name, io_type; cache_dir=cache_dir)
+
+io_type = "output"
+download_dataset(organization, dataset, case_name, io_type; formulation=formulation , cache_dir=cache_dir)

examples/flux/flux_forecaster_script.jl

@@ -3,26 +3,32 @@ TestEnv.activate()
 
 using Arrow
 using CSV
+using MLJFlux
 using Flux
+using MLJ
 using DataFrames
 using PowerModels
 using L2O
 
 # Paths
 case_name = "pglib_opf_case300_ieee" # pglib_opf_case300_ieee # pglib_opf_case5_pjm
-network_formulation = SOCWRConicPowerModel # SOCWRConicPowerModel # DCPPowerModel
+network_formulation = DCPPowerModel # SOCWRConicPowerModel # DCPPowerModel
 filetype = ArrowFile # ArrowFile # CSVFile
 path_dataset = joinpath(pwd(), "examples", "powermodels", "data")
-case_file_path = joinpath(path_dataset, case_name, string(network_formulation))
+case_file_path = joinpath(path_dataset, case_name)
+case_file_path_output = joinpath(case_file_path, "output", string(network_formulation))
+case_file_path_input = joinpath(case_file_path, "input")
 
 # Load input and output data tables
-iter_files = readdir(joinpath(case_file_path))
-iter_files = filter(x -> occursin(string(filetype), x), iter_files)
+iter_files_in = readdir(joinpath(case_file_path_input))
+iter_files_in = filter(x -> occursin(string(filetype), x), iter_files_in)
 file_ins = [
-    joinpath(case_file_path, file) for file in iter_files if occursin("input", file)
+    joinpath(case_file_path_input, file) for file in iter_files_in if occursin("input", file)
 ]
+iter_files_out = readdir(joinpath(case_file_path_output))
+iter_files_out = filter(x -> occursin(string(filetype), x), iter_files_out)
 file_outs = [
-    joinpath(case_file_path, file) for file in iter_files if occursin("output", file)
+    joinpath(case_file_path_output, file) for file in iter_files_out if occursin("output", file)
 ]
 batch_ids = [split(split(file, "_")[end], ".")[1] for file in file_ins]
 

examples/powermodels/pglib_datagen.jl

@@ -127,6 +127,7 @@ function pm_primal_builder!(
                 set_dual_variable!(recorder, real_balance)
             end
         end
+        set_model!(recorder)
         return model, parameters, variable_refs
     end
 
@@ -185,13 +186,13 @@ function generate_dataset_pglib(
         [network_data["load"]["$l"]["pd"] for l in 1:num_loads],
         [network_data["load"]["$l"]["qd"] for l in 1:num_loads],
     )
-    p = load_parameter_factory(model, 1:num_inputs; load_set=POI.Parameter.(original_load))
+    p = load_parameter_factory(model, 1:num_inputs; load_set=MOI.Parameter.(original_load))
 
     # Build model and Recorder
     file = joinpath(
         data_sim_dir, case_name * "_" * string(network_formulation) * "_output_" * batch_id
     )
-    recorder = Recorder{filetype}(file; filterfn=filterfn)
+    recorder = Recorder{filetype}(file; filterfn=filterfn,model=model)
     pm_primal_builder!(
         model, p, network_data, network_formulation; recorder=recorder, record_duals=true
     )
@@ -250,7 +251,7 @@ function generate_worst_case_dataset_Nonconvex(
         [l["qd"] for l in values(network_data["load"])],
     )
     p = load_parameter_factory(
-        model, 1:(num_loads * 2); load_set=POI.Parameter.(original_load)
+        model, 1:(num_loads * 2); load_set=MOI.Parameter.(original_load)
     )
 
     # Define batch id
@@ -265,7 +266,7 @@ function generate_worst_case_dataset_Nonconvex(
         data_sim_dir, case_name * "_" * string(network_formulation) * "_output_" * batch_id
     )
     recorder = Recorder{filetype}(
-        file_output; filename_input=file_input, primal_variables=[], dual_variables=[]
+        file_output; filename_input=file_input, primal_variables=[], dual_variables=[], model=model
     )
 
     # Build model
@@ -372,7 +373,7 @@ function generate_worst_case_dataset(
         data_sim_dir, case_name * "_" * string(network_formulation) * "_output_" * batch_id
     )
     recorder = Recorder{filetype}(
-        file_output; filename_input=file_input, primal_variables=[], dual_variables=[]
+        file_output; filename_input=file_input, primal_variables=[], dual_variables=[], model=JuMP.Model() # dummy model
     )
 
     # Solve all problems and record solutions

examples/unitcommitment/Project.toml

@@ -0,0 +1,14 @@
+[deps]
+Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Gurobi = "2e9cd046-0924-5485-92f1-d5272153d98b"
+JuMP = "4076af6c-e467-56ae-b986-b466b2749572"
+L2O = "e1d8bfa7-c465-446a-84b9-451470f6e76c"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+ParametricOptInterface = "0ce4ce61-57bf-432b-a095-efac525d185e"
+SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+UnitCommitment = "64606440-39ea-11e9-0f29-3303a1d3d877"