Merge pull request #238 from JuliaDynamics/hw/df_extension

add dataframe extension

Author: hexaeder

Project.toml

@@ -1,7 +1,7 @@
 name = "NetworkDynamics"
 uuid = "22e9dc34-2a0d-11e9-0de0-8588d035468b"
 authors = ["Frank Hellmann <[email protected]>, Michael Lindner <[email protected]>, Hans Würfel <[email protected]"]
-version = "0.9.16"
+version = "0.9.17"
 
 [deps]
 ArgCheck = "dce04be8-c92d-5529-be00-80e4d2c0e197"
@@ -36,13 +36,15 @@ TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
 [weakdeps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78"
 SymbolicUtils = "d1185830-fcd6-423d-90d6-eec64667417b"
 Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7"
 
 [extensions]
 NetworkDynamicsCUDAExt = ["CUDA", "Adapt"]
+NetworkDynamicsDataFramesExt = ["DataFrames"]
 NetworkDynamicsMTKExt = ["ModelingToolkit", "SymbolicUtils", "Symbolics"]
 NetworkDynamicsSymbolicsExt = ["Symbolics", "MacroTools"]
 
@@ -51,6 +53,7 @@ Adapt = "4.0.4"
 ArgCheck = "2.3.0"
 Atomix = "1"
 CUDA = "5.5.2"
+DataFrames = "1"
 DiffEqCallbacks = "4.2.2"
 DocStringExtensions = "0.9.3"
 FastClosures = "0.3.2"

docs/Project.toml

@@ -3,6 +3,7 @@ name = "ND.jl docs"
 [deps]
 Bonito = "824d6782-a2ef-11e9-3a09-e5662e0c26f8"
 CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DataInterpolations = "82cc6244-b520-54b8-b5a6-8a565e85f1d0"
 DiffEqCallbacks = "459566f4-90b8-5000-8ac3-15dfb0a30def"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
@@ -44,6 +45,7 @@ path = "../NetworkDynamicsInspector"
 [compat]
 Bonito = "≥0.0.1"
 CairoMakie = "0.13.1"
+DataFrames = "≥0.0.1"
 DataInterpolations = "7, 8"
 DiffEqCallbacks = "4.2.2"
 Distributions = "0.25.117"

docs/make.jl

@@ -7,6 +7,7 @@ using NetworkDynamicsInspector: NetworkDynamicsInspector as NDI
 using SciMLBase
 using Literate
 using ModelingToolkit
+using DataFrames: DataFrames
 using DocumenterInterLinks
 using Electron
 
@@ -34,10 +35,11 @@ for example in filter(contains(r".jl$"), readdir(example_dir, join=true))
 end
 
 mtkext = Base.get_extension(NetworkDynamics, :NetworkDynamicsMTKExt)
+dfext = Base.get_extension(NetworkDynamics, :NetworkDynamicsDataFramesExt)
 kwargs = (;
     root=joinpath(pkgdir(NetworkDynamics), "docs"),
     sitename="NetworkDynamics",
-    modules=[NetworkDynamics, mtkext, NetworkDynamicsInspector],
+    modules=[NetworkDynamics, mtkext, dfext, NetworkDynamicsInspector],
     linkcheck=true, # checks if external links resolve
     pagesonly=true,
     plugins=[links],

docs/src/API.md

@@ -63,6 +63,8 @@ NWParameter
 NWParameter(::Any)
 NWParameter(::NWParameter)
 NWParameter(::SciMLBase.DEIntegrator)
+pflat
+parameter_symbols
 ```
 
 ### Network State Object
@@ -73,7 +75,7 @@ NWState(::NWState)
 NWState(::NWParameter)
 NWState(::SciMLBase.DEIntegrator)
 uflat
-pflat
+variable_symbols
 ```
 
 ### Symbolic Indices
@@ -93,18 +95,14 @@ vpidxs
 epidxs
 ```
 
-### Solution Inspection
-```@docs
-dump_state
-```
-
 ## Metadata API
 ### Component Metadata API
 ```@docs
 metadata
 has_metadata(::NetworkDynamics.ComponentModel, ::Symbol)
 get_metadata(::NetworkDynamics.ComponentModel, ::Symbol)
 set_metadata!(::NetworkDynamics.ComponentModel, ::Symbol, ::Any)
+delete_metadata!(::NetworkDynamics.ComponentModel, ::Symbol)
 has_graphelement
 get_graphelement
 set_graphelement!
@@ -121,18 +119,41 @@ symmetadata
 get_metadata(::NetworkDynamics.ComponentModel, ::Symbol, ::Symbol)
 has_metadata(::NetworkDynamics.ComponentModel, ::Symbol, ::Symbol)
 set_metadata!(::NetworkDynamics.ComponentModel, ::Symbol, ::Symbol, ::Any)
+delete_metadata!(::NetworkDynamics.ComponentModel, ::Symbol, ::Symbol)
 has_default
 get_default
 set_default!
+delete_default!
 has_guess
 get_guess
 set_guess!
+delete_guess!
 has_init
 get_init
 set_init!
+delete_init!
 has_bounds
 get_bounds
 set_bounds!
+delete_bounds!
+set_defaults!
+set_interface_defaults!
+```
+
+### Metadata and Inspection Utils
+```@docs
+dump_state
+dump_initial_state
+get_initial_state
+describe_vertices
+describe_edges
+```
+
+## Initialization
+```@docs
+find_fixpoint
+initialize_component!
+init_residual
 ```
 
 ## Callbacks API
@@ -156,17 +177,6 @@ set_callback!
 add_callback!
 ```
 
-## Initialization
-```@docs
-find_fixpoint
-initialize_component!
-init_residual
-get_initial_state
-dump_initial_state
-set_defaults!
-set_interface_defaults!
-```
-
 ## Execution Types
 ```@docs
 ExecutionStyle

docs/src/data_structure.md

@@ -20,7 +20,7 @@ A typical workflow is:
 
 1. Set default values in the models using the metadata (see [Metadata](@ref)).
 2. Create a network (see [Network Construction](@ref)).
-3. Generate a state `s = NWState(nw)` which will be prefilled with the default values from the component metadata (see [SymbolicIndexing](@ref)).
+3. Generate a state `s = NWState(nw)` which will be prefilled with the default values from the component metadata (see [Symbolic Indexing](@ref)).
 4. Change the values of `s`, i.e., `s.v[1,:x] = 1.0`: This changes the **underlying flat array** but not the metadata of the models.
 5. Build a problem with the updated flat arrays using `uflat(s)` and `pflat(s)`.
 

docs/src/metadata.md

@@ -1,11 +1,10 @@
 # Metadata
-
-Component model such as [`VertexModel`](@ref) and [`EdgeModel`](@ref) can store metadata. We distinguish between two kinds of metadata: component metadata and symbol metadata.
+Component models such as [`VertexModel`](@ref) and [`EdgeModel`](@ref) can store metadata. We distinguish between two kinds of metadata: component metadata and symbol metadata.
 
 ## Component Metadata
 Component metadata is a `Dict{Symbol,Any}` attached to each component to store various information. Use [`metadata`](@ref) to retrieve the full dict.
 
-To access the data, you can use the methods `has_metadata`, `get_metadata` and `set_metadata!` (see [Component Metadata API](@ref)).
+To access the data, you can use the methods `has_metadata`, `get_metadata`, `set_metadata!` and `delete_metadata!` (see [Component Metadata API](@ref)).
 
 Special metadata: 
 
@@ -19,18 +18,28 @@ Special metadata:
 
 
 ## Symbol Metadata
-Each component stores symbol metadata. The symbol metadata is a `Dict{Symbol, Dict{Symbol, Any}}` which stores a metadate dict per symbol. Symbols are everything that appears in [`sym`](@ref), [`psym`](@ref), [`obssym`](@ref) and [`insym`](@ref).
+Each component stores symbol metadata. The symbol metadata is a `Dict{Symbol, Dict{Symbol, Any}}` which stores a metadata dict per symbol. Symbols are everything that appears in [`sym`](@ref), [`psym`](@ref), [`obssym`](@ref) and [`insym`](@ref).
 
-To access the data, you can use the methods `has_metadata`, `get_metadata` and `set_metadata!` (see [Per Symbol Metadata API](@ref)).
+To access the data, you can use the methods `has_metadata`, `get_metadata`, `set_metadata!` and `delete_metadata!` (see [Per Symbol Metadata API](@ref)).
 
 Special cases for symbol metadata are:
 
 - `default`: Stores default values for states/parameters. In initialization, those are considered fixed.
-- `guess`: Stores a guess for a state/parameter which needs to solved during initialization ("free" variables).
-- `bounds`: Store bounds for variables/parameters
-- `init`: Stores the solution of the "free" variables, this is rarely set manual but instead when calling [`initialize_component!`](@ref).
+- `guess`: Stores a guess for a state/parameter which needs to be solved during initialization ("free" variables).
+- `bounds`: Stores bounds for variables/parameters
+- `init`: Stores the solution of the "free" variables, this is rarely set manually but instead when calling [`initialize_component!`](@ref).
+
+For those, there are special functions `has_*`, `get_*`, `set_*!` and `delete_*!`. See [Per Symbol Metadata API](@ref).
+
 
-Fore those, there are special functions `has_*`, `get_*` and `set_*!`. See [Per Symbol Metadata API](@ref).
+These are closely aligned with the [metadata use in ModelingToolkit](https://docs.sciml.ai/ModelingToolkit/stable/basics/Variable_metadata/). They are automatically copied from the `ODESystem` if you use MTK models to create NetworkDynamics models.
 
+## Metadata Utils
+Accessing metadata (especially defaults) of states and parameters is a very
+common task. We provide several helper methods to do so. Please check out their docstrings for further explanation:
 
-Those are closely aligned to the [metadata use in ModelingToolkit](https://docs.sciml.ai/ModelingToolkit/stable/basics/Variable_metadata/). They are automatically copied from the `ODESystem` if you use MTK models to create NetworkDynamics models.
+- [`dump_state`](@ref)
+- [`dump_initial_state`](@ref)
+- [`get_initial_state`](@ref)
+- [`describe_vertices`](@ref) (needs `DataFrames.jl` loaded)
+- [`describe_edges`](@ref) (needs `DataFrames.jl` loaded)

docs/src/symbolic_indexing.md

@@ -92,7 +92,9 @@ For both `NWState` and `NWParameter` objects, there is a more convenient way to
 ```
 
 The `NWState` and `NWParameter` objects are mutable, thus changing them will also change the underlying wrapped flat arrays.
-You can always access the flat representations by calling [`uflat`](@ref) and [`pflat`](@ref).
+You can always access the flat representations by calling [`uflat`](@ref) and [`pflat`](@ref). The ordering of elements 
+in these flat arrays corresponds exactly to the order returned by [`variable_symbols`](@ref) and 
+[`parameter_symbols`](@ref) respectively.
 
 !!! note
     The `NWState` and `NWParameter` wrappers can be constructed from various objects.
@@ -156,4 +158,8 @@ and
 ```@example si
 SII.parameter_symbols(nw)
 ```
+These functions return the symbolic indices in the exact order they appear in the flat arrays
+returned by [`uflat`](@ref) and [`pflat`](@ref), making them essential when you need to map
+between flat array indices and symbolic representations.
+
 All above examples also work on other "symbolic containers", e.g. `SII.variable_symbols(::NWState)`.

ext/NetworkDynamicsDataFramesExt.jl

@@ -0,0 +1,130 @@
+module NetworkDynamicsDataFramesExt
+
+using NetworkDynamics: NetworkDynamics, Network
+using DataFrames: DataFrames, DataFrame
+using OrderedCollections: OrderedDict
+
+function _df_from_list(list, f)
+    df = DataFrame()
+    for (i, v) in list
+        row = OrderedDict{Symbol,Any}()
+        row[:idx] = i
+        for sym in f(v)
+            if NetworkDynamics.has_default_or_init(v, sym)
+                row[sym] = NetworkDynamics.get_default_or_init(v, sym)
+            else
+                row[sym] = missing
+            end
+        end
+        push!(df, NamedTuple(row); cols=:union)
+    end
+    df
+end
+function _df_from_pair(list, pair)
+    key, f = pair
+    df = DataFrame()
+    for (i, v) in list
+        row = OrderedDict{Symbol,Any}(:idx => i, key => f(v))
+        push!(df, NamedTuple(row); cols=:union)
+    end
+    df
+end
+
+"""
+    describe_vertices(nw::Network, extras...; parameters=true, states=true, batch=nothing)
+
+Creates a DataFrame containing information about the vertices in a Network.
+
+# Arguments
+- `nw::Network`: The network to describe
+- `extras...`: Additional pairs of (key, function) to include as columns,
+   where the function gets the [`VertexModel`](@ref NetworkDynamics.VertexModel) as its only parameter
+   to extract a custom metadata field for example..
+- `parameters=true`: Whether to include parameter values
+- `states=true`: Whether to include state values
+- `batch=nothing`: Optionally filter by specific batches
+
+# Returns
+A DataFrame with columns for vertex indices, names, batch numbers, and any parameter/state values.
+"""
+function NetworkDynamics.describe_vertices(nw::Network, extras...; parameters=true, states=true, batch=nothing)
+    pairs = enumerate(nw.im.vertexm);
+    batches = map(idx -> findfirst(batch -> idx ∈ batch.indices, nw.vertexbatches), first.(pairs))
+
+    if !isnothing(batch)
+        idxs = findall(b -> b ∈ batch, batches)
+        pairs = collect(pairs)[idxs]
+        batch = collect(batches)[idxs]
+    end
+    isempty(pairs) && return DataFrame()
+
+    basedf = DataFrame(
+        idx = first.(pairs),
+        name = map(v->last(v).name, pairs),
+        batch = map(idx -> findfirst(batch -> idx ∈ batch.indices, nw.vertexbatches), first.(pairs)),
+    )
+
+    dfs = [basedf,]
+    if parameters
+        push!(dfs, _df_from_list(pairs, NetworkDynamics.psym))
+    end
+    if states
+        push!(dfs, _df_from_list(pairs, NetworkDynamics.sym))
+    end
+    for p in extras
+        push!(dfs, _df_from_pair(pairs, p))
+    end
+
+    foldl((a,b) -> DataFrames.leftjoin(a,b; on=:idx), dfs)
+end
+
+"""
+    describe_edges(nw::Network, extras...; parameters=true, states=true, batch=nothing)
+
+Creates a DataFrame containing information about the edges in a Network.
+
+# Arguments
+- `nw::Network`: The network to describe
+- `extras...`: Additional pairs of (key, function) to include as columns,
+   where the function gets the [`EdgeModel`](@ref NetworkDynamics.EdgeModel) as its only parameter
+   to extract a custom metadata field for example..
+- `parameters=true`: Whether to include parameter values
+- `states=true`: Whether to include state values
+- `batch=nothing`: Optionally filter by specific batches
+
+# Returns
+A DataFrame with columns for edge indices, source-destination pairs, names, batch numbers, and any parameter/state values.
+"""
+function NetworkDynamics.describe_edges(nw::Network, extras...; parameters=true, states=true, batch=nothing)
+    pairs = enumerate(nw.im.edgem);
+    batches = map(idx -> findfirst(batch -> idx ∈ batch.indices, nw.layer.edgebatches), first.(pairs))
+
+    if !isnothing(batch)
+        idxs = findall(b -> b ∈ batch, batches)
+        pairs = collect(pairs)[idxs]
+        batch = collect(batches)[idxs]
+    end
+    isempty(pairs) && return DataFrame()
+
+    basedf = DataFrame(
+        idx = first.(pairs),
+        srcdst = map(idx -> nw.im.edgevec[idx].src => nw.im.edgevec[idx].dst, first.(pairs)),
+        name = map(v->last(v).name, pairs),
+        batch = map(idx -> findfirst(batch -> idx ∈ batch.indices, nw.layer.edgebatches), first.(pairs)),
+    )
+
+    dfs = [basedf,]
+    if parameters
+        push!(dfs, _df_from_list(pairs, NetworkDynamics.psym))
+    end
+    if states
+        push!(dfs, _df_from_list(pairs, NetworkDynamics.sym))
+    end
+    for p in extras
+        push!(dfs, _df_from_pair(pairs, p))
+    end
+
+    foldl((a,b) -> DataFrames.leftjoin(a,b; on=:idx), dfs)
+end
+
+end

ext/NetworkDynamicsMTKExt.jl

@@ -220,8 +220,15 @@ function _get_metadata(sys, name)
         if def isa Symbolic
             def = fixpoint_sub(def, alldefaults)
         end
-        def isa Symbolic && error("Could not resolve default $(ModelingToolkit.getdefault(sym)) for $name")
-        nt = (; nt..., default=def)
+
+        if def isa Symbolic
+            # do nothing, as the warning can get annoying
+            # @warn "Could not resolve rhs for default term $name = $(ModelingToolkit.getdefault(sym)). Some rhs symbols might not have default values. Leave free."
+        elseif def == ModelingToolkit.NoValue || def isa ModelingToolkit.NoValue
+            # skip NoValue thing
+        else
+            nt = (; nt..., default=def)
+        end
     end
 
     # check for guess both in symbol metadata and in guesses of system