Skip to content

feat: implement DataVariable type with broadcasting support and metad… #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Beforerr merged 1 commit into from
Aug 11, 2025
Merged

feat: implement DataVariable type with broadcasting support and metad… #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
# SpaceDataModel

[![Build Status](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml?query=branch%3Amain)
[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaSpacePhysics.github.io/SpaceDataModel.jl/dev/)
[![Documentation](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaSpacePhysics.github.io/SpaceDataModel.jl/dev/)
[![DOI](https://zenodo.org/badge/958430775.svg)](https://doi.org/10.5281/zenodo.15207556)

[![Build Status](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml?query=branch%3Amain)
[![](https://img.shields.io/badge/%F0%9F%9B%A9%EF%B8%8F_tested_with-JET.jl-233f9a)](https://github.com/aviatesk/JET.jl)
[![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
[![Coverage](https://codecov.io/gh/JuliaSpacePhysics/SpaceDataModel.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/JuliaSpacePhysics/SpaceDataModel.jl)
Expand All @@ -23,13 +24,15 @@ Pkg.add("SpaceDataModel")
## Usage

```julia
using SpaceDataModel
using SpaceDataModel: Project, Instrument, DataSet, DataVariable

# Create a project
project = Project(; name="Project Name")
instrument = Instrument(; name="Instrument Name")
dataset = DataSet(name="Dataset Name")
var = DataVariable([1.0, 2.0, 3.0], Dict())

push!(project, instrument, dataset)
push!(instrument, dataset)
push!(dataset, var)
```
17 changes: 12 additions & 5 deletions src/dataset.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@

A concrete dataset with a name, data (parameters), and metadata.
"""
@kwdef struct DataSet{T} <: AbstractDataSet
@kwdef struct DataSet{T, MD} <: AbstractDataSet
name::String = ""
data::T = Dict()
metadata::Dict = Dict()
metadata::MD = NoMetadata()
end

"""Construct a `DataSet` from a name and data, with optional metadata."""
Expand Down Expand Up @@ -49,11 +49,11 @@ ds = DataSet(lds; probe=1, data_rate="fast", data_type="des")
The format string and variable patterns use placeholders like `{probe}`, `{data_rate}`,
which are replaced with actual values when creating a concrete `DataSet`.
"""
@kwdef struct LDataSet <: AbstractDataSet
@kwdef struct LDataSet{MD} <: AbstractDataSet
name::String = ""
format::String = ""
data::Dict{String,String} = Dict()
metadata::Dict = Dict()
metadata::MD = NoMetadata()
end

"Construct a `LDataSet` from a dictionary."
Expand Down Expand Up @@ -83,4 +83,11 @@ for f in (:getindex, :iterate, :size, :length, :keys)
@eval Base.$f(var::AbstractDataSet, args...) = $f(parent(var), args...)
end

Base.getindex(ds::DataSet{<:Dict}, i::Integer) = ds[collect(keys(ds))[i]]
# https://github.com/JuliaLang/julia/issues/54454
_nth(itr, n) = begin
y = iterate(Base.Iterators.drop(itr, n-1))
isnothing(y) ? throw(BoundsError(itr, n)) : first(y)
end

Base.getindex(ds::DataSet, i::Integer) = _nth(values(ds.data), i)
Base.push!(ds::DataSet, v) = push!(ds.data, v)
46 changes: 38 additions & 8 deletions src/variable.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Optional:
* `meta(v)`: the metadata of the variable
* `name(v)`: the name of the variable
"""
abstract type AbstractDataVariable{T,N} <: AbstractArray{T,N} end
abstract type AbstractDataVariable{T, N} <: AbstractArray{T, N} end

# https://docs.julialang.org/en/v1/manual/interfaces/#man-interface-array
Base.parent(var::AbstractDataVariable) = var.data
Expand All @@ -21,21 +21,21 @@ end

for f in (:getindex,)
@eval @propagate_inbounds Base.$f(var::AbstractDataVariable, I::Vararg{Int}) = $f(parent(var), I...)
@eval Base.$f(var::AbstractDataVariable, s::Union{String,Symbol}) = $f(meta(var), s)
@eval Base.$f(var::AbstractDataVariable, s::Union{String, Symbol}) = $f(meta(var), s)
end

@propagate_inbounds Base.setindex!(var::AbstractDataVariable, v, I::Vararg{Int}) = setindex!(parent(var), v, I...)
Base.setindex!(var::AbstractDataVariable, v, s::Union{String,Symbol}) = setindex!(meta(var), v, s)
Base.setindex!(var::AbstractDataVariable, v, s::Union{String, Symbol}) = setindex!(meta(var), v, s)

Base.get(var::AbstractDataVariable, s::Union{String,Symbol}, d=nothing) = get(meta(var), s, d)
Base.get(f::Function, var::AbstractDataVariable, s::Union{String,Symbol}) = get(f, meta(var), s)
Base.get(var::AbstractDataVariable, s::Union{String, Symbol}, d = nothing) = get(meta(var), s, d)
Base.get(f::Function, var::AbstractDataVariable, s::Union{String, Symbol}) = get(f, meta(var), s)

tmin(v) = minimum(times(v))
tmax(v) = maximum(times(v))

_timerange_str(times) = "Time Range: $(minimum(times)) to $(maximum(times))"

function Base.show(io::IO, var::T) where {T<:AbstractDataVariable}
function Base.show(io::IO, var::T) where {T <: AbstractDataVariable}
ismissing(var) && return
print_name(io, var)
print(io, " [")
Expand All @@ -45,10 +45,14 @@ function Base.show(io::IO, var::T) where {T<:AbstractDataVariable}
isnothing(u) || print(io, " Units: ", u, ",")
print(io, " Size: ", size(var))
print(io, "]")
return
end

_is_valid(::Nothing) = false
_is_valid(x) = applicable(isempty, x) ? !isempty(x) : true

# Add Base.show methods for pretty printing
function Base.show(io::IO, m::MIME"text/plain", var::T) where {T<:AbstractDataVariable}
function Base.show(io::IO, m::MIME"text/plain", var::T) where {T <: AbstractDataVariable}
ismissing(var) && return
print(io, "$T: ")
print_name(io, var)
Expand All @@ -59,8 +63,34 @@ function Base.show(io::IO, m::MIME"text/plain", var::T) where {T<:AbstractDataVa
isnothing(u) || println(io, " Units: ", u)
println(io, " Size: ", size(var))
println(io, " Memory Usage: ", Base.format_bytes(Base.summarysize(var)))
if (m = meta(var)) !== nothing
if (m = meta(var)) |> _is_valid
print(io, " Metadata:")
_println_value(io, m)
end
return
end

# DataVariable (a example of AbstractDataVariable)
struct DataVariable{T, N, A <: AbstractArray{T, N}, D} <: AbstractDataVariable{T, N}
data::A
metadata::D
end

Base.similar(A::DataVariable, ::Type{T}, dims::Dims) where {T} = DataVariable(similar(A.data, T, dims), A.metadata)

# Broadcast
# Reference: https://docs.julialang.org/en/v1/manual/interfaces/#man-interfaces-broadcasting
# Reference: https://github.com/rafaqz/DimensionalData.jl/blob/main/src/array/broadcast.jl
using Base.Broadcast: ArrayStyle, Broadcasted
Base.BroadcastStyle(::Type{<:AbstractDataVariable}) = ArrayStyle{AbstractDataVariable}()

Base.similar(bc::Broadcasted{ArrayStyle{AbstractDataVariable}}, ::Type{T}) where {T} = similar(find_datavariable(bc), T)

find_datavariable(x::Broadcasted) = find_datavariable(x.args)
function find_datavariable(x::Tuple)
found = find_datavariable(x[1])
return isnothing(found) ? find_datavariable(Base.tail(x)) : found
end
find_datavariable(::Tuple{}) = nothing
find_datavariable(x::AbstractDataVariable) = x
find_datavariable(::Any) = nothing
13 changes: 10 additions & 3 deletions src/workload.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,15 @@ function workload()
io = IOContext(IOBuffer(), :color => true)
var1 = [1, 2, 3]
var2 = (4, 5, 6)
dataset = DataSet("Dataset Name", (var1, var2))
project = Project(name="Project Name", abbreviation="Proj", links="links")
var = DataVariable([1, 2, 3], Dict())
dataset = DataSet("Dataset Name", [var1, var2])
project = Project(name = "Project Name", abbreviation = "Proj", links = "links")
instrument = Instrument(name = "Instrument Name")
show(io, MIME"text/plain"(), var)
show(io, MIME"text/plain"(), dataset)
show(io, MIME"text/plain"(), project)
end
push!(project, instrument, dataset)
push!(instrument, dataset)
push!(dataset, var)
return
end
6 changes: 4 additions & 2 deletions test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ using Test

@run_package_tests

@testset "SpaceDataModel.jl" begin
# Write your tests here.
@testitem "SpaceDataModel.jl" begin
@test_nowarn SpaceDataModel.workload()
end

@testitem "Aqua" begin
Expand Down Expand Up @@ -41,6 +41,7 @@ end
@test dataset2["key1"] === var1
@test dataset2["key2"] === var2
@test dataset2[2] ∈ (var1, var2)
@test_throws BoundsError dataset2[3]
end

@testitem "parse_datetime" begin
Expand Down Expand Up @@ -78,6 +79,7 @@ end
end
end


@testitem "JET - Workload" begin
using JET
println(@report_opt ignored_modules = (Base,) SpaceDataModel.workload())
Expand Down
13 changes: 13 additions & 0 deletions test/variable.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
@testitem "AbstractDataVariable Broadcasting" begin
using SpaceDataModel: AbstractDataVariable, DataVariable
using Base.Broadcast: ArrayStyle, Broadcasted

# Test BroadcastStyle
var = DataVariable([1.0, 2.0, 3.0], Dict("istest" => true))
@test Base.BroadcastStyle(typeof(var)) == ArrayStyle{AbstractDataVariable}()

# Test broadcasting operations
result1 = var .+ 1
@test result1 isa DataVariable
@test parent(result1) == [2.0, 3.0, 4.0]
end