feat: implement DataVariable type with broadcasting support and metadata handling (#11)

Author: Beforerr

README.md

@@ -1,8 +1,9 @@
 # SpaceDataModel
 
-[![Build Status](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml?query=branch%3Amain)
-[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaSpacePhysics.github.io/SpaceDataModel.jl/dev/)
+[![Documentation](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaSpacePhysics.github.io/SpaceDataModel.jl/dev/)
 [![DOI](https://zenodo.org/badge/958430775.svg)](https://doi.org/10.5281/zenodo.15207556)
+
+[![Build Status](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml/badge.svg?branch=main)](https://github.com/JuliaSpacePhysics/SpaceDataModel.jl/actions/workflows/CI.yml?query=branch%3Amain)
 [![](https://img.shields.io/badge/%F0%9F%9B%A9%EF%B8%8F_tested_with-JET.jl-233f9a)](https://github.com/aviatesk/JET.jl)
 [![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
 [![Coverage](https://codecov.io/gh/JuliaSpacePhysics/SpaceDataModel.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/JuliaSpacePhysics/SpaceDataModel.jl)
@@ -23,13 +24,15 @@ Pkg.add("SpaceDataModel")
 ## Usage
 
 ```julia
-using SpaceDataModel
+using SpaceDataModel: Project, Instrument, DataSet, DataVariable
 
 # Create a project
 project = Project(; name="Project Name")
 instrument = Instrument(; name="Instrument Name")
 dataset = DataSet(name="Dataset Name")
+var = DataVariable([1.0, 2.0, 3.0], Dict())
 
 push!(project, instrument, dataset)
 push!(instrument, dataset)
+push!(dataset, var)
 ```

src/dataset.jl

@@ -3,10 +3,10 @@
 
 A concrete dataset with a name, data (parameters), and metadata.
 """
-@kwdef struct DataSet{T} <: AbstractDataSet
+@kwdef struct DataSet{T, MD} <: AbstractDataSet
     name::String = ""
     data::T = Dict()
-    metadata::Dict = Dict()
+    metadata::MD = NoMetadata()
 end
 
 """Construct a `DataSet` from a name and data, with optional metadata."""
@@ -49,11 +49,11 @@ ds = DataSet(lds; probe=1, data_rate="fast", data_type="des")
 The format string and variable patterns use placeholders like `{probe}`, `{data_rate}`, 
 which are replaced with actual values when creating a concrete `DataSet`.
 """
-@kwdef struct LDataSet <: AbstractDataSet
+@kwdef struct LDataSet{MD} <: AbstractDataSet
     name::String = ""
     format::String = ""
     data::Dict{String,String} = Dict()
-    metadata::Dict = Dict()
+    metadata::MD = NoMetadata()
 end
 
 "Construct a `LDataSet` from a dictionary."
@@ -83,4 +83,11 @@ for f in (:getindex, :iterate, :size, :length, :keys)
     @eval Base.$f(var::AbstractDataSet, args...) = $f(parent(var), args...)
 end
 
-Base.getindex(ds::DataSet{<:Dict}, i::Integer) = ds[collect(keys(ds))[i]]
+# https://github.com/JuliaLang/julia/issues/54454
+_nth(itr, n) = begin
+    y = iterate(Base.Iterators.drop(itr, n-1))
+    isnothing(y) ? throw(BoundsError(itr, n)) : first(y)
+end
+
+Base.getindex(ds::DataSet, i::Integer) = _nth(values(ds.data), i)
+Base.push!(ds::DataSet, v) = push!(ds.data, v)

src/variable.jl

@@ -10,7 +10,7 @@ Optional:
 * `meta(v)`: the metadata of the variable
 * `name(v)`: the name of the variable
 """
-abstract type AbstractDataVariable{T,N} <: AbstractArray{T,N} end
+abstract type AbstractDataVariable{T, N} <: AbstractArray{T, N} end
 
 # https://docs.julialang.org/en/v1/manual/interfaces/#man-interface-array
 Base.parent(var::AbstractDataVariable) = var.data
@@ -21,21 +21,21 @@ end
 
 for f in (:getindex,)
     @eval @propagate_inbounds Base.$f(var::AbstractDataVariable, I::Vararg{Int}) = $f(parent(var), I...)
-    @eval Base.$f(var::AbstractDataVariable, s::Union{String,Symbol}) = $f(meta(var), s)
+    @eval Base.$f(var::AbstractDataVariable, s::Union{String, Symbol}) = $f(meta(var), s)
 end
 
 @propagate_inbounds Base.setindex!(var::AbstractDataVariable, v, I::Vararg{Int}) = setindex!(parent(var), v, I...)
-Base.setindex!(var::AbstractDataVariable, v, s::Union{String,Symbol}) = setindex!(meta(var), v, s)
+Base.setindex!(var::AbstractDataVariable, v, s::Union{String, Symbol}) = setindex!(meta(var), v, s)
 
-Base.get(var::AbstractDataVariable, s::Union{String,Symbol}, d=nothing) = get(meta(var), s, d)
-Base.get(f::Function, var::AbstractDataVariable, s::Union{String,Symbol}) = get(f, meta(var), s)
+Base.get(var::AbstractDataVariable, s::Union{String, Symbol}, d = nothing) = get(meta(var), s, d)
+Base.get(f::Function, var::AbstractDataVariable, s::Union{String, Symbol}) = get(f, meta(var), s)
 
 tmin(v) = minimum(times(v))
 tmax(v) = maximum(times(v))
 
 _timerange_str(times) = "Time Range: $(minimum(times)) to $(maximum(times))"
 
-function Base.show(io::IO, var::T) where {T<:AbstractDataVariable}
+function Base.show(io::IO, var::T) where {T <: AbstractDataVariable}
     ismissing(var) && return
     print_name(io, var)
     print(io, " [")
@@ -45,10 +45,14 @@ function Base.show(io::IO, var::T) where {T<:AbstractDataVariable}
     isnothing(u) || print(io, " Units: ", u, ",")
     print(io, " Size: ", size(var))
     print(io, "]")
+    return
 end
 
+_is_valid(::Nothing) = false
+_is_valid(x) = applicable(isempty, x) ? !isempty(x) : true
+
 # Add Base.show methods for pretty printing
-function Base.show(io::IO, m::MIME"text/plain", var::T) where {T<:AbstractDataVariable}
+function Base.show(io::IO, m::MIME"text/plain", var::T) where {T <: AbstractDataVariable}
     ismissing(var) && return
     print(io, "$T: ")
     print_name(io, var)
@@ -59,8 +63,34 @@ function Base.show(io::IO, m::MIME"text/plain", var::T) where {T<:AbstractDataVa
     isnothing(u) || println(io, "  Units: ", u)
     println(io, "  Size: ", size(var))
     println(io, "  Memory Usage: ", Base.format_bytes(Base.summarysize(var)))
-    if (m = meta(var)) !== nothing
+    if (m = meta(var)) |> _is_valid
         print(io, "  Metadata:")
         _println_value(io, m)
     end
+    return
+end
+
+# DataVariable (a example of AbstractDataVariable)
+struct DataVariable{T, N, A <: AbstractArray{T, N}, D} <: AbstractDataVariable{T, N}
+    data::A
+    metadata::D
+end
+
+Base.similar(A::DataVariable, ::Type{T}, dims::Dims) where {T} = DataVariable(similar(A.data, T, dims), A.metadata)
+
+# Broadcast
+# Reference: https://docs.julialang.org/en/v1/manual/interfaces/#man-interfaces-broadcasting
+# Reference: https://github.com/rafaqz/DimensionalData.jl/blob/main/src/array/broadcast.jl
+using Base.Broadcast: ArrayStyle, Broadcasted
+Base.BroadcastStyle(::Type{<:AbstractDataVariable}) = ArrayStyle{AbstractDataVariable}()
+
+Base.similar(bc::Broadcasted{ArrayStyle{AbstractDataVariable}}, ::Type{T}) where {T} = similar(find_datavariable(bc), T)
+
+find_datavariable(x::Broadcasted) = find_datavariable(x.args)
+function find_datavariable(x::Tuple)
+    found = find_datavariable(x[1])
+    return isnothing(found) ? find_datavariable(Base.tail(x)) : found
 end
+find_datavariable(::Tuple{}) = nothing
+find_datavariable(x::AbstractDataVariable) = x
+find_datavariable(::Any) = nothing

src/workload.jl

@@ -2,8 +2,15 @@ function workload()
     io = IOContext(IOBuffer(), :color => true)
     var1 = [1, 2, 3]
     var2 = (4, 5, 6)
-    dataset = DataSet("Dataset Name", (var1, var2))
-    project = Project(name="Project Name", abbreviation="Proj", links="links")
+    var = DataVariable([1, 2, 3], Dict())
+    dataset = DataSet("Dataset Name", [var1, var2])
+    project = Project(name = "Project Name", abbreviation = "Proj", links = "links")
+    instrument = Instrument(name = "Instrument Name")
+    show(io, MIME"text/plain"(), var)
     show(io, MIME"text/plain"(), dataset)
     show(io, MIME"text/plain"(), project)
-end
+    push!(project, instrument, dataset)
+    push!(instrument, dataset)
+    push!(dataset, var)
+    return
+end

test/runtests.jl

@@ -3,8 +3,8 @@ using Test
 
 @run_package_tests
 
-@testset "SpaceDataModel.jl" begin
-    # Write your tests here.
+@testitem "SpaceDataModel.jl" begin
+    @test_nowarn SpaceDataModel.workload()
 end
 
 @testitem "Aqua" begin
@@ -41,6 +41,7 @@ end
     @test dataset2["key1"] === var1
     @test dataset2["key2"] === var2
     @test dataset2[2] ∈ (var1, var2)
+    @test_throws BoundsError dataset2[3]
 end
 
 @testitem "parse_datetime" begin
@@ -78,6 +79,7 @@ end
     end
 end
 
+
 @testitem "JET - Workload" begin
     using JET
     println(@report_opt ignored_modules = (Base,) SpaceDataModel.workload())

test/variable.jl

@@ -0,0 +1,13 @@
+@testitem "AbstractDataVariable Broadcasting" begin
+    using SpaceDataModel: AbstractDataVariable, DataVariable
+    using Base.Broadcast: ArrayStyle, Broadcasted
+
+    # Test BroadcastStyle
+    var = DataVariable([1.0, 2.0, 3.0], Dict("istest" => true))
+    @test Base.BroadcastStyle(typeof(var)) == ArrayStyle{AbstractDataVariable}()
+
+    # Test broadcasting operations
+    result1 = var .+ 1
+    @test result1 isa DataVariable
+    @test parent(result1) == [2.0, 3.0, 4.0]
+end