refactor: parameterize types and add NoData type for dataset handling (#12)

Author: Beforerr

src/dataset.jl

@@ -3,9 +3,9 @@
 
 A concrete dataset with a name, data (parameters), and metadata.
 """
-@kwdef struct DataSet{T, MD} <: AbstractDataSet
+@kwdef struct DataSet{D, MD} <: AbstractDataSet
     name::String = ""
-    data::T = Dict()
+    data::D = NoData()
     metadata::MD = NoMetadata()
 end
 
@@ -49,18 +49,11 @@ ds = DataSet(lds; probe=1, data_rate="fast", data_type="des")
 The format string and variable patterns use placeholders like `{probe}`, `{data_rate}`, 
 which are replaced with actual values when creating a concrete `DataSet`.
 """
-@kwdef struct LDataSet{MD} <: AbstractDataSet
+@kwdef struct LDataSet{D, MD} <: AbstractDataSet
     name::String = ""
-    format::String = ""
-    data::Dict{String,String} = Dict()
+    data::D = NoData()
     metadata::MD = NoMetadata()
-end
-
-"Construct a `LDataSet` from a dictionary."
-function LDataSet(d::Dict)
-    dict = symbolify(d)
-    rename!(dict, :parameters, :data)
-    LDataSet(; dict...)
+    format::String = ""
 end
 
 """
@@ -83,11 +76,5 @@ for f in (:getindex, :iterate, :size, :length, :keys)
     @eval Base.$f(var::AbstractDataSet, args...) = $f(parent(var), args...)
 end
 
-# https://github.com/JuliaLang/julia/issues/54454
-_nth(itr, n) = begin
-    y = iterate(Base.Iterators.drop(itr, n-1))
-    isnothing(y) ? throw(BoundsError(itr, n)) : first(y)
-end
-
 Base.getindex(ds::DataSet, i::Integer) = _nth(values(ds.data), i)
 Base.push!(ds::DataSet, v) = push!(ds.data, v)

src/metadata.jl

@@ -10,6 +10,8 @@ always returns `false`.
 """
 struct NoMetadata end
 
+const NoData = NoMetadata
+
 Base.keys(::NoMetadata) = ()
 
 # Allow merging NoMetadata with a Dict or keyword arguments

src/product.jl

@@ -1,11 +1,11 @@
-struct Product{A,F} <: AbstractProduct
+struct Product{A,F,MD} <: AbstractProduct
     data::A
     transformation::F
     name::Union{String,Symbol}
-    metadata::Any
-    function Product(data::A, transformation::F, name="", metadata=NoMetadata(); kwargs...) where {A,F}
+    metadata::MD
+    function Product(data::A, transformation::F, name="", metadata::MD=NoMetadata(); kwargs...) where {A,F,MD}
         metadata = merge(metadata, kwargs)
-        new{A,F}(data, transformation, name, metadata)
+        new{A,F,MD}(data, transformation, name, metadata)
     end
 end
 

src/types.jl

@@ -17,16 +17,16 @@ A representation of a project or mission containing instruments and datasets.
 - `instruments`: Collection of instruments
 - `datasets`: Collection of datasets
 """
-mutable struct Project <: AbstractProject
+mutable struct Project{I, D, MD} <: AbstractProject
     name::String
-    metadata::Dict
-    instruments::Dict
-    datasets::Dict
+    instruments::I
+    datasets::D
+    metadata::MD
 end
 
 "keyword-based constructor"
 function Project(; name="", instruments=Dict(), datasets=Dict(), metadata=SDict(), kwargs...)
-    Project(name, compat_dict(metadata, kwargs), instruments, datasets)
+    Project(name, instruments, datasets, compat_dict(metadata, kwargs))
 end
 
 """
@@ -37,19 +37,19 @@ end
 - `metadata`: Additional metadata
 - `datasets`: Collection of datasets
 """
-struct Instrument <: AbstractInstrument
+struct Instrument{D, MD} <: AbstractInstrument
     name::String
-    metadata::Dict
-    datasets::Dict
+    datasets::D
+    metadata::MD
 end
 
 "keyword-based constructor"
 function Instrument(; name="", metadata=SDict(), datasets=Dict(), kwargs...)
-    Instrument(name, compat_dict(metadata, kwargs), datasets)
+    Instrument(name, datasets, compat_dict(metadata, kwargs))
 end
 
 "Construct an `Instrument` from a dictionary."
-Instrument(d::Dict) = Instrument(; symbolify(d)...)
+Instrument(d::AbstractDict) = Instrument(; (Symbol(k) => v for (k, v) in d)...)
 
 Base.insert!(p::Project, i, v::AbstractInstrument) = (p.instruments[i] = v; p)
 Base.insert!(p::Union{Project,Instrument}, i, v::AbstractDataSet) = (p.datasets[i] = v; p)

src/utils.jl

@@ -60,12 +60,16 @@ function _insert!(d::AbstractDict{K}, kw) where {K}
     return d
 end
 
+# https://github.com/JuliaLang/julia/issues/54454
+_nth(itr, n) = begin
+    y = iterate(Base.Iterators.drop(itr, n-1))
+    isnothing(y) ? throw(BoundsError(itr, n)) : first(y)
+end
+
 compat_dict(K::Type, m) = _insert!(Dict{K,Any}(), m)
 compat_dict(K::Type, m, kw) = _insert!(compat_dict(K, m), kw)
 compat_dict(m, kw) = compat_dict(String, m, kw)
 
-symbolify(d::Dict) = Dict{Symbol,Any}(Symbol(k) => v for (k, v) in d)
-
 function format_pattern(pattern; kwargs...)
     pairs = ("{$k}" => v for (k, v) in kwargs)
     return replace(pattern, pairs...)

test/Project.toml

@@ -1,5 +1,6 @@
 [deps]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
+CheckConcreteStructs = "73c92db5-9da6-4938-911a-6443a7e94a58"
 JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"

test/dataset.jl

@@ -0,0 +1,28 @@
+@testitem "DataSet" begin
+    var1 = [1, 2, 3]
+    var2 = (4, 5, 6)
+    dataset = DataSet("Dataset Name", [var1, var2])
+    @test size(dataset) == (2,)
+    @test length(dataset) == size(dataset, 1) == 2
+    @test dataset[1] === var1
+    @test dataset[2] === var2
+    @test dataset[1:2] == [var1, var2]
+
+    dataset2 = DataSet("Dataset Name", Dict("key1" => var1, "key2" => var2))
+    @test length(dataset2) == 2
+    @test dataset2["key1"] === var1
+    @test dataset2["key2"] === var2
+    @test dataset2[2] ∈ (var1, var2)
+    @test_throws BoundsError dataset2[3]
+end
+
+@testitem "Loading dataset from LDataSet" begin
+    using SpaceDataModel: LDataSet, DataSet
+    data = Dict("numberdensity" => "mms{probe}_{data_type}_numberdensity_{data_rate}")
+    format = "MMS{probe}_FPI_{data_rate}_L2_{data_type}-MOMS"
+    metadata = Dict("data_rates" => ["fast", "brst"])
+    ld = @test_nowarn LDataSet(; name = "Test", data, format, metadata)
+    ds = @test_nowarn DataSet(ld; probe = 1, data_rate = "fast", data_type = "des")
+    @test ds.name == "MMS1_FPI_FAST_L2_DES-MOMS"
+    @test ds.data["numberdensity"] == "mms1_des_numberdensity_fast"
+end

test/runtests.jl

@@ -7,9 +7,11 @@ using Test
     @test_nowarn SpaceDataModel.workload()
 end
 
-@testitem "Aqua" begin
+@testitem "General Checks" begin
     using Aqua
+    using CheckConcreteStructs
     Aqua.test_all(SpaceDataModel)
+    @test all_concrete(SpaceDataModel)
 end
 
 @testitem "Project" begin
@@ -26,24 +28,6 @@ end
     @test abbr(project) == "Proj"
 end
 
-@testitem "DataSet" begin
-    var1 = [1, 2, 3]
-    var2 = (4, 5, 6)
-    dataset = DataSet("Dataset Name", [var1, var2])
-    @test size(dataset) == (2,)
-    @test length(dataset) == size(dataset, 1) == 2
-    @test dataset[1] === var1
-    @test dataset[2] === var2
-    @test dataset[1:2] == [var1, var2]
-
-    dataset2 = DataSet("Dataset Name", Dict("key1" => var1, "key2" => var2))
-    @test length(dataset2) == 2
-    @test dataset2["key1"] === var1
-    @test dataset2["key2"] === var2
-    @test dataset2[2] ∈ (var1, var2)
-    @test_throws BoundsError dataset2[3]
-end
-
 @testitem "parse_datetime" begin
     using SpaceDataModel: parse_datetime
     using SpaceDataModel.Dates: DateTime