Merge branch 'Alexander-Barth:master' into diskarrays

Author: tcarion

docs/Project.toml

@@ -1,4 +1,8 @@
 [deps]
+CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
+DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
-DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+
+[compat]
+Documenter = "0.27"

docs/make.jl

@@ -1,11 +1,11 @@
 using Pkg
 Pkg.activate(@__DIR__)
 CI = get(ENV, "CI", nothing) == "true"
-using Documenter, NCDatasets
+using Documenter, NCDatasets, CommonDataModel
 
 makedocs(
-    modules = [NCDatasets],
-    sitename= "NCDatasets.jl",
+    modules = [NCDatasets, CommonDataModel],
+    sitename = "NCDatasets.jl",
     doctest = false,
     format = Documenter.HTML(
         prettyurls = CI,
@@ -21,6 +21,7 @@ makedocs(
         "Experimental features" => "experimental.md",
         "Tutorials" => "tutorials.md",
     ],
+    checkdocs = :none,
 )
 
 if CI

docs/src/dataset.md

@@ -31,10 +31,9 @@ Otherwise, we attempt to use standard structures from the Julia standard library
 ## Groups
 
 ```@docs
-defGroup(ds::NCDataset,groupname)
+defGroup
 getindex(g::NCDatasets.Groups,groupname::AbstractString)
 Base.keys(g::NCDatasets.Groups)
-    groupname(ds::NCDataset)
 ```
 
 ## Common methods

docs/src/index.md

@@ -122,6 +122,7 @@ subdata = NCDataset("/tmp/test.nc")["temperature"][10:30,30:5:end]
 ```
 
 This might be useful in an interactive session. However, the file `test.nc` is not closed, which can be a problem if you open many files. On Linux the number of opened files is often limited to 1024 (soft limit). If you write to a file, you should also always close the file to make sure that the data is properly written to the disk.
+(open files will get closed eventually when the dataset variable is finalized by julia's garbage collector).
 
 An alternative way to ensure the file has been closed is to use a `do` block: the file will be closed automatically when leaving the block.
 

docs/src/performance.md

@@ -1,23 +1,39 @@
 # [Performance tips](@id performance_tips)
 
-* Reading data from a file is not type-stable, because the type of the output of the read operation is dependent on the type defined in the NetCDF files and the value of various attribute (like `scale_factor`, `add_offset` and `units` for time conversion). All this information cannot be inferred from a static analysis of the source code. It is therefore recommended to use [type annotation](https://docs.julialang.org/en/v1/manual/types/index.html#Type-Declarations-1) if resulting type of a read operation in known:
+* Reading data from a file is not type-stable, because the type of the output of the read operation is dependent on the type defined in the NetCDF files and the value of various attribute (like `scale_factor`, `add_offset` and `units` for time conversion). All this information cannot be inferred from a static analysis of the source code. It is therefore recommended to use [type annotation](https://docs.julialang.org/en/v1/manual/types/index.html#Type-Declarations-1) if the resulting type of a read operation in known:
 
 ```julia
 ds = NCDataset("file.nc")
 nctemp = ds["temp"]
-temp = nctemp[:,:] :: Array{Float64,2}
-# or
-# call_barrier_function(nctemp)
+temp = nctemp[:,:] :: Array{Float32,2}
 
-Alternatively, one can also use so called "[function barriers](https://docs.julialang.org/en/v1/manual
-# call_barrier_function(temp)
-close(ds)
-```/performance-tips/index.html#kernel-functions-1)" or the in-place `NCDatasets.load!` function (which is unexported, so it has to be prefixed with the module name):
+# heavy computation using temp
+# ...
+```
+
+Alternatively, one can also use so-called [function barriers](https://docs.julialang.org/en/v1/manual/performance-tips/index.html#kernel-functions-1)
+since the function `heavy_computation` will be specialized based on the type its input parameters.
+
+
+```julia
+function heavy_computation(temp)
+# heavy computation using temp
+# ...
+end
+
+ds = NCDataset("file.nc")
+nctemp = ds["temp"]
+temp = nctemp[:,:]
+output = heavy_computation(temp)
+```
+
+Calling the barrier function with `nctemp` would also be type-stable.
+Using the in-place `NCDatasets.load!` function (which is unexported, so it has to be prefixed with the module name) does also lead to type-stable code and allows to reuse a memory buffer:
 
 ```julia
 ds = NCDataset("file.nc")
 
-temp = zeros(10,20)
+temp = zeros(Float32,10,20)
 NCDatasets.load!(variable(ds,"temp"),temp,:,:)
 ```
 

src/groupes.jl

@@ -32,7 +32,7 @@ function Base.getindex(g::Groups,groupname::SymbolOrString)
 end
 
 """
-    defGroup(ds::NCDataset,groupname, attrib = []))
+    defGroup(ds::NCDataset,groupname; attrib = []))
 
 Create the group with the name `groupname` in the dataset `ds`.
 `attrib` is a list of attribute name and attribute value pairs (see `NCDataset`).

src/variable.jl

@@ -138,11 +138,11 @@ load!(ds["temp"].var,data,:,1) # loads the 1st column
     julia `Char` type uses 4 bytes and the NetCDF `NC_CHAR` only 1 byte.
 """
 @inline function load!(ncvar::Variable{T,N}, data::AbstractArray{T}, indices::Union{Integer, UnitRange, StepRange, Colon}...) where {T,N}
-    @inline unsafe_load!(ncvar, data, indices...)
+    unsafe_load!(ncvar, data, indices...)
 end
 
 @inline function load!(ncvar::Variable{Char,N}, data::AbstractArray{UInt8}, indices::Union{Integer, UnitRange, StepRange, Colon}...) where N
-    @inline unsafe_load!(ncvar, data, indices...)
+    unsafe_load!(ncvar, data, indices...)
 end
 
 @inline function load!(ncvar::Variable{T,2}, data::AbstractArray{T}, i::Colon,j::UnitRange) where T

test/test_https.jl

@@ -3,8 +3,9 @@ using Test
 
 sample_url = "https://rda.ucar.edu/thredds/dodsC/files/g/ds084.1/2018/20181231/gfs.0p25.2018123118.f003.grib2"
 
-@test begin
-    NCDataset(sample_url) do ds
-        haskey(ds,"lon")
-    end
-end
+# opendal server too unreliable
+# @test begin
+#     NCDataset(sample_url) do ds
+#         haskey(ds,"lon")
+#     end
+# end