feat: add .versions property to Dataset

Author: mortenpi

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Added
+
+* Information about dataset versions can now be accessed via the the `.versions` property of a `Dataset` object. (#2)
+
 ## Version v0.1.0 - 2023-06-20
 
 Initial package release.

docs/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.9.1"
 manifest_format = "2.0"
-project_hash = "069906b5302a0260a868f88e6f63d03397df4071"
+project_hash = "c65c5d89b95b6f812384895c79ee446e88c10682"
 
 [[deps.ANSIColoredPrinters]]
 git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c"
@@ -136,7 +136,7 @@ version = "0.21.4"
 deps = ["Base64", "Dates", "Glob", "HTTP", "JSON", "Mocking", "Pkg", "PkgAuthentication", "Printf", "Rclone_jll", "SHA", "TOML", "Tar", "TimeZones", "URIs", "UUIDs"]
 path = ".."
 uuid = "bc7fa6ce-b75e-4d60-89ad-56c957190b6e"
-version = "0.0.2-DEV"
+version = "0.1.0"
 
 [[deps.LazyArtifacts]]
 deps = ["Artifacts", "Pkg"]

docs/Project.toml

@@ -4,4 +4,5 @@ DocumenterMermaid = "a078cd44-4d9c-4618-b545-3ab9d77f9177"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 JuliaHub = "bc7fa6ce-b75e-4d60-89ad-56c957190b6e"
 Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533"
+TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"

docs/make.jl

@@ -1,6 +1,10 @@
 using JuliaHub
 using Documenter, DocumenterMermaid
 
+# Timestamp printing is dependent on the timezone, so we force a specific (non-UTC)
+# timezone to make sure that the doctests don't fail because of timezone differences.
+ENV["TZ"] = "America/New_York"
+
 DocMeta.setdocmeta!(
     JuliaHub, :DocTestSetup,
     quote

docs/src/guides/datasets.md

@@ -30,12 +30,13 @@ julia> JuliaHub.datasets()
 
 If you know the name of the dataset, you can also directly access it with the [`dataset`](@ref) function, and you can access the dataset metadata via the properties of the [`Dataset`](@ref) object.
 
-```jldoctest
+```jldoctest example-dataset
 julia> ds = JuliaHub.dataset("example-dataset")
 Dataset: example-dataset (Blob)
  owner: username
  description: An example dataset
- size: 57 bytes
+ versions: 2
+ size: 388 bytes
  tags: tag1, tag2
 
 julia> ds.owner
@@ -45,7 +46,7 @@ julia> ds.description
 "An example dataset"
 
 julia> ds.size
-57
+388
 ```
 
 If you want to work with dataset that you do not own but is shared with you in JuliaHub, you can pass `shared=true` to [`datasets`](@ref), or specify the username.
@@ -65,6 +66,7 @@ julia> JuliaHub.dataset(("anotheruser", "publicdataset"))
 Dataset: publicdataset (Blob)
  owner: anotheruser
  description: An example dataset
+ versions: 1
  size: 57 bytes
  tags: tag1, tag2
 ```
@@ -79,6 +81,41 @@ Elapsed time:         2.1s
 "/home/username/my-project/mydata"
 ```
 
+As datasets can have multiple versions, the [`.versions` property of `Dataset`](@ref Dataset) can be used to see information about the individual versions (represented with [`DatasetVersion`](@ref) objects).
+When downloading, you can also specify the version you wish to download (with the default being the newest version).
+
+```jldoctest example-dataset; filter = r"\"/.+/mydata\""
+julia> ds.versions
+2-element Vector{JuliaHub.DatasetVersion}:
+ JuliaHub.DatasetVersion(dataset = ("username", "example-dataset"), version = 1)
+ JuliaHub.DatasetVersion(dataset = ("username", "example-dataset"), version = 2)
+
+julia> ds.versions[1]
+DatasetVersion: example-dataset @ v1
+ owner: username
+ timestamp: 2022-10-13T01:39:42.963-04:00
+ size: 57 bytes
+
+julia> JuliaHub.download_dataset("example-dataset", "mydata", version=ds.versions[1].id)
+Transferred:       86.767 KiB / 86.767 KiB, 100%, 0 B/s, ETA -
+Transferred:            1 / 1, 100%
+Elapsed time:         2.1s
+"/home/username/my-project/mydata"
+
+```
+
+The dataset version are sorted with oldest first.
+To explicitly access the newest dataset, you can use the `last` function on the `.versions` property.
+
+```jldoctest example-dataset
+julia> last(ds.versions)
+DatasetVersion: example-dataset @ v2
+ owner: username
+ timestamp: 2022-10-14T01:39:43.237-04:00
+ size: 331 bytes
+
+```
+
 !!! tip "Tip: DataSets.jl"
 
     In JuliaHub jobs and Cloud IDEs you can also use the [DataSets.jl](https://github.com/JuliaComputing/DataSets.jl) package to access and work with datasets.

docs/src/reference/datasets.md

@@ -41,14 +41,15 @@ The versions are indexed with a linear list of integers starting from `1`.
 ## Reference
 
 ```@docs
+JuliaHub.Dataset
+JuliaHub.DatasetVersion
 JuliaHub.datasets
 JuliaHub.DatasetReference
 JuliaHub.dataset
 JuliaHub.download_dataset
 JuliaHub.upload_dataset
 JuliaHub.update_dataset
 JuliaHub.delete_dataset
-JuliaHub.Dataset
 ```
 
 ## Index

src/datasets.jl

@@ -13,6 +13,63 @@ Base.@kwdef struct _DatasetStorage
     prefix::String
 end
 
+"""
+    struct DatasetVersion
+
+Represents one version of a dataset.
+
+Objects have the following properties:
+
+- `.id`: unique dataset version identifier (used e.g. in [`download_dataset`](@ref) to
+  identify the dataset version).
+- `.size :: Int`: size of the dataset version in bytes
+- `.timestamp :: ZonedDateTime`: dataset version timestamp
+
+```
+julia> JuliaHub.datasets()
+```
+
+See also: [`Dataset`](@ref), [`datasets`](@ref), [`dataset`](@ref).
+
+$(_DOCS_no_constructors_admonition)
+"""
+struct DatasetVersion
+    _dsref::Tuple{String, String}
+    id::Int
+    size::Int
+    timestamp::TimeZones.ZonedDateTime
+    _blobstore_path::String
+
+    function DatasetVersion(json::Dict; owner::AbstractString, name::AbstractString)
+        msg = "Unable to parse dataset version info for ($owner, $name)"
+        version = _get_json(json, "version", Int; msg)
+        size = _get_json(json, "size", Int; msg)
+        timestamp = _parse_tz(_get_json(json, "date", String; msg); msg)
+        blobstore_path = _get_json(json, "blobstore_path", String; msg)
+        new((owner, name), version, size, timestamp, blobstore_path)
+    end
+end
+
+function Base.show(io::IO, dsv::DatasetVersion)
+    owner, name = dsv._dsref
+    print(
+        io,
+        "JuliaHub.DatasetVersion(dataset = (\"",
+        owner,
+        "\", \"",
+        name,
+        "\"), version = $(dsv.id))",
+    )
+end
+function Base.show(io::IO, ::MIME"text/plain", dsv::DatasetVersion)
+    owner, name = dsv._dsref
+    printstyled(io, "DatasetVersion:"; bold=true)
+    print(io, " ", name, " @ v", dsv.id)
+    print(io, "\n owner: ", owner)
+    print(io, "\n timestamp: ", dsv.timestamp)
+    print(io, "\n size: ", dsv.size, " bytes")
+end
+
 """
     struct Dataset
 
@@ -23,6 +80,8 @@ public API:
 - `owner :: String`: username of the dataset owner
 - `name :: String`: dataset name
 - `dtype :: String`: generally either `Blob` or `BlobTree`, but additional values may be added in the future
+- `versions :: Vector{DatasetVersion}`: an ordered list of [`DatasetVersion`](@ref) objects, one for
+  each dataset version, sorted from oldest to latest (i.e. you can use `last` to get the newest version).
 - `size :: Int`: total size of the whole dataset (including all the dataset versions) in bytes
 - Fields to access user-provided dataset metadata:
   - `description :: String`: dataset description
@@ -44,38 +103,38 @@ Base.@kwdef struct Dataset
     uuid::UUIDs.UUID
     dtype::String
     size::Int64
+    versions::Vector{DatasetVersion}
     # User-set metadata
     description::String
     tags::Vector{String}
     # Additional metadata, but not part of public API
     _last_modified::Union{Nothing, TimeZones.ZonedDateTime}
     _downloadURL::String
-    _version::Union{Nothing, String}
-    _versions::Vector
     _storage::_DatasetStorage
     # Should not be used in code, but stores the full server
     # response for developer convenience.
     _json::Dict
 end
 
 function Dataset(d::Dict)
+    owner = d["owner"]["username"]
+    name = d["name"]
+    versions_json = _get_json_or(d, "versions", Vector, [])
+    versions = sort([DatasetVersion(json; owner, name) for json in versions_json]; by=dsv -> dsv.id)
     Dataset(;
         uuid=UUIDs.UUID(d["id"]),
-        name=d["name"],
-        owner=d["owner"]["username"],
+        name, owner, versions,
         dtype=d["type"],
         description=d["description"],
         size=d["size"],
         tags=d["tags"],
-        _versions=d["versions"],
         _downloadURL=d["downloadURL"],
         _last_modified=_nothing_or(d["lastModified"]) do last_modified
             datetime_utc = Dates.DateTime(
                 last_modified, Dates.dateformat"YYYY-mm-ddTHH:MM:SS.ss"
             )
             _utc2localtz(datetime_utc)
         end,
-        _version=isnothing(d["version"]) ? nothing : d["version"],
         _storage=_DatasetStorage(;
             credentials_url=d["credentials_url"],
             region=d["storage"]["bucket_region"],
@@ -94,19 +153,19 @@ function Base.show(io::IO, ::MIME"text/plain", d::Dataset)
     print(io, " ", d.name, " (", d.dtype, ")")
     print(io, "\n owner: ", d.owner)
     print(io, "\n description: ", d.description)
+    print(io, "\n versions: ", length(d.versions))
     print(io, "\n size: ", d.size, " bytes")
     isempty(d.tags) || print(io, "\n tags: ", join(d.tags, ", "))
 end
 
 function Base.:(==)(d1::Dataset, d2::Dataset)
     d1.name == d2.name &&
         d1.description == d2.description &&
-        d1._versions == d2._versions &&
+        d1.versions == d2.versions &&
         d1._downloadURL == d2._downloadURL &&
         d1.size == d2.size &&
         d1.tags == d2.tags &&
         d1._last_modified == d2._last_modified &&
-        d1._version == d2._version &&
         d1.dtype == d2.dtype &&
         d1.uuid == d2.uuid
 end
@@ -297,14 +356,16 @@ julia> dataset = JuliaHub.dataset("example-dataset")
 Dataset: example-dataset (Blob)
  owner: username
  description: An example dataset
- size: 57 bytes
+ versions: 2
+ size: 388 bytes
  tags: tag1, tag2
 
 julia> JuliaHub.dataset(dataset)
 Dataset: example-dataset (Blob)
  owner: username
  description: An example dataset
- size: 57 bytes
+ versions: 2
+ size: 388 bytes
  tags: tag1, tag2
 ```
 
@@ -703,8 +764,10 @@ function _parse_dataset_version(version::AbstractString)
 end
 
 function _find_dataset_version(dataset::Dataset, version::Integer)
-    for version_dict in dataset._versions
-        version_dict["version"] == version && return version_dict
+    # Starting form latest first, assuming that it's more common to
+    # try to find newer versions.
+    for dsv in Iterators.reverse(dataset.versions)
+        dsv.id == version && return dsv
     end
     return nothing
 end
@@ -722,7 +785,9 @@ If the dataset is a `Blob`, then the created `local_path` will be a file, and if
 the `local_path` will be a directory.
 
 By default, it downloads the latest version, but an older version can be downloaded by specifying
-the `version` keyword argument.
+the `version` keyword argument. Caution: you should never assume that the index of the `.versions` property
+of [`Dataset`](@ref) matches the version number -- always explicitly use the `.id` propert of the
+[`DatasetVersion`](@ref) object.
 
 The function also prints download progress to standard output. This can be disabled by setting `quiet=true`.
 Any error output from the download is still printed.
@@ -766,12 +831,14 @@ function download_dataset(
         ),
     )
 
+    isempty(dataset.versions) &&
+        throw(InvalidRequestError("Dataset '$(dataset.name)' does not have any versions"))
     if isnothing(version)
-        version = _parse_dataset_version(dataset._version)
+        version = last(dataset.versions).id
     end
     version_info = _find_dataset_version(dataset, version)
     isnothing(version_info) &&
-        throw(InvalidRequestError("Dataset '$(dataset.name)' does not have version '$version'"))
+        throw(InvalidRequestError("Dataset '$(dataset.name)' does not have version 'v$version'"))
 
     credentials = Mocking.@mock _get_dataset_credentials(auth, dataset)
     credentials["vendor"] == "aws" ||
@@ -780,8 +847,7 @@ function download_dataset(
 
     bucket = dataset._storage.bucket
     prefix = dataset._storage.prefix
-    version_path = version_info["blobstore_path"]
-    remote_uri = "juliahub_remote:$bucket/$prefix/$(dataset.uuid)/$version_path"
+    remote_uri = "juliahub_remote:$bucket/$prefix/$(dataset.uuid)/$(version_info._blobstore_path)"
     if dataset.dtype == "Blob"
         remote_uri *= "/data"
     end

src/utils.jl

@@ -372,3 +372,30 @@ function _throw_or_nothing(
     isnothing(nothrow_extra_logic_f) || nothrow_extra_logic_f(msg)
     return nothing
 end
+
+# Parses a timezoned timestamp string into a local timezone object
+const _VALID_TZ_DATEFORMATS = [
+    Dates.dateformat"yyyy-mm-ddTHH:MM:SS.ssszzz",
+    Dates.dateformat"yyyy-mm-ddTHH:MM:SS.sszzz",
+    Dates.dateformat"yyyy-mm-ddTHH:MM:SS.szzz",
+    Dates.dateformat"yyyy-mm-ddTHH:MM:SSzzz",
+]
+function _parse_tz(timestamp_str::AbstractString; msg::Union{AbstractString, Nothing}=nothing)
+    timestamp = nothing
+    for dateformat in _VALID_TZ_DATEFORMATS
+        timestamp = try
+            TimeZones.ZonedDateTime(timestamp_str, dateformat)
+        catch e
+            isa(e, ArgumentError) && continue
+            rethrow(e)
+        end
+    end
+    if isnothing(timestamp)
+        errmsg = "Unable to parse timestamp '$timestamp_str'"
+        if !isnothing(msg)
+            errmsg = string(msg, '\n', errmsg)
+        end
+        throw(JuliaHubError(errmsg))
+    end
+    return TimeZones.astimezone(timestamp, TimeZones.localzone())
+end

test/datasets-live.jl

@@ -135,7 +135,7 @@ try
         dataset = JuliaHub.dataset(datasetname; auth)
         @test dataset.name == datasetname
         @test dataset.dtype == "Blob"
-        @test length(dataset._versions) == 2
+        @test length(dataset.versions) == 2
     end
 
     # Updating metadata