Merge remote-tracking branch 'origin/main' into mp/project-datasets

Author: mortenpi

CHANGELOG.md

@@ -4,6 +4,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 ## Unreleased
 
+### Added
+
+* All the public API names are now correctly marked `public` in Julia 1.11 and above. ([#83])
+
+### Changed
+
+* The string `repr` of `DatasetVersion` (e.g. `dataset.versions`) is now valid Julia code. ([#84])
+
 ### Fixed
 
 * The `JuliaHub.update_dataset` function now correctly accepts the `license=(:fulltext, ...)` argument. ([#74])
@@ -141,3 +149,5 @@ Initial package release.
 [#53]: https://github.com/JuliaComputing/JuliaHub.jl/issues/53
 [#58]: https://github.com/JuliaComputing/JuliaHub.jl/issues/58
 [#74]: https://github.com/JuliaComputing/JuliaHub.jl/issues/74
+[#83]: https://github.com/JuliaComputing/JuliaHub.jl/issues/83
+[#84]: https://github.com/JuliaComputing/JuliaHub.jl/issues/84

Makefile

@@ -17,6 +17,9 @@ docs-manifest:
 docs: docs/Manifest.toml
 	${JULIA} --project=docs/ docs/make.jl
 
+fix-doctests: docs/Manifest.toml
+	${JULIA} --project=docs/ docs/make.jl --fix-doctests
+
 changelog:
 	${JULIA} --project=docs/ docs/changelog.jl
 

docs/src/guides/datasets.md

@@ -87,8 +87,8 @@ When downloading, you can also specify the version you wish to download (with th
 ```jldoctest example-dataset; filter = r"\"/.+/mydata\""
 julia> ds.versions
 2-element Vector{JuliaHub.DatasetVersion}:
- JuliaHub.DatasetVersion(dataset = ("username", "example-dataset"), version = 1)
- JuliaHub.DatasetVersion(dataset = ("username", "example-dataset"), version = 2)
+ JuliaHub.dataset(("username", "example-dataset")).versions[1]
+ JuliaHub.dataset(("username", "example-dataset")).versions[2]
 
 julia> ds.versions[1]
 DatasetVersion: example-dataset @ v1

src/JuliaHub.jl

@@ -39,4 +39,31 @@ function __init__()
     _LOCAL_TZ[] = _localtz()
 end
 
+# JuliaHub.jl follows the convention that all private names are
+# prefixed with an underscore.
+function _find_public_names()
+    return filter(names(@__MODULE__; all=true)) do s
+        # We don't need to check or mark public the main module itself
+        (s == :JuliaHub) && return false
+        startswith(string(s), "_") && return false
+        # Internal functions and types, prefixed by _
+        startswith(string(s), "_") && return false
+        # Internal macros, prefixed by _
+        startswith(string(s), "@_") && return false
+        # Strange generated functions
+        startswith(string(s), "#") && return false
+        # Some core functions that are not relevant for the package
+        s in [:eval, :include] && return false
+        return true
+    end
+end
+macro _mark_names_public()
+    if !Base.isdefined(Base, :ispublic)
+        return nothing
+    end
+    public_names = _find_public_names()
+    return esc(Expr(:public, public_names...))
+end
+@_mark_names_public
+
 end

src/datasets.jl

@@ -26,8 +26,25 @@ Objects have the following properties:
 - `.size :: Int`: size of the dataset version in bytes
 - `.timestamp :: ZonedDateTime`: dataset version timestamp
 
-```
-julia> JuliaHub.datasets()
+```jldoctest
+julia> dataset = JuliaHub.dataset("example-dataset")
+Dataset: example-dataset (Blob)
+ owner: username
+ description: An example dataset
+ versions: 2
+ size: 388 bytes
+ tags: tag1, tag2
+
+julia> dataset.versions
+2-element Vector{JuliaHub.DatasetVersion}:
+ JuliaHub.dataset(("username", "example-dataset")).versions[1]
+ JuliaHub.dataset(("username", "example-dataset")).versions[2]
+
+julia> dataset.versions[end]
+DatasetVersion: example-dataset @ v2
+ owner: username
+ timestamp: 2022-10-14T01:39:43.237-04:00
+ size: 331 bytes
 ```
 
 See also: [`Dataset`](@ref), [`datasets`](@ref), [`dataset`](@ref).
@@ -47,28 +64,25 @@ struct DatasetVersion
         size = _get_json(json, "size", Int; msg)
         timestamp = _parse_tz(_get_json(json, "date", String; msg); msg)
         blobstore_path = _get_json(json, "blobstore_path", String; msg)
-        new((owner, name), version, size, timestamp, blobstore_path)
+        return new((owner, name), version, size, timestamp, blobstore_path)
     end
 end
 
 function Base.show(io::IO, dsv::DatasetVersion)
     owner, name = dsv._dsref
-    print(
-        io,
-        "JuliaHub.DatasetVersion(dataset = (\"",
-        owner,
-        "\", \"",
-        name,
-        "\"), version = $(dsv.id))",
-    )
+    dsref = string("(\"", owner, "\", \"", name, "\")")
+    print(io, "JuliaHub.dataset($dsref).versions[$(dsv.id)]")
+    return nothing
 end
+
 function Base.show(io::IO, ::MIME"text/plain", dsv::DatasetVersion)
     owner, name = dsv._dsref
     printstyled(io, "DatasetVersion:"; bold=true)
     print(io, " ", name, " @ v", dsv.id)
     print(io, "\n owner: ", owner)
     print(io, "\n timestamp: ", dsv.timestamp)
     print(io, "\n size: ", dsv.size, " bytes")
+    return nothing
 end
 
 """

test/datasets.jl

@@ -56,6 +56,20 @@ end
             @test ds.versions[2].id == 2
             @test ds.versions[2].size == 331
 
+            # Test that .versions repr()-s to a valid array
+            # and DatasetVersion reprs to a valid JuliaHub.dataset().versions[...]
+            # call.
+            let versions = eval(Meta.parse(string(ds.versions)))
+                @test versions isa Vector{JuliaHub.DatasetVersion}
+                @test length(versions) == 2
+                @test versions == ds.versions
+            end
+            let expr = Meta.parse(string(ds.versions[1]))
+                @test expr == :((JuliaHub.dataset(("username", "example-dataset"))).versions[1])
+                version = eval(expr)
+                @test version == ds.versions[1]
+            end
+
             ds_updated = JuliaHub.dataset("example-dataset")
             @test ds_updated isa JuliaHub.Dataset
             @test ds_updated.name == ds.name

test/runtests.jl

@@ -173,19 +173,7 @@ end
     # This set tests that we haven't accidentally added or removed any public-looking
     # functions (i.e. ones that are not prefixed by _ basically).
     @testset "Public API" begin
-        public_symbols = Set(
-            filter(names(JuliaHub; all=true)) do s
-                # Internal functions and types, prefixed by _
-                startswith(string(s), "_") && return false
-                # Internal macros, prefixed by _
-                startswith(string(s), "@_") && return false
-                # Strange generated functions
-                startswith(string(s), "#") && return false
-                # Some core functions that are not relevant for the package
-                s in [:eval, :include] && return false
-                return true
-            end,
-        )
+        public_symbols = Set(JuliaHub._find_public_names())
         expected_public_symbols = Set([
             Symbol("@script_str"),
             :AbstractJobConfig, :AbstractJuliaHubApp,
@@ -194,7 +182,7 @@ end
             :Dataset, :DatasetReference, :DatasetVersion,
             :DefaultApp, :FileHash, :InvalidAuthentication, :InvalidRequestError, :Job,
             :WorkloadConfig, :JobFile, :JobLogMessage, :JobReference, :JobStatus,
-            :JuliaHub, :JuliaHubConnectionError, :JuliaHubError,
+            :JuliaHubConnectionError, :JuliaHubError,
             :JuliaHubException,
             :Limit, :NodeSpec, :PackageApp, :PackageJob, :Unlimited,
             :PermissionError, :script, :Secret, :UserApp,
@@ -223,6 +211,21 @@ end
             extra_expected_symbols = $(sprint(show, MIME"text/plain"(), extra_expected_symbols))
             """
         @test isempty(extra_expected_symbols)
+        # Make sure that on Julia versions that support the `public` keyword,
+        # we are also marking the right symbols as public.
+        if Base.isdefined(Base, :ispublic)
+            @testset "ispublic: $(name)" for name in public_symbols
+                @test Base.ispublic(JuliaHub, name)
+            end
+            private_names = setdiff(
+                names(JuliaHub; all=true),
+                public_symbols,
+                [:JuliaHub],
+            )
+            @testset "!ispublic: $(name)" for name in private_names
+                @test !Base.ispublic(JuliaHub, name)
+            end
+        end
     end
 
     @testset "Utilities" begin