diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a3acf34..241bb99 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,38 +1,41 @@ name: CI on: + pull_request: push: branches: - main tags: '*' - pull_request: -concurrency: - # Skip intermediate builds: always. - # Cancel intermediate builds: only if it is a pull request build. - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: test: - - services: - mlflow: - image: adacotechjp/mlflow:2.3.1 - ports: - - 5000:5000 - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: fail-fast: false + max-parallel: 1 matrix: version: - - '1.6' - - '1' + - '1.10' + - '1' # automatically expands to the latest stable 1.x release of Julia. os: - ubuntu-latest arch: - x64 steps: - uses: actions/checkout@v2 + - name: Setup custom python requirements + if: hashFiles('**/requirements.txt', '**/pyproject.toml') == '' + run: | + touch ./requirements.txt + echo "mlflow==2.20.1" > ./requirements.txt + - uses: actions/setup-python@v4 + with: + python-version: '3.12.3' + cache: 'pip' + - name: Setup mlflow locally + run: | + pip install -r ./requirements.txt + python3 /opt/hostedtoolcache/Python/3.12.3/x64/bin/mlflow server --app-name basic-auth --host 0.0.0.0 --port 5000 & + sleep 5 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} @@ -50,9 +53,10 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: + JULIA_NUM_THREADS: '2' MLFLOW_TRACKING_URI: "http://localhost:5000/api" - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v3 with: files: lcov.info docs: diff --git a/.gitignore b/.gitignore index f4f6dab..24e4fdd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ Manifest.toml /docs/build/ mlruns coverage +Pipfile +Pipfile.lock +*.db diff --git a/Project.toml b/Project.toml index 59d3b69..f069775 100644 --- a/Project.toml +++ b/Project.toml @@ -5,7 +5,6 @@ version = "0.5.1" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" ShowCases = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" @@ -13,16 +12,15 @@ URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] -FilePathsBase = "0.9" -HTTP = "1.9" +HTTP = "1.0" JSON = "0.21" ShowCases = "0.1" URIs = "1.0" julia = "1.0" [extras] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] - +test = ["Base64", "Test"] diff --git a/README.md b/README.md index 51e9194..4cf00a3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,3 @@ Julia client for [MLFlow](https://www.mlflow.org/) - -This package is still under development and interfaces may change. See the documentation for current features and limitations. - -Tested against `mlflow==1.21.0` and `mlflow==1.22.0`. diff --git a/docs/Project.toml b/docs/Project.toml index dee67d4..d364e82 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,8 +1,6 @@ [deps] -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -MLFlowClient = "64a0f543-368b-4a9a-827a-e71edb2a0b83" +ShowCases = "605ecd9f-84a6-4c9e-81e2-4798472b76a3" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/docs/make.jl b/docs/make.jl index 3e6244d..905a055 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,27 +1,18 @@ -using MLFlowClient +push!(LOAD_PATH, "../src/") using Documenter - -DocMeta.setdocmeta!(MLFlowClient, :DocTestSetup, :(using MLFlowClient); recursive=true) +using MLFlowClient makedocs(; - modules=[MLFlowClient], - authors="@deyandyankov and contributors", - repo="https://github.com/JuliaAI.jl/blob/{commit}{path}#{line}", sitename="MLFlowClient.jl", - format=Documenter.HTML(; - prettyurls=get(ENV, "CI", "false") == "true", - canonical="https://juliaai.github.io/MLFlowClient.jl", - assets=String[] - ), - pages=[ - "Home" => "index.md", - "Tutorial" => "tutorial.md", - "Reference" => "reference.md" - ], - checkdocs=:exports -) + authors="@deyandyankov and contributors", + pages=["Home" => "index.md", "Tutorial" => "tutorial.md", "Reference" => [ + "Types" => "reference/types.md", "Artifact operations" => "reference/artifact.md", + "Experiment operations" => "reference/experiment.md", + "Logging operations" => "reference/logger.md", + "Miscellaneous operations" => "reference/misc.md", + "Run operations" => "reference/run.md", + "Registered model operations" => "reference/registered_model.md", + "Model version operations" => "reference/model_version.md", + "User operations" => "reference/user.md",]]) -deploydocs(; - repo="github.com/JuliaAI/MLFlowClient.jl", - devbranch="main" -) +deploydocs(; repo="github.com/JuliaAI/MLFlowClient.jl", devbranch="main") diff --git a/docs/src/mlflowexp.png b/docs/src/images/mlflowexp.png similarity index 100% rename from docs/src/mlflowexp.png rename to docs/src/images/mlflowexp.png diff --git a/docs/src/mlflowexpmetric1.png b/docs/src/images/mlflowexpmetric1.png similarity index 100% rename from docs/src/mlflowexpmetric1.png rename to docs/src/images/mlflowexpmetric1.png diff --git a/docs/src/withoutmlflow.png b/docs/src/images/withoutmlflow.png similarity index 100% rename from docs/src/withoutmlflow.png rename to docs/src/images/withoutmlflow.png diff --git a/docs/src/reference.md b/docs/src/reference.md deleted file mode 100644 index 7f79daf..0000000 --- a/docs/src/reference.md +++ /dev/null @@ -1,59 +0,0 @@ -# Reference - -```@meta -CurrentModule = MLFlowClient -``` - -# Types - -TODO: Document accessors. - -```@docs -MLFlow -MLFlowExperiment -MLFlowRun -MLFlowRunInfo -MLFlowRunData -MLFlowRunDataParam -MLFlowRunDataMetric -MLFlowRunStatus -MLFlowArtifactFileInfo -MLFlowArtifactDirInfo -``` - -# Experiments - -```@docs -createexperiment -getexperiment -getorcreateexperiment -deleteexperiment -searchexperiments -restoreexperiment -``` - -# Runs - -```@docs -createrun -getrun -updaterun -deleterun -searchruns -logparam -logmetric -logbatch -logartifact -listartifacts -``` - -# Utilities - -```@docs -mlfget -mlfpost -uri -generatefilterfromentity_type -generatefilterfromparams -generatefilterfromattributes -``` diff --git a/docs/src/reference/artifact.md b/docs/src/reference/artifact.md new file mode 100644 index 0000000..0e28093 --- /dev/null +++ b/docs/src/reference/artifact.md @@ -0,0 +1,4 @@ +# Artifact operations +```@docs +listartifacts +``` diff --git a/docs/src/reference/experiment.md b/docs/src/reference/experiment.md new file mode 100644 index 0000000..ad3ced6 --- /dev/null +++ b/docs/src/reference/experiment.md @@ -0,0 +1,15 @@ +# Experiment operations +```@docs +createexperiment +getexperiment +getexperimentbyname +deleteexperiment +restoreexperiment +updateexperiment +searchexperiments +setexperimenttag +createexperimentpermission +getexperimentpermission +updateexperimentpermission +deleteexperimentpermission +``` diff --git a/docs/src/reference/logger.md b/docs/src/reference/logger.md new file mode 100644 index 0000000..2bad6ce --- /dev/null +++ b/docs/src/reference/logger.md @@ -0,0 +1,7 @@ +# Logging operations +```@docs +logmetric +logbatch +loginputs +logparam +``` diff --git a/docs/src/reference/misc.md b/docs/src/reference/misc.md new file mode 100644 index 0000000..7baadfc --- /dev/null +++ b/docs/src/reference/misc.md @@ -0,0 +1,5 @@ +# Miscellaneous operations +```@docs +getmetrichistory +refresh +``` diff --git a/docs/src/reference/model_version.md b/docs/src/reference/model_version.md new file mode 100644 index 0000000..3f6e4f7 --- /dev/null +++ b/docs/src/reference/model_version.md @@ -0,0 +1,15 @@ +# Model version operations + +```@docs +getlatestmodelversions +createmodelversion +getmodelversion +updatemodelversion +deletemodelversion +searchmodelversions +getdownloaduriformodelversionartifacts +transitionmodelversionstage +setmodelversiontag +deletemodelversiontag +getmodelversionbyalias +``` diff --git a/docs/src/reference/registered_model.md b/docs/src/reference/registered_model.md new file mode 100644 index 0000000..a062ed4 --- /dev/null +++ b/docs/src/reference/registered_model.md @@ -0,0 +1,17 @@ +# Registered model operations +```@docs +createregisteredmodel +getregisteredmodel +renameregisteredmodel +updateregisteredmodel +deleteregisteredmodel +searchregisteredmodels +setregisteredmodeltag +deleteregisteredmodeltag +deleteregisteredmodelalias +setregisteredmodelalias +createregisteredmodelpermission +getregisteredmodelpermission +updateregisteredmodelpermission +deleteregisteredmodelpermission +``` diff --git a/docs/src/reference/run.md b/docs/src/reference/run.md new file mode 100644 index 0000000..5fbd25c --- /dev/null +++ b/docs/src/reference/run.md @@ -0,0 +1,11 @@ +# Run operations +```@docs +createrun +deleterun +restorerun +getrun +setruntag +deleteruntag +searchruns +updaterun +``` diff --git a/docs/src/reference/types.md b/docs/src/reference/types.md new file mode 100644 index 0000000..b6782ba --- /dev/null +++ b/docs/src/reference/types.md @@ -0,0 +1,25 @@ +# Types +```@docs +MLFlow +Tag +ViewType +RunStatus +ModelVersionStatus +Dataset +DatasetInput +FileInfo +ModelVersion +RegisteredModel +RegisteredModelAlias +Experiment +Run +Param +Metric +RunData +RunInfo +RunInputs +User +Permission +ExperimentPermission +RegisteredModelPermission +``` diff --git a/docs/src/reference/user.md b/docs/src/reference/user.md new file mode 100644 index 0000000..9fcc78f --- /dev/null +++ b/docs/src/reference/user.md @@ -0,0 +1,8 @@ +# User operations +```@docs +createuser +getuser +updateuserpassword +updateuseradmin +deleteuser +``` diff --git a/docs/src/tutorial.md b/docs/src/tutorial.md index 3fbf101..d7565c3 100644 --- a/docs/src/tutorial.md +++ b/docs/src/tutorial.md @@ -40,7 +40,7 @@ p This could result in the following plot: -![](withoutmlflow.png) +![](images/withoutmlflow.png) Now, suppose that you are interested in turning this into an experiment which stores its metadata and results in MLFlow using `MLFlowClient`. You could amend the code like this: @@ -114,8 +114,8 @@ updaterun(mlf, exprun, "FINISHED") This will result in the folowing experiment created in your `MLFlow` which is running on `http://localhost/`: -![](mlflowexp.png) +![](images/mlflowexp.png) You can also observe series logged against individual metrics, i.e. `pricepath1` looks like this in `MLFlow`: -![](mlflowexpmetric1.png) +![](images/mlflowexpmetric1.png) diff --git a/src/MLFlowClient.jl b/src/MLFlowClient.jl index 18c7ddb..2a46265 100644 --- a/src/MLFlowClient.jl +++ b/src/MLFlowClient.jl @@ -1,14 +1,13 @@ """ MLFlowClient -[MLFlowClient](https://github.com/JuliaAI.jl) is a [Julia](https://julialang.org/) package for working with [MLFlow](https://mlflow.org/) using the REST [API v2.0](https://www.mlflow.org/docs/latest/rest-api.html). +[MLFlowClient](https://github.com/JuliaAI.jl) is a [Julia](https://julialang.org/) package +for working with [MLFlow](https://mlflow.org/) using the REST +[API v2.0](https://www.mlflow.org/docs/latest/rest-api.html). -`MLFlowClient` allows you to create and manage `MLFlow` experiments, runs, and log metrics and artifacts. If you are not familiar with `MLFlow` and its concepts, please refer to [MLFlow documentation](https://mlflow.org/docs/latest/index.html). - -# Limitations - -- no authentication support. -- when storing artifacts, the assumption is that MLFlow and this library run on the same server. Artifacts are stored using plain filesystem operations. Therefore, `/mlruns` or the specified `artifact_location` must be accessible to both the MLFlow server (read), and this library (write). +`MLFlowClient` allows you to create and manage `MLFlow` experiments, runs, and log metrics +and artifacts. If you are not familiar with `MLFlow` and its concepts, please refer to +[MLFlow documentation](https://mlflow.org/docs/latest/index.html). """ module MLFlowClient @@ -18,70 +17,74 @@ using HTTP using URIs using JSON using ShowCases -using FilePathsBase: AbstractPath include("types/mlflow.jl") -export - MLFlow +export MLFlow + +include("types/tag.jl") +export Tag + +include("types/enums.jl") +export ViewType, RunStatus, ModelVersionStatus, Permission + +include("types/dataset.jl") +export Dataset, DatasetInput + +include("types/artifact.jl") +export FileInfo + +include("types/model_version.jl") +export ModelVersion + +include("types/registered_model.jl") +export RegisteredModel, RegisteredModelAlias, RegisteredModelPermission include("types/experiment.jl") -export - MLFlowExperiment +export Experiment, ExperimentPermission include("types/run.jl") -export - MLFlowRunStatus, - MLFlowRunInfo, - MLFlowRunDataMetric, - MLFlowRunDataParam, - MLFlowRunData, - MLFlowRun, - get_info, - get_data, - get_run_id, - get_params +export Run, Param, Metric, RunData, RunInfo, RunInputs -include("types/artifact.jl") -export - MLFlowArtifactFileInfo, - MLFlowArtifactDirInfo, - get_path, - get_size +include("types/user.jl") +export User include("api.jl") include("utils.jl") -export - generatefilterfromparams - generatefilterfromattributes - generatefilterfromentity_type - -include("experiments.jl") -export - createexperiment, - getexperiment, - getorcreateexperiment, - deleteexperiment, - restoreexperiment, - searchexperiments - -include("runs.jl") -export - createrun, - getrun, - updaterun, - deleterun, - searchruns - -include("loggers.jl") -export - logbatch, - logparam, - logmetric, - logartifact, - listartifacts, - settag - -include("deprecated.jl") + +include("services/experiment.jl") +export getexperiment, createexperiment, deleteexperiment, setexperimenttag, + updateexperiment, restoreexperiment, searchexperiments, getexperimentbyname, + createexperimentpermission, getexperimentpermission, updateexperimentpermission, + deleteexperimentpermission + +include("services/run.jl") +export getrun, createrun, deleterun, setruntag, updaterun, restorerun, searchruns, + deleteruntag + +include("services/logger.jl") +export logbatch, loginputs, logmetric, logparam + +include("services/artifact.jl") +export listartifacts + +include("services/misc.jl") +export refresh, getmetrichistory + +include("services/registered_model.jl") +export getregisteredmodel, createregisteredmodel, deleteregisteredmodel, + renameregisteredmodel, updateregisteredmodel, searchregisteredmodels, + setregisteredmodeltag, deleteregisteredmodeltag, deleteregisteredmodelalias, + setregisteredmodelalias, createregisteredmodelpermission, getregisteredmodelpermission, + updateregisteredmodelpermission, deleteregisteredmodelpermission + +include("services/model_version.jl") +export getlatestmodelversions, getmodelversion, createmodelversion, deletemodelversion, + updatemodelversion, searchmodelversions, getdownloaduriformodelversionartifacts, + transitionmodelversionstage, setmodelversiontag, deletemodelversiontag, + getmodelversionbyalias + +include("services/user.jl") +export createuser, getuser, updateuserpassword, updateuseradmin, deleteuser end diff --git a/src/api.jl b/src/api.jl index fb2edf8..13763be 100644 --- a/src/api.jl +++ b/src/api.jl @@ -1,34 +1,113 @@ +""" + uri(mlf::MLFlow, endpoint::String; parameters=missing) + +Retrieves an URI based on `mlf`, `endpoint`, and, optionally, `parameters`. + +# Examples +```@example +MLFlowClient.uri(mlf, "experiments/get", Dict(:experiment_id=>10)) +``` +""" +uri(mlf::MLFlow, endpoint::String; + parameters::Dict{Symbol,<:Any}=Dict{Symbol,NumberOrString}()) = + URI("$(mlf.apiroot)/$(mlf.apiversion)/mlflow/$(endpoint)"; query=parameters) + +""" + headers(mlf::MLFlow,custom_headers::AbstractDict) + +Retrieves HTTP headers based on `mlf` and merges with user-provided `custom_headers` + +# Examples +```@example +headers(mlf,Dict("Content-Type"=>"application/json")) +``` +""" +headers(mlf::MLFlow, custom_headers::AbstractDict) = merge(mlf.headers, custom_headers) + """ mlfget(mlf, endpoint; kwargs...) Performs a HTTP GET to a specified endpoint. kwargs are turned into GET params. """ function mlfget(mlf, endpoint; kwargs...) - apiuri = uri(mlf, endpoint, kwargs) - apiheaders = headers(mlf, Dict("Content-Type" => "application/json")) + apiuri = uri(mlf, endpoint; + parameters=Dict(k => v for (k, v) in kwargs if v !== missing)) + apiheaders = headers(mlf, ("Content-Type" => "application/json") |> Dict) try response = HTTP.get(apiuri, apiheaders) - return JSON.parse(String(response.body)) + return response.body |> String |> JSON.parse catch e - throw(e) + error_response = e.response.body |> String |> JSON.parse + error_message = "$(error_response["error_code"]) - $(error_response["message"])" + @error error_message + throw(ErrorException(error_message)) end end """ mlfpost(mlf, endpoint; kwargs...) -Performs a HTTP POST to the specified endpoint. kwargs are converted to JSON and become the POST body. +Performs a HTTP POST to the specified endpoint. kwargs are converted to JSON and become the +POST body. """ function mlfpost(mlf, endpoint; kwargs...) - apiuri = uri(mlf, endpoint) + apiuri = uri(mlf, endpoint;) apiheaders = headers(mlf, Dict("Content-Type" => "application/json")) body = JSON.json(kwargs) try response = HTTP.post(apiuri, apiheaders, body) - return JSON.parse(String(response.body)) + return response.body |> String |> JSON.parse + catch e + error_response = e.response.body |> String |> JSON.parse + error_message = "$(error_response["error_code"]) - $(error_response["message"])" + @error error_message + throw(ErrorException(error_message)) + end +end + +""" + mlfpatch(mlf, endpoint; kwargs...) + +Performs a HTTP PATCH to the specified endpoint. kwargs are converted to JSON and become +the PATCH body. +""" +function mlfpatch(mlf, endpoint; kwargs...) + apiuri = uri(mlf, endpoint;) + apiheaders = headers(mlf, Dict("Content-Type" => "application/json")) + body = JSON.json(kwargs) + + try + response = HTTP.patch(apiuri, apiheaders, body) + return response.body |> String |> JSON.parse catch e - throw(e) + error_response = e.response.body |> String |> JSON.parse + error_message = "$(error_response["error_code"]) - $(error_response["message"])" + @error error_message + throw(ErrorException(error_message)) end -end \ No newline at end of file +end + +""" + mlfdelete(mlf, endpoint; kwargs...) + +Performs a HTTP DELETE to the specified endpoint. kwargs are converted to JSON and become +the DELETE body. +""" +function mlfdelete(mlf, endpoint; kwargs...) + apiuri = uri(mlf, endpoint; + parameters=Dict(k => v for (k, v) in kwargs if v !== missing)) + apiheaders = headers(mlf, Dict("Content-Type" => "application/json")) + body = JSON.json(kwargs) + + try + response = HTTP.delete(apiuri, apiheaders, body) + return response.body |> String |> JSON.parse + catch e + error_response = e.response.body |> String |> JSON.parse + error_message = "$(error_response["error_code"]) - $(error_response["message"])" + @error error_message + throw(ErrorException(error_message)) + end +end diff --git a/src/deprecated.jl b/src/deprecated.jl deleted file mode 100644 index 85b9d12..0000000 --- a/src/deprecated.jl +++ /dev/null @@ -1,12 +0,0 @@ -""" - listexperiments(mlf::MLFlow) - -Returns a list of MLFlow experiments. - -Deprecated (last MLFlow version: 1.30.1) in favor of [`searchexperiments`](@ref). -""" - -function listexperiments(mlf::MLFlow) - endpoint = "experiments/list" - mlfget(mlf, endpoint) -end diff --git a/src/experiments.jl b/src/experiments.jl deleted file mode 100644 index 7b34efd..0000000 --- a/src/experiments.jl +++ /dev/null @@ -1,247 +0,0 @@ -""" - createexperiment(mlf::MLFlow; name=missing, artifact_location=missing, tags=missing) - -Creates an MLFlow experiment. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `name`: experiment name. If not specified, MLFlow sets it. -- `artifact_location`: directory where artifacts of this experiment will be stored. If not specified, MLFlow uses its default configuration. -- `tags`: a Vector of Dictionaries which tag the experiment. - - example tags: [Dict("key" => "foo", "value" => "bar"), Dict("key" => "missy", "value" => "gala")] - -# Returns -An object of type [`MLFlowExperiment`](@ref). - -""" -function createexperiment(mlf::MLFlow; name=missing, artifact_location=missing, tags=missing) - endpoint = "experiments/create" - - if ismissing(name) - name = string(UUIDs.uuid4()) - end - - try - result = mlfpost(mlf, endpoint; name=name, artifact_location=artifact_location, tags=tags) - experiment_id = parse(Int, result["experiment_id"]) - return getexperiment(mlf, experiment_id) - catch e - if isa(e, HTTP.ExceptionRequest.StatusError) && e.status == 400 - error_code = JSON.parse(String(e.response.body))["error_code"] - if error_code == MLFLOW_ERROR_CODES.RESOURCE_ALREADY_EXISTS - error("Experiment with name \"$name\" already exists") - end - end - throw(e) - end -end - -""" - getexperiment(mlf::MLFlow, experiment_id::Integer) - -Retrieves an MLFlow experiment by id. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_id`: Experiment identifier. - -# Returns -An instance of type [`MLFlowExperiment`](@ref) - -""" -function getexperiment(mlf::MLFlow, experiment_id::Integer) - try - endpoint = "experiments/get" - arguments = (:experiment_id => experiment_id,) - result = mlfget(mlf, endpoint; arguments...)["experiment"] - return MLFlowExperiment(result) - catch e - if isa(e, HTTP.ExceptionRequest.StatusError) && e.status == 404 - return missing - end - throw(e) - end -end -""" - getexperiment(mlf::MLFlow, experiment_name::String) - -Retrieves an MLFlow experiment by name. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_name`: Experiment name. - -# Returns -An instance of type [`MLFlowExperiment`](@ref) - -""" -function getexperiment(mlf::MLFlow, experiment_name::String) - try - endpoint = "experiments/get-by-name" - arguments = (:experiment_name => experiment_name,) - result = mlfget(mlf, endpoint; arguments...)["experiment"] - return MLFlowExperiment(result) - catch e - if isa(e, HTTP.ExceptionRequest.StatusError) && e.status == 404 - return missing - end - throw(e) - end -end - -""" - getorcreateexperiment(mlf::MLFlow, experiment_name::String; artifact_location=missing, tags=missing) - -Gets an experiment if one alrady exists, or creates a new one. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_name`: Experiment name. -- `artifact_location`: directory where artifacts of this experiment will be stored. If not specified, MLFlow uses its default configuration. -- `tags`: a Vector of Dictionaries which tag the experiment. - - example tags: [Dict("key" => "foo", "value" => "bar"), Dict("key" => "missy", "value" => "gala")] - -# Returns -An instance of type [`MLFlowExperiment`](@ref) - -""" -function getorcreateexperiment(mlf::MLFlow, experiment_name::String; artifact_location=missing, tags=missing) - experiment = getexperiment(mlf, experiment_name) - - if ismissing(experiment) - return createexperiment(mlf, name=experiment_name, artifact_location=artifact_location, tags=tags) - end - return experiment -end - -""" - deleteexperiment(mlf::MLFlow, experiment_id::Integer) - -Deletes an MLFlow experiment. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_id`: experiment identifier. - -# Returns - -`true` if successful. Otherwise, raises exception. -""" -function deleteexperiment(mlf::MLFlow, experiment_id::Integer) - endpoint = "experiments/delete" - try - mlfpost(mlf, endpoint; experiment_id=experiment_id) - return true - catch e - if isa(e, HTTP.ExceptionRequest.StatusError) && e.status == 404 - # experiment already deleted - return true - end - throw(e) - end -end - -""" - deleteexperiment(mlf::MLFlow, experiment::MLFlowExperiment) - -Deletes an MLFlow experiment. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment`: an object of type [`MLFlowExperiment`](@ref) - -Dispatches to `deleteexperiment(mlf::MLFlow, experiment_id::Integer)`. - -""" -deleteexperiment(mlf::MLFlow, experiment::MLFlowExperiment) = - deleteexperiment(mlf, experiment.experiment_id) - -""" - restoreexperiment(mlf::MLFlow, experiment_id::Integer) - -Restores a deleted MLFlow experiment. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_id`: experiment identifier. - -# Returns - -`true` if successful. Otherwise, raises exception. -""" -function restoreexperiment(mlf::MLFlow, experiment_id::Integer) - endpoint = "experiments/restore" - try - mlfpost(mlf, endpoint; experiment_id=experiment_id) - return true - catch e - if isa(e, HTTP.ExceptionRequest.StatusError) && e.status == 404 - error_code = JSON.parse(String(e.response.body))["error_code"] - if error_code == MLFLOW_ERROR_CODES.RESOURCE_DOES_NOT_EXIST - error("Experiment with id \"$experiment_id\" does not exist") - end - end - throw(e) - end -end - -restoreexperiment(mlf::MLFlow, experiment::MLFlowExperiment) = - restoreexperiment(mlf, experiment.experiment_id) - -""" - searchexperiments(mlf::MLFlow) - -Searches for experiments in an MLFlow instance. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. - -# Keywords -- `filter::String`: filter as defined in [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-experiments) -- `filter_attributes::AbstractDict{K,V}`: if provided, `filter` is automatically generated based on `filter_attributes` using [`generatefilterfromattributes`](@ref). One can only provide either `filter` or `filter_attributes`, but not both. -- `run_view_type::String`: one of `ACTIVE_ONLY`, `DELETED_ONLY`, or `ALL`. -- `max_results::Integer`: 50,000 by default. -- `order_by::String`: as defined in [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-experiments) -- `page_token::String`: paging functionality, handled automatically. Not meant to be passed by the user. - -# Returns -- vector of [`MLFlowExperiment`](@ref) experiments that were found in the MLFlow instance - -""" -function searchexperiments(mlf::MLFlow; - filter::String="", - filter_attributes::AbstractDict{K,V}=Dict{}(), - run_view_type::String="ACTIVE_ONLY", - max_results::Int64=50000, - order_by::AbstractVector{<:String}=["attribute.last_update_time"], - page_token::String="" -) where {K,V} - endpoint = "experiments/search" - run_view_type ∈ ["ACTIVE_ONLY", "DELETED_ONLY", "ALL"] || error("Unsupported run_view_type = $run_view_type") - - if length(filter_attributes) > 0 && length(filter) > 0 - error("Cannot specify both filter and filter_attributes") - end - - if length(filter_attributes) > 0 - filter = generatefilterfromattributes(filter_attributes) - end - - kwargs = (; filter, run_view_type, max_results, order_by) - if !isempty(page_token) - kwargs = (; kwargs..., page_token=page_token) - end - - result = mlfpost(mlf, endpoint; kwargs...) - haskey(result, "experiments") || return MLFlowExperiment[] - - experiments = map(x -> MLFlowExperiment(x), result["experiments"]) - - if haskey(result, "next_page_token") && !isempty(result["next_page_token"]) - kwargs = (; filter, run_view_type, max_results, order_by, page_token=result["next_page_token"]) - next_experiments = searchexperiments(mlf; kwargs...) - return vcat(experiments, next_experiments) - end - - experiments -end diff --git a/src/loggers.jl b/src/loggers.jl deleted file mode 100644 index b8f0639..0000000 --- a/src/loggers.jl +++ /dev/null @@ -1,264 +0,0 @@ -""" - settag(mlf::MLFlow, run, key, value) - settag(mlf::MLFlow, run, kv) - -Associates a tag (a key and a value) to the particular run. - -Refer to [the official MLflow REST API -docs](https://mlflow.org/docs/latest/rest-api.html#set-tag) for restrictions on -`key` and `value`. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref), or `String`. -- `key`: tag key (name). Automatically converted to string before sending to MLFlow because this is the only type that MLFlow supports. -- `value`: parameter value. Automatically converted to string before sending to MLFlow because this is the only type that MLFlow supports. - -One could also specify `kv::Dict` instead of separate `key` and `value` arguments. -""" -function settag(mlf::MLFlow, run_id::String, key, value) - endpoint ="runs/set-tag" - mlfpost(mlf, endpoint; run_id=run_id, key=string(key), value=string(value)) -end -settag(mlf::MLFlow, run_info::MLFlowRunInfo, key, value) = - settag(mlf, run_info.run_id, key, value) -settag(mlf::MLFlow, run::MLFlowRun, key, value) = - settag(mlf, run.info, key, value) -function settag(mlf::MLFlow, run::Union{String,MLFlowRun,MLFlowRunInfo}, kv) - for (k, v) in kv - logparam(mlf, run, k, v) - end -end - - -""" - logparam(mlf::MLFlow, run, key, value) - logparam(mlf::MLFlow, run, kv) - -Associates a key/value pair of parameters to the particular run. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref), or `String`. -- `key`: parameter key (name). Automatically converted to string before sending to MLFlow because this is the only type that MLFlow supports. -- `value`: parameter value. Automatically converted to string before sending to MLFlow because this is the only type that MLFlow supports. - -One could also specify `kv::Dict` instead of separate `key` and `value` arguments. -""" -function logparam(mlf::MLFlow, run_id::String, key, value) - endpoint = "runs/log-parameter" - mlfpost(mlf, endpoint; run_id=run_id, key=string(key), value=string(value)) -end -logparam(mlf::MLFlow, run_info::MLFlowRunInfo, key, value) = - logparam(mlf, run_info.run_id, key, value) -logparam(mlf::MLFlow, run::MLFlowRun, key, value) = - logparam(mlf, run.info, key, value) -function logparam(mlf::MLFlow, run::Union{String,MLFlowRun,MLFlowRunInfo}, kv) - for (k, v) in kv - logparam(mlf, run, k, v) - end -end - - -""" - logmetric(mlf::MLFlow, run, key, value::T; timestamp, step) where T<:Real - logmetric(mlf::MLFlow, run, key, values::AbstractArray{T}; timestamp, step) where T<:Real - -Logs a metric value (or values) against a particular run. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref), or `String` -- `key`: metric name. -- `value`: metric value, must be numeric. - -# Keywords -- `timestamp`: if provided, must be a UNIX timestamp in milliseconds. By default, set to current time. -- `step`: step at which the metric value has been taken. -""" -function logmetric(mlf::MLFlow, run_id::String, key, value::T; timestamp=missing, step=missing) where {T<:Real} - endpoint = "runs/log-metric" - if ismissing(timestamp) - timestamp = Int(trunc(datetime2unix(now(UTC)) * 1000)) - end - mlfpost(mlf, endpoint; run_id=run_id, key=key, value=value, timestamp=timestamp, step=step) -end -logmetric(mlf::MLFlow, run_info::MLFlowRunInfo, key, value::T; timestamp=missing, step=missing) where {T<:Real} = - logmetric(mlf::MLFlow, run_info.run_id, key, value; timestamp=timestamp, step=step) -logmetric(mlf::MLFlow, run::MLFlowRun, key, value::T; timestamp=missing, step=missing) where {T<:Real} = - logmetric(mlf, run.info, key, value; timestamp=timestamp, step=step) - -function logmetric(mlf::MLFlow, run::Union{String,MLFlowRun,MLFlowRunInfo}, key, values::AbstractArray{T}; timestamp=missing, step=missing) where {T<:Real} - for v in values - logmetric(mlf, run, key, v; timestamp=timestamp, step=step) - end -end - - -""" - logartifact(mlf::MLFlow, run, basefilename, data) - -Stores an artifact (file) in the run's artifact location. - -!!! note - Assumes that artifact_uri is mapped to a local directory. - At the moment, this only works if both MLFlow and the client are running on the same host or they map a directory that leads to the same location over NFS, for example. - -# Arguments -- `mlf::MLFlow`: [`MLFlow`](@ref) onfiguration. Currently not used, but when this method is extended to support `S3`, information from `mlf` will be needed. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref) or `String`. -- `basefilename`: name of the file to be written. -- `data`: artifact content, an object that can be written directly to a file handle. - -# Throws -- an `ErrorException` if an exception occurs during writing artifact. - -# Returns -path of the artifact that was created. -""" -function logartifact(mlf::MLFlow, run_id::AbstractString, basefilename::AbstractString, data) - mlflowrun = getrun(mlf, run_id) - artifact_uri = mlflowrun.info.artifact_uri - mkpath(artifact_uri) - filepath = joinpath(artifact_uri, basefilename) - try - f = open(filepath, "w") - write(f, data) - close(f) - catch e - error("Unable to create artifact $(filepath): $e") - end - filepath -end -logartifact(mlf::MLFlow, run::MLFlowRun, basefilename::AbstractString, data) = - logartifact(mlf, run.info, basefilename, data) -logartifact(mlf::MLFlow, run_info::MLFlowRunInfo, basefilename::AbstractString, data) = - logartifact(mlf, run_info.run_id, basefilename, data) - -""" - logartifact(mlf::MLFlow, run, filepath) - -Stores an artifact (file) in the run's artifact location. -The name of the artifact is calculated using `basename(filepath)`. - -Dispatches on `logartifact(mlf::MLFlow, run, basefilename, data)` where `data` is the contents of `filepath`. - -# Throws -- an `ErrorException` if `filepath` does not exist. -- an exception if such occurs while trying to read the contents of `filepath`. - -""" -function logartifact(mlf::MLFlow, run_id::AbstractString, filepath::Union{AbstractPath,AbstractString}) - isfile(filepath) || error("File $filepath does not exist.") - try - f = open(filepath, "r") - data = read(f) - close(f) - return logartifact(mlf, run_id, basename(filepath), data) - catch e - throw(e) - finally - if @isdefined f - close(f) - end - end -end -logartifact(mlf::MLFlow, run::MLFlowRun, filepath::Union{AbstractPath,AbstractString}) = - logartifact(mlf, run.info, filepath) -logartifact(mlf::MLFlow, run_info::MLFlowRunInfo, filepath::Union{AbstractPath,AbstractString}) = - logartifact(mlf, run_info.run_id, filepath) - -""" - listartifacts(mlf::MLFlow, run) - -Lists the artifacts associated with an experiment run. -According to [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#list-artifacts), this API endpoint should return paged results, similar to [`searchruns`](@ref). -However, after some experimentation, this doesn't seem to be the case. Therefore, the paging functionality is not implemented here. - -# Arguments -- `mlf::MLFlow`: [`MLFlow`](@ref) onfiguration. Currently not used, but when this method is extended to support `S3`, information from `mlf` will be needed. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref) or `String`. - -# Keywords -- `path::String`: path of a directory within the artifact location. If set, returns the contents of the directory. By default, this is the root directory of the artifacts. -- `maxdepth::Int64`: depth of listing. Default is 1. This will only return the files/directories in the current `path`. To return all artifacts files and directories, use `maxdepth=-1`. - -# Returns -A vector of `Union{MLFlowArtifactFileInfo,MLFlowArtifactDirInfo}`. -""" -function listartifacts(mlf::MLFlow, run_id::String; path::String="", maxdepth::Int64=1) - endpoint = "artifacts/list" - kwargs = ( - run_id=run_id, - ) - kwargs = (; kwargs..., path=path) - httpresult = mlfget(mlf, endpoint; kwargs...) - "files" ∈ keys(httpresult) || return Vector{Union{MLFlowArtifactFileInfo,MLFlowArtifactDirInfo}}() - "root_uri" ∈ keys(httpresult) || error("Malformed response from MLFlow REST API.") - root_uri = httpresult["root_uri"] - result = Vector{Union{MLFlowArtifactFileInfo,MLFlowArtifactDirInfo}}() - maxdepth == 0 && return result - - for resultentry ∈ httpresult["files"] - if resultentry["is_dir"] == false - filepath = joinpath(root_uri, resultentry["path"]) - file_size = resultentry["file_size"] - if typeof(file_size) <: Int - filesize = file_size - else - filesize = parse(Int, file_size) - end - push!(result, MLFlowArtifactFileInfo(filepath, filesize)) - elseif resultentry["is_dir"] == true - dirpath = joinpath(root_uri, resultentry["path"]) - push!(result, MLFlowArtifactDirInfo(dirpath)) - if maxdepth != 0 - nextdepthresult = listartifacts(mlf, run_id, path=resultentry["path"], maxdepth=maxdepth - 1) - result = vcat(result, nextdepthresult) - end - else - isdirval = resultentry["is_dir"] - @warn "Malformed response from MLFlow REST API is_dir=$isdirval - skipping" - continue - end - end - result -end -listartifacts(mlf::MLFlow, run::MLFlowRun; kwargs...) = - listartifacts(mlf, run.info.run_id; kwargs...) -listartifacts(mlf::MLFlow, run_info::MLFlowRunInfo; kwargs...) = - listartifacts(mlf, run_info.run_id; kwargs...) - -""" - logbatch(mlf::MLFlow, run_id::String, metrics, params, tags) - -Logs a batch of metrics, parameters and tags to an experiment run. - -# Arguments -- `mlf::MLFlow`: [`MLFlow`](@ref) onfiguration. -- `run_id::String`: ID of the run to log to. -- `metrics`: a vector of [`MLFlowRunDataMetric`](@ref) or a vector of -NamedTuples of `(name, value, timestamp)`. -- `params`: a vector of [`MLFlowRunDataParam`](@ref) or a vector of NamedTuples -of `(name, value)`. -- `tags`: a vector of strings. -""" -logbatch(mlf::MLFlow, run_id::String; tags=String[], metrics=Any[], - params=Any[]) = logbatch(mlf, run_id, tags, metrics, params) -function logbatch(mlf::MLFlow, run_id::String, - tags::Union{AbstractVector{<:String}, AbstractVector{Any}}, - metrics::Union{AbstractVector{<:MLFlowRunDataMetric}, AbstractVector{Any}}, - params::Union{AbstractVector{<:MLFlowRunDataParam}, AbstractVector{Any}}) - endpoint = "runs/log-batch" - mlfpost(mlf, endpoint; - run_id=run_id, metrics=metrics, params=params, tags=tags) -end -function logbatch(mlf::MLFlow, run_id::String, - tags::Union{AbstractVector{<:String}, AbstractVector{Any}}, - metrics::Union{AbstractVector{<:AbstractDict}, AbstractVector{Any}}, - params::Union{AbstractVector{<:AbstractDict}, AbstractVector{Any}}) - endpoint = "runs/log-batch" - mlfpost(mlf, endpoint; run_id=run_id, - metrics=MLFlowRunDataMetric.(metrics), - params=MLFlowRunDataParam.(params), tags=tags) -end diff --git a/src/runs.jl b/src/runs.jl deleted file mode 100644 index 1caa140..0000000 --- a/src/runs.jl +++ /dev/null @@ -1,190 +0,0 @@ -""" - createrun(mlf::MLFlow, experiment_id; run_name=missing, start_time=missing, tags=missing) - -Creates a run associated to an experiment. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_id`: experiment identifier. - -# Keywords -- `run_name`: run name. If not specified, MLFlow sets it. -- `start_time`: if provided, must be a UNIX timestamp in milliseconds. By default, set to current time. -- `tags`: if provided, must be a key-value structure such as for example: - - [Dict("key" => "foo", "value" => "bar"), Dict("key" => "missy", "value" => "gala")] - -# Returns -- an instance of type [`MLFlowRun`](@ref) -""" -function createrun(mlf::MLFlow, experiment_id; run_name=missing, start_time=missing, tags=missing) - endpoint = "runs/create" - if ismissing(start_time) - start_time = Int(trunc(datetime2unix(now(UTC)) * 1000)) - end - result = mlfpost(mlf, endpoint; experiment_id=experiment_id, run_name=run_name, start_time=start_time, tags=tags) - MLFlowRun(result["run"]["info"], result["run"]["data"]) -end -""" - createrun(mlf::MLFlow, experiment::MLFlowExperiment; run_name=missing, start_time=missing, tags=missing) - -Dispatches to `createrun(mlf::MLFlow, experiment_id; run_name=run_name, start_time=start_time, tags=tags)` -""" -createrun(mlf::MLFlow, experiment::MLFlowExperiment; run_name=missing, start_time=missing, tags=missing) = - createrun(mlf, experiment.experiment_id; run_name=run_name, start_time=start_time, tags=tags) - -""" - getrun(mlf::MLFlow, run_id) - -Retrieves information about an MLFlow run. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `run_id::String`: run identifier. - -# Returns -- an instance of type [`MLFlowRun`](@ref) -""" -function getrun(mlf::MLFlow, run_id) - endpoint = "runs/get" - result = mlfget(mlf, endpoint; run_id=run_id) - MLFlowRun(result["run"]["info"], result["run"]["data"]) -end - -""" - updaterun(mlf::MLFlow, run, status; end_time=missing) - -Updates the status of an experiment's run. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref), or `String`. -- `status`: either `String` and one of ["RUNNING", "SCHEDULED", "FINISHED", "FAILED", "KILLED"], or an instance of `MLFlowRunStatus` - -# Keywords -- `run_name`: if provided, must be a String. By default, not set. -- `end_time`: if provided, must be a UNIX timestamp in milliseconds. By default, set to current time. -""" -function updaterun(mlf::MLFlow, run_id::String, status::MLFlowRunStatus; run_name=missing, end_time=missing) - endpoint = "runs/update" - kwargs = Dict( - :run_id => run_id, - :status => status.status, - :run_name => run_name, - :end_time => end_time - ) - if ismissing(end_time) && status.status == "FINISHED" - end_time = Int(trunc(datetime2unix(now(UTC)) * 1000)) - kwargs[:end_time] = string(end_time) - end - result = mlfpost(mlf, endpoint; kwargs...) - MLFlowRun(result["run_info"]) -end -updaterun(mlf::MLFlow, run_id::String, status::String; run_name=missing, end_time=missing) = - updaterun(mlf, run_id, MLFlowRunStatus(status); run_name=run_name, end_time=end_time) -updaterun(mlf::MLFlow, run_info::MLFlowRunInfo, status::String; run_name=missing, end_time=missing) = - updaterun(mlf, run_info.run_id, MLFlowRunStatus(status); run_name=run_name, end_time=end_time) -updaterun(mlf::MLFlow, run::MLFlowRun, status::String; run_name=missing, end_time=missing) = - updaterun(mlf, run.info, MLFlowRunStatus(status); run_name=run_name, end_time=end_time) -updaterun(mlf::MLFlow, run_info::MLFlowRunInfo, status::MLFlowRunStatus; run_name=missing, end_time=missing) = - updaterun(mlf, run_info.run_id, status; run_name=run_name, end_time=end_time) -updaterun(mlf::MLFlow, run::MLFlowRun, status::MLFlowRunStatus; run_name=missing, end_time=missing) = - updaterun(mlf, run.info, status; run_name=run_name, end_time=end_time) - -""" - deleterun(mlf::MLFlow, run) - -Deletes an experiment's run. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `run`: one of [`MLFlowRun`](@ref), [`MLFlowRunInfo`](@ref), or `String`. - -# Returns -`true` if successful. - -""" -function deleterun(mlf::MLFlow, run_id::String) - endpoint = "runs/delete" - mlfpost(mlf, endpoint; run_id=run_id) - true -end -deleterun(mlf::MLFlow, run_info::MLFlowRunInfo) = deleterun(mlf, run_info.run_id) -deleterun(mlf::MLFlow, run::MLFlowRun) = deleterun(mlf, run.info) - -""" - searchruns(mlf::MLFlow, experiment_ids) - -Searches for runs in an experiment. - -# Arguments -- `mlf`: [`MLFlow`](@ref) configuration. -- `experiment_ids::AbstractVector{Integer}`: `experiment_id`s in which to search for runs. Can also be a single `Integer`. - -# Keywords -- `filter::String`: filter as defined in [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-runs) -- `filter_params::AbstractDict{K,V}`: if provided, `filter` is automatically generated based on `filter_params` using [`generatefilterfromparams`](@ref). One can only provide either `filter` or `filter_params`, but not both. -- `run_view_type::String`: one of `ACTIVE_ONLY`, `DELETED_ONLY`, or `ALL`. -- `max_results::Integer`: 50,000 by default. -- `order_by::String`: as defined in [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-runs) -- `page_token::String`: paging functionality, handled automatically. Not meant to be passed by the user. - -# Returns -- vector of [`MLFlowRun`](@ref) runs that were found in the list of experiments. - -""" -function searchruns(mlf::MLFlow, experiment_ids::AbstractVector{<:Integer}; - filter::String="", - filter_params::AbstractDict{K,V}=Dict{}(), - run_view_type::String="ACTIVE_ONLY", - max_results::Int64=50000, - order_by::AbstractVector{<:String}=["attribute.end_time"], - page_token::String="" -) where {K,V} - endpoint = "runs/search" - run_view_type ∈ ["ACTIVE_ONLY", "DELETED_ONLY", "ALL"] || error("Unsupported run_view_type = $run_view_type") - - if length(filter_params) > 0 && length(filter) > 0 - error("Can only use either filter or filter_params, but not both at the same time.") - end - - if length(filter_params) > 0 - filter = generatefilterfromparams(filter_params) - end - - kwargs = ( - experiment_ids=experiment_ids, - filter=filter, - run_view_type=run_view_type, - max_results=max_results, - order_by=order_by - ) - if !isempty(page_token) - kwargs = (; kwargs..., page_token=page_token) - end - - result = mlfpost(mlf, endpoint; kwargs...) - haskey(result, "runs") || return MLFlowRun[] - - runs = map(x -> MLFlowRun(x["info"], x["data"]), result["runs"]) - - # paging functionality using recursion - if haskey(result, "next_page_token") && !isempty(result["next_page_token"]) - kwargs = ( - filter=filter, - run_view_type=run_view_type, - max_results=max_results, - order_by=order_by, - page_token=result["next_page_token"] - ) - next_runs = searchruns(mlf, experiment_ids; kwargs...) - return vcat(runs, next_runs) - end - - runs -end -searchruns(mlf::MLFlow, experiment_id::Integer; kwargs...) = - searchruns(mlf, [experiment_id]; kwargs...) -searchruns(mlf::MLFlow, exp::MLFlowExperiment; kwargs...) = - searchruns(mlf, exp.experiment_id; kwargs...) -searchruns(mlf::MLFlow, exps::AbstractVector{MLFlowExperiment}; kwargs...) = - searchruns(mlf, getfield.(exps, :experiment_id); kwargs...) diff --git a/src/services/artifact.jl b/src/services/artifact.jl new file mode 100644 index 0000000..5c8c7de --- /dev/null +++ b/src/services/artifact.jl @@ -0,0 +1,31 @@ +""" + listartifacts(instance::MLFlow, run_id::String; path::String="", page_token::String="") + listartifacts(instance::MLFlow, run::Run; path::String="", page_token::String="") + +List artifacts for a [`Run`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) whose artifacts to list. +- `path`: Filter artifacts matching this path (a relative path from the root artifact + directory). +- `page_token`: Token indicating the page of artifact results to fetch + +# Returns +- Root artifact directory for the [`Run`](@ref). +- List of file location and metadata for artifacts. +- Token that can be used to retrieve the next page of artifact results. +""" +function listartifacts(instance::MLFlow, run_id::String; path::String="", + page_token::String="")::Tuple{String,Array{FileInfo},Union{String,Nothing}} + result = mlfget(instance, "artifacts/list"; run_id=run_id, path=path, + page_token=page_token) + + root_uri = get(result, "root_uri", "") + files = get(result, "files", []) |> (x -> [FileInfo(y) for y in x]) + next_page_token = get(result, "next_page_token", nothing) + + return root_uri, files, next_page_token +end +listartifacts(instance::MLFlow, run::Run; path::String="", page_token::String="") = + listartifacts(instance, run.info.run_id; path=path, page_token=page_token) diff --git a/src/services/experiment.jl b/src/services/experiment.jl new file mode 100644 index 0000000..f321116 --- /dev/null +++ b/src/services/experiment.jl @@ -0,0 +1,321 @@ +""" + createexperiment(instance::MLFlow, name::String; + artifact_location::Union{String, Missing}=missing, + tags::MLFlowUpsertData{Tag}=Tag[]) + +Create an [`Experiment`](@ref) with a name. Returns the newly created [`Experiment`](@ref). +Validates that another [`Experiment`](@ref) with the same name does not already exist and +fails if another [`Experiment`](@ref) with the same name already exists. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `name`: [`Experiment`](@ref) name. This field is required. +- `artifact_location`: Location where all artifacts for the [`Experiment`](@ref) + are stored. If not provided, the remote server will select an appropriate + default. +- `tags`: A collection of [`Tag`](@ref) to set on the [`Experiment`](@ref). + +# Returns +The ID of the newly created [`Experiment`](@ref). +""" +function createexperiment(instance::MLFlow, name::String; + artifact_location::Union{String,Missing}=missing, + tags::MLFlowUpsertData{Tag}=Tag[])::String + result = mlfpost(instance, "experiments/create"; name=name, + artifact_location=artifact_location, tags=parse(Tag, tags)) + return result["experiment_id"] +end + +""" + getexperiment(instance::MLFlow, experiment_id::String) + getexperiment(instance::MLFlow, experiment_id::Integer) + +Get metadata for an [`Experiment`](@ref). This method works on deleted experiments. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: ID of the associated [`Experiment`](@ref). + +# Returns +An instance of type [`Experiment`](@ref). +""" +function getexperiment(instance::MLFlow, experiment_id::String)::Experiment + result = mlfget(instance, "experiments/get"; experiment_id=experiment_id) + return result["experiment"] |> Experiment +end +getexperiment(instance::MLFlow, experiment_id::Integer)::Experiment = + getexperiment(instance, string(experiment_id)) + +""" + getexperimentbyname(instance::MLFlow, experiment_name::String) + +Get metadata for an [`Experiment`](@ref). + +This endpoint will return deleted experiments, but prefers the active [`Experiment`](@ref) +if an active and deleted [`Experiment`](@ref) share the same name. If multiple deleted +experiments share the same name, the API will return one of them. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_name`: Name of the associated [`Experiment`](@ref). + +# Returns +An instance of type [`Experiment`](@ref). +""" +function getexperimentbyname(instance::MLFlow, experiment_name::String)::Experiment + result = mlfget(instance, "experiments/get-by-name"; experiment_name=experiment_name) + return result["experiment"] |> Experiment +end + +""" + deleteexperiment(instance::MLFlow, experiment_id::String) + deleteexperiment(instance::MLFlow, experiment_id::Integer) + deleteexperiment(instance::MLFlow, experiment::Experiment) + +Mark an [`Experiment`](@ref) and associated metadata, runs, metrics, params, and tags for +deletion. If the [`Experiment`](@ref) uses FileStore, artifacts associated with +[`Experiment`](@ref) are also deleted. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: ID of the associated [`Experiment`](@ref). + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteexperiment(instance::MLFlow, experiment_id::String)::Bool + mlfpost(instance, "experiments/delete"; experiment_id=experiment_id) + return true +end +deleteexperiment(instance::MLFlow, experiment_id::Integer)::Bool = + deleteexperiment(instance, string(experiment_id)) +deleteexperiment(instance::MLFlow, experiment::Experiment)::Bool = + deleteexperiment(instance, experiment.experiment_id) + +""" + restoreexperiment(instance::MLFlow, experiment_id::String) + restoreexperiment(instance::MLFlow, experiment_id::Integer) + restoreexperiment(instance::MLFlow, experiment::Experiment) + +Restore an [`Experiment`](@ref) marked for deletion. This also restores associated +metadata, runs, metrics, params, and tags. If [`Experiment`](@ref) uses FileStore, +underlying artifacts associated with [`Experiment`](@ref) are also restored. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: ID of the associated [`Experiment`](@ref). + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function restoreexperiment(instance::MLFlow, experiment_id::String)::Bool + mlfpost(instance, "experiments/restore"; experiment_id=experiment_id) + return true +end +restoreexperiment(instance::MLFlow, experiment_id::Integer)::Bool = + restoreexperiment(instance, string(experiment_id)) +restoreexperiment(instance::MLFlow, experiment::Experiment)::Bool = + restoreexperiment(instance, experiment.experiment_id) + +""" + updateexperiment(instance::MLFlow, experiment_id::String, new_name::String) + updateexperiment(instance::MLFlow, experiment_id::Integer, new_name::String) + updateexperiment(instance::MLFlow, experiment::Experiment, new_name::String) + +Update [`Experiment`](@ref) metadata. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: ID of the associated [`Experiment`](@ref). +- `new_name`: If provided, the [`Experiment`](@ref) name is changed to the new name. The new name + must be unique. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function updateexperiment(instance::MLFlow, experiment_id::String, new_name::String)::Bool + mlfpost(instance, "experiments/update"; experiment_id=experiment_id, new_name=new_name) + return true +end +updateexperiment(instance::MLFlow, experiment_id::Integer, new_name::String)::Bool = + updateexperiment(instance, string(experiment_id), new_name) +updateexperiment(instance::MLFlow, experiment::Experiment, new_name::String)::Bool = + updateexperiment(instance, experiment.experiment_id, new_name) + +""" + searchexperiments(instance::MLFlow; max_results::Int64=20000, page_token::String="", + filter::String="", order_by::Array{String}=String[], + view_type::ViewType=ACTIVE_ONLY) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `max_results`: Maximum number of experiments desired. +- `page_token`: Token indicating the page of experiments to fetch. +- `filter`: A filter expression over [`Experiment`](@ref) attributes and tags that allows returning a + subset of experiments. See [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-experiments). +- `order_by`: List of columns for ordering search results, which can include [`Experiment`](@ref) + name and id with an optional “DESC” or “ASC” annotation, where “ASC” is the default. +- `view_type`: Qualifier for type of experiments to be returned. If unspecified, return + only active experiments. For more values, see [`ViewType`](@ref). + +# Returns +- Vector of [`Experiment`](@ref) that were found in the [`MLFlow`](@ref) instance. +- The next page token if there are more results. +""" +function searchexperiments(instance::MLFlow; max_results::Int64=20000, + page_token::String="", filter::String="", order_by::Array{String}=String[], + view_type::ViewType=ACTIVE_ONLY)::Tuple{Array{Experiment},Union{String,Nothing}} + parameters = (; max_results, page_token, filter, :view_type => view_type |> Integer) + + if order_by |> !isempty + parameters = (; order_by, parameters...) + end + + result = mlfget(instance, "experiments/search"; parameters...) + + experiments = get(result, "experiments", []) |> (x -> [Experiment(y) for y in x]) + next_page_token = get(result, "next_page_token", nothing) + + return experiments, next_page_token +end + +""" + setexperimenttag(instance::MLFlow, experiment_id::String, key::String, value::String) + setexperimenttag(instance::MLFlow, experiment_id::Integer, key::String, value::String) + setexperimenttag(instance::MLFlow, experiment::Experiment, key::String, value::String) + +Set a tag on an [`Experiment`](@ref). [`Experiment`](@ref) tags are metadata that can be +updated. + +# Arguments +- `experiment_id`: ID of the [`Experiment`](@ref) under which to log the tag. +- `key`: Name of the tag. +- `value`: String value of the tag being logged. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function setexperimenttag(instance::MLFlow, experiment_id::String, key::String, + value::String)::Bool + mlfpost(instance, "experiments/set-experiment-tag"; experiment_id=experiment_id, + key=key, value=value) + return true +end +setexperimenttag(instance::MLFlow, experiment_id::Integer, key::String, + value::String)::Bool = + setexperimenttag(instance, string(experiment_id), key, value) +setexperimenttag(instance::MLFlow, experiment::Experiment, key::String, + value::String)::Bool = + setexperimenttag(instance, experiment.experiment_id, key, value) + +""" + createexperimentpermission(instance::MLFlow, experiment_id::String, username::String, + permission::Permission) + createexperimentpermission(instance::MLFlow, experiment_id::Integer, username::String, + permission::Permission) + createexperimentpermission(instance::MLFlow, experiment::Experiment, username::String, + permission::Permission) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: [`Experiment`](@ref) id. +- `username`: [`User`](@ref) username. +- `permission`: [`Permission`](@ref) to grant. + +# Returns +An instance of type [`ExperimentPermission`](@ref). +""" +function createexperimentpermission(instance::MLFlow, experiment_id::String, + username::String, permission::Permission)::ExperimentPermission + result = mlfpost(instance, "experiments/permissions/create"; + experiment_id=experiment_id, username=username, permission=permission) + return result["experiment_permission"] |> ExperimentPermission +end +createexperimentpermission(instance::MLFlow, experiment_id::Integer, + username::String, permission::Permission)::ExperimentPermission = + createexperimentpermission(instance, experiment_id |> string, username, permission) +createexperimentpermission(instance::MLFlow, experiment::Experiment, + username::String, permission::Permission)::ExperimentPermission = + createexperimentpermission(instance, experiment.experiment_id, username, permission) + +""" + getexperimentpermission(instance::MLFlow, experiment_id::String, username::String) + getexperimentpermission(instance::MLFlow, experiment_id::Integer, username::String) + getexperimentpermission(instance::MLFlow, experiment::Experiment, username::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: [`Experiment`](@ref) id. +- `username`: [`User`](@ref) username. + +# Returns +An instance of type [`ExperimentPermission`](@ref). +""" +function getexperimentpermission(instance::MLFlow, experiment_id::String, + username::String)::ExperimentPermission + result = mlfget(instance, "experiments/permissions/get"; experiment_id=experiment_id, + username=username) + return result["experiment_permission"] |> ExperimentPermission +end +getexperimentpermission(instance::MLFlow, experiment_id::Integer, + username::String)::ExperimentPermission = + getexperimentpermission(instance, experiment_id |> string, username) +getexperimentpermission(instance::MLFlow, experiment::Experiment, + username::String)::ExperimentPermission = + getexperimentpermission(instance, experiment.experiment_id, username) + +""" + updateexperimentpermission(instance::MLFlow, experiment_id::String, username::String, + permission::Permission) + updateexperimentpermission(instance::MLFlow, experiment_id::Integer, username::String, + permission::Permission) + updateexperimentpermission(instance::MLFlow, experiment::Experiment, username::String, + permission::Permission) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: [`Experiment`](@ref) id. +- `username`: [`User`](@ref) username. +- `permission`: [`Permission`](@ref) to grant. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function updateexperimentpermission(instance::MLFlow, experiment_id::String, + username::String, permission::Permission)::Bool + mlfpatch(instance, "experiments/permissions/update"; experiment_id=experiment_id, + username=username, permission=permission) + return true +end +updateexperimentpermission(instance::MLFlow, experiment_id::Integer, + username::String, permission::Permission)::Bool = + updateexperimentpermission(instance, experiment_id |> string, username, permission) +updateexperimentpermission(instance::MLFlow, experiment::Experiment, + username::String, permission::Permission)::Bool = + updateexperimentpermission(instance, experiment.experiment_id, username, permission) + +""" + deleteexperimentpermission(instance::MLFlow, experiment_id::String, username::String) + deleteexperimentpermission(instance::MLFlow, experiment_id::Integer, username::String) + deleteexperimentpermission(instance::MLFlow, experiment::Experiment, username::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: [`Experiment`](@ref) id. +- `username`: [`User`](@ref) username. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteexperimentpermission(instance::MLFlow, experiment_id::String, + username::String)::Bool + mlfdelete(instance, "experiments/permissions/delete"; experiment_id=experiment_id, + username=username) + return true +end +deleteexperimentpermission(instance::MLFlow, experiment_id::Integer, + username::String)::Bool = + deleteexperimentpermission(instance, experiment_id |> string, username) +deleteexperimentpermission(instance::MLFlow, experiment::Experiment, + username::String)::Bool = + deleteexperimentpermission(instance, experiment.experiment_id, username) diff --git a/src/services/logger.jl b/src/services/logger.jl new file mode 100644 index 0000000..43c2a39 --- /dev/null +++ b/src/services/logger.jl @@ -0,0 +1,125 @@ +""" + logmetric(instance::MLFlow, run_id::String, key::String, value::Float64; + timestamp::Int64=round(Int, now() |> datetime2unix), + step::Union{Int64, Missing}=missing) + logmetric(instance::MLFlow, run::Run, key::String, value::Float64; + timestamp::Int64=round(Int, now() |> datetime2unix), + step::Union{Int64, Missing}=missing) + +Log a [`Metric`](@ref) for a [`Run`](@ref). A [`Metric`](@ref) is a key-value pair (string +key, float value) with an associated timestamp. Examples include the various metrics that +represent ML model accuracy. A [`Metric`](@ref) can be logged multiple times. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) under which to log the [`Metric`](@ref). +- `key`: Name of the [`Metric`](@ref). +- `value`: Double value of the [`Metric`](@ref) being logged. +- `timestamp`: Unix timestamp in milliseconds at the time [`Metric`](@ref) was logged. +- `step`: Step at which to log the [`Metric`](@ref). + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function logmetric(instance::MLFlow, run_id::String, key::String, value::Float64; + timestamp::Int64=round(Int, now() |> datetime2unix), + step::Union{Int64,Missing}=missing)::Bool + mlfpost(instance, "runs/log-metric"; run_id=run_id, key=key, value=value, + timestamp=timestamp, step=step) + return true +end +logmetric(instance::MLFlow, run::Run, key::String, value::Float64; + timestamp::Int64=round(Int, now() |> datetime2unix), + step::Union{Int64,Missing}=missing)::Bool = + logmetric(instance, run.info.run_id, key, value; timestamp=timestamp, step=step) +logmetric(instance::MLFlow, run_id::String, metric::Metric)::Bool = + logmetric(instance, run_id, metric.key, metric.value, timestamp=metric.timestamp, + step=metric.step) +logmetric(instance::MLFlow, run::Run, metric::Metric)::Bool = + logmetric(instance, run.info.run_id, metric.key, metric.value, + timestamp=metric.timestamp, step=metric.step) + +""" + logbatch(instance::MLFlow, run_id::String; metrics::MLFlowUpsertData{Metric}, + params::MLFlowUpsertData{Param}, tags::MLFlowUpsertData{Tag}) + logbatch(instance::MLFlow, run::Run; metrics::Array{Metric}, + params::MLFlowUpsertData{Param}, tags::MLFlowUpsertData{Tag}) + +Log a batch of metrics, params, and tags for a [`Run`](@ref). In case of error, partial +data may be written. + +For more information about this function, check [MLFlow official documentation](https://mlflow.org/docs/latest/rest-api.html#log-batch). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) to log under. +- `metrics`: A collection of [`Metric`](@ref) to log. +- `params`: A collection of [`Param`](@ref) to log. +- `tags`: A collection of [`Tag`](@ref) to log. + +**Note**: A single request can contain up to 1000 metrics, and up to 1000 metrics, params, +and tags in total. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function logbatch(instance::MLFlow, run_id::String; + metrics::MLFlowUpsertData{Metric}=Metric[], params::MLFlowUpsertData{Param}=Param[], + tags::MLFlowUpsertData{Tag}=Tag[])::Bool + mlfpost(instance, "runs/log-batch"; run_id=run_id, metrics=parse(Metric, metrics), + params=parse(Param, params), tags=parse(Tag, tags)) + return true +end +logbatch(instance::MLFlow, run::Run; metrics::MLFlowUpsertData{Metric}=Metric[], + params::MLFlowUpsertData{Param}=Param[], tags::MLFlowUpsertData{Tag}=Tag[])::Bool = + logbatch(instance, run.info.run_id; metrics=metrics, params=params, tags=tags) + +""" + loginputs(instance::MLFlow, run_id::String; datasets::Array{DatasetInput}) + loginputs(instance::MLFlow, run::Run; datasets::Array{DatasetInput}) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) to log under this field is required. +- `datasets`: A collection of [`DatasetInput`](@ref) to log. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function loginputs(instance::MLFlow, run_id::String, datasets::Array{DatasetInput})::Bool + mlfpost(instance, "runs/log-inputs"; run_id=run_id, datasets=datasets) + return true +end +loginputs(instance::MLFlow, run::Run, datasets::Array{DatasetInput})::Bool = + loginputs(instance, run.info.run_id, datasets) + +""" + logparam(instance::MLFlow, run_id::String, key::String, value::String) + logparam(instance::MLFlow, run::Run, key::String, value::String) + logparam(instance::MLFlow, run_id::String, param::Param) + logparam(instance::MLFlow, run::Run, param::Param) + +Log a [`Param`](@ref) used for a [`Run`](@ref). A [`Param`](@ref) is a key-value pair +(string key, string value). Examples include hyperparameters used for ML model training and +constant dates and values used in an ETL pipeline. A [`Param`](@ref) can be logged only +once for a [`Run`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) under which to log the [`Param`](@ref). +- `key`: Name of the [`Param`](@ref). +- `value`: String value of the [`Param`](@ref) being logged. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function logparam(instance::MLFlow, run_id::String, key::String, value::String)::Bool + mlfpost(instance, "runs/log-parameter"; run_id=run_id, key=key, value=value) + return true +end +logparam(instance::MLFlow, run::Run, key::String, value::String)::Bool = + logparam(instance, run.info.run_id, key, value) +logparam(instance::MLFlow, run_id::String, param::Param)::Bool = + logparam(instance, run_id, param.key, param.value) +logparam(instance::MLFlow, run::Run, param::Param)::Bool = + logparam(instance, run.info.run_id, param.key, param.value) diff --git a/src/services/misc.jl b/src/services/misc.jl new file mode 100644 index 0000000..c7a023a --- /dev/null +++ b/src/services/misc.jl @@ -0,0 +1,58 @@ +""" + getmetrichistory(instance::MLFlow, run_id::String, metric_key::String; + page_token::String="", max_results::Union{Int64, Missing}=missing) + +Get a list of all values for the specified [`Metric`](@ref) for a given [`Run`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) from which to fetch [`Metric`](@ref) values. +- `metric_key`: Name of the [`Metric`](@ref) to fetch. +- `page_token`: Token indicating the page of [`Metric`](@ref) history to fetch. +- `max_results`: Maximum number of logged instances of a [`Metric`](@ref) for a + [`Run`](@ref) to return per call. + +# Returns +- A list of all historical values for the specified [`Metric`](@ref) in the specified + [`Run`](@ref). +- The next page token if there are more results. +""" +function getmetrichistory(instance::MLFlow, run_id::String, metric_key::String; + page_token::String="", max_results::Union{Int64,Missing}=missing +)::Tuple{Array{Metric},Union{String,Nothing}} + result = mlfget(instance, "metrics/get-history"; run_id=run_id, metric_key=metric_key, + page_token=page_token, + max_results=(ismissing(max_results) ? max_results : (max_results |> Int32))) + + metrics = result["metrics"] |> (x -> [Metric(y) for y in x]) + next_page_token = get(result, "next_page_token", nothing) + + return metrics, next_page_token +end +getmetrichistory(instance::MLFlow, run::Run, metric_key::String; page_token::String="", + max_results::Union{Int64,Missing}=missing +)::Tuple{Array{Metric},Union{String,Nothing}} = + getmetrichistory(instance, run.info.run_id, metric_key; page_token=page_token, + max_results=max_results) +getmetrichistory(instance::MLFlow, run::Run, metric::Metric; page_token::String="", + max_results::Union{Int64,Missing}=missing +)::Tuple{Array{Metric},Union{String,Nothing}} = + getmetrichistory(instance, run.info.run_id, metric.key; page_token=page_token, + max_results=max_results) + +""" + refresh(instance::MLFlow, run::Run) + refresh(instance::MLFlow, experiment::Experiment) + +Get the latest metadata for a [`Run`](@ref) or [`Experiment`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run` or `experiment`: [`Run`](@ref) or [`Experiment`](@ref) to refresh. + +# Returns +An instance of type [`Run`](@ref) or [`Experiment`](@ref). +""" +refresh(instance::MLFlow, experiment::Experiment)::Experiment = + getexperiment(instance, experiment.experiment_id) +refresh(instance::MLFlow, run::Run)::Run = getrun(instance, run.info.run_id) diff --git a/src/services/model_version.jl b/src/services/model_version.jl new file mode 100644 index 0000000..2bf0ff0 --- /dev/null +++ b/src/services/model_version.jl @@ -0,0 +1,226 @@ +""" + getlatestmodelversions(instance::MLFlow, name::String; + stages::Array{String}=String[]) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `stages:` List of stages. + +# Returns +Latest [`ModelVersion`](@ref) for each requests stage. +""" +function getlatestmodelversions(instance::MLFlow, name::String; + stages::Array{String}=String[])::Array{ModelVersion} + result = mlfpost(instance, "registered-models/get-latest-versions"; name=name, + stages=stages) + return result["model_versions"] .|> ModelVersion +end + +""" + createmodelversion(instance::MLFlow, name::String, source::String; + run_id::Union{String, Missing}=missing, tags::MLFlowUpsertData{Tag}=Tag[], + run_link::Union{String, Missing}=missing, + description::Union{String, Missing}=missing) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Register model under this name. +- `source:` URI indicating the location of the model artifacts. +- `run_id`: [`Run`](@ref) id for correlation. +- `tags:` List of [`Tag`](@ref) to associate with the model version. +- `run_link:` Link to the [`Run`](@ref) that generated the [`ModelVersion`](@ref). +- `description:` Optional description for [`ModelVersion`](@ref). + +# Returns +[`ModelVersion`](@ref) created. +""" +function createmodelversion(instance::MLFlow, name::String, source::String; + run_id::Union{String,Missing}=missing, tags::MLFlowUpsertData{Tag}=Tag[], + run_link::Union{String,Missing}=missing, + description::Union{String,Missing}=missing)::ModelVersion + result = mlfpost(instance, "model-versions/create"; name=name, source=source, + run_id=run_id, tags=parse(Tag, tags), run_link=run_link, description=description) + return result["model_version"] |> ModelVersion +end + +""" + getmodelversion(instance::MLFlow, name::String, version::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `version:` [`ModelVersion`](@ref) number. + +# Returns +[`ModelVersion`](@ref) requested. +""" +function getmodelversion(instance::MLFlow, name::String, version::String)::ModelVersion + result = mlfget(instance, "model-versions/get"; name=name, version=version) + return result["model_version"] |> ModelVersion +end + +""" + updatemodelversion(instance::MLFlow, name::String, version::String; + description::Union{String, Missing}=missing) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `version:` [`ModelVersion`](@ref) number. +- `description:` Optional description for [`ModelVersion`](@ref). + +# Returns +[`ModelVersion`](@ref) generated for this model in registry. +""" +function updatemodelversion(instance::MLFlow, name::String, version::String; + description::Union{String,Missing}=missing)::ModelVersion + result = mlfpatch(instance, "model-versions/update"; name=name, version=version, + description=description) + return result["model_version"] |> ModelVersion +end + +""" + deletemodelversion(instance::MLFlow, name::String, version::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `version:` [`ModelVersion`](@ref) number. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deletemodelversion(instance::MLFlow, name::String, version::String)::Bool + mlfdelete(instance, "model-versions/delete"; name=name, version=version) + return true +end + +""" + searchmodelversions(instance::MLFlow, filter::String, max_results::Int64, + order_by::String, page_token::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `filter`: String filter condition. See [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-modelversions). +- `max_results`: Maximum number of models desired. +- `order_by`: List of columns to be ordered by including model name, version, stage with an + optional “DESC” or “ASC” annotation, where “ASC” is the default. Tiebreaks are done by + latest stage transition timestamp, followed by name ASC, followed by version DESC. +- `page_token`: Pagination token to go to next page based on previous search query. + +# Returns +- Vector of [`ModelVersion`](@ref) that were found in the [`MLFlow`](@ref) instance. +- The next page token if there are more results. +""" +function searchmodelversions(instance::MLFlow; filter::String="", + max_results::Int64=200000, order_by::Array{String}=String[], + page_token::String="")::Tuple{Array{ModelVersion},Union{String,Nothing}} + parameters = (; max_results, page_token, filter) + + if order_by |> !isempty + parameters = (; order_by, parameters...) + end + + result = mlfget(instance, "model-versions/search"; parameters...) + + model_versions = get(result, "model_versions", []) |> (x -> [ModelVersion(y) for y in x]) + next_page_token = get(result, "next_page_token", nothing) + + return model_versions, next_page_token +end + +""" + getdownloaduriformodelversionartifacts(instance::MLFlow, name::String, version::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `version:` [`ModelVersion`](@ref) number. + +# Returns +URI corresponding to where artifacts for this [`ModelVersion`](@ref) are stored. +""" +function getdownloaduriformodelversionartifacts(instance::MLFlow, name::String, + version::String)::String + result = mlfget(instance, "model-versions/get-download-uri"; name=name, version=version) + return result["artifact_uri"] +end + +""" + transitionmodelversionstage(instance::MLFlow, name::String, version::String, + stage::String, archive_existing_versions::Bool) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `version:` [`ModelVersion`](@ref) number. +- `stage:` Transition [`ModelVersion`](@ref) to new stage. +- `archive_existing_versions:` When transitioning a model version to a particular stage, + this flag dictates whether all existing model versions in that stage should be atomically + moved to the “archived” stage. This ensures that at-most-one model version exists in the + target stage. + +# Returns +Updated [`ModelVersion`](@ref). +""" +function transitionmodelversionstage(instance::MLFlow, name::String, version::String, + stage::String, archive_existing_versions::Bool)::ModelVersion + result = mlfpost(instance, "model-versions/transition-stage"; name=name, + version=version, stage=stage, archive_existing_versions=archive_existing_versions) + return result["model_version"] |> ModelVersion +end + +""" + setmodelversiontag(instance::MLFlow, name::String, key::String, value::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Unique name of the model. +- `version:` Model version number. +- `key:` Name of the [`Tag`](@ref). +- `value:` String value of the tag being logged. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function setmodelversiontag(instance::MLFlow, name::String, version::String, key::String, + value::String)::Bool + mlfpost(instance, "model-versions/set-tag"; name=name, version=version, key=key, + value=value) + return true +end + +""" + deletemodelversiontag(instance::MLFlow, name::String, version::String, key::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref) that the tag was logged under. +- `version:` [`ModelVersion`](@ref) number that the tag was logged under. +- `key:` Name of the [`Tag`](@ref). + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deletemodelversiontag(instance::MLFlow, name::String, version::String, + key::String)::Bool + mlfdelete(instance, "model-versions/delete-tag"; name=name, version=version, key=key) + return true +end + +""" + getmodelversionbyalias(instance::MLFlow, name::String, alias::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `alias:` Name of the alias. + +# Returns +[`ModelVersion`](@ref) requested. +""" +function getmodelversionbyalias(instance::MLFlow, name::String, + alias::String)::ModelVersion + result = mlfget(instance, "registered-models/alias"; name=name, alias=alias) + return result["model_version"] |> ModelVersion +end diff --git a/src/services/registered_model.jl b/src/services/registered_model.jl new file mode 100644 index 0000000..e7b99df --- /dev/null +++ b/src/services/registered_model.jl @@ -0,0 +1,268 @@ +""" + createregisteredmodel(instance::MLFlow, name::String; + tags::MLFlowUpsertData{Tag}=Tag[], description::Union{String, Missing}=missing) + +Create a [`RegisteredModel`](@ref) with a name. Returns the newly created +[`RegisteredModel`](@ref). Validates that another [`RegisteredModel`](@ref) with the same +name does not already exist and fails if another [`RegisteredModel`](@ref) with the same +name already exists. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `name`: Register models under this name. +- `tags`: A collection of [`Tag`](@ref). +- `description`: Optional description for [`RegisteredModel`](@ref). + +# Returns +An instance of type [`RegisteredModel`](@ref). +""" +function createregisteredmodel(instance::MLFlow, name::String; + tags::MLFlowUpsertData{Tag}=Tag[], + description::Union{String,Missing}=missing)::RegisteredModel + result = mlfpost(instance, "registered-models/create"; name=name, + tags=parse(Tag, tags), description=description) + return result["registered_model"] |> RegisteredModel +end + +""" + getregisteredmodel(instance::MLFlow, name::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `name`: [`RegisteredModel`](@ref) model unique name identifier. + +# Returns +An instance of type [`RegisteredModel`](@ref). +""" +function getregisteredmodel(instance::MLFlow, name::String)::RegisteredModel + result = mlfget(instance, "registered-models/get"; name=name) + return result["registered_model"] |> RegisteredModel +end + +""" + renameregisteredmodel(instance::MLFlow, name::String, new_name::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `name`: [`RegisteredModel`](@ref) unique name identifier. +- `new_name`: If provided, updates the name for this [`RegisteredModel`](@ref). + +# Returns +An instance of type [`RegisteredModel`](@ref). +""" +function renameregisteredmodel(instance::MLFlow, name::String, + new_name::String)::RegisteredModel + result = mlfpost(instance, "registered-models/rename"; name=name, new_name=new_name) + return result["registered_model"] |> RegisteredModel +end + +""" + updateregisteredmodel(instance::MLFlow, name::String; + description::Union{String, Missing}=missing) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `name`: [`RegisteredModel`](@ref) unique name identifier. +- `description`: If provided, updates the description for this [`RegisteredModel`](@ref). + +# Returns +An instance of type [`RegisteredModel`](@ref). +""" +function updateregisteredmodel(instance::MLFlow, name::String; + description::Union{String,Missing}=missing)::RegisteredModel + result = mlfpatch(instance, "registered-models/update"; name=name, + description=description) + return result["registered_model"] |> RegisteredModel +end + +""" + deleteregisteredmodel(instance::MLFlow, name::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `name`: [`RegisteredModel`](@ref) unique name identifier. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteregisteredmodel(instance::MLFlow, name::String)::Bool + mlfdelete(instance, "registered-models/delete"; name=name) + return true +end + +""" + searchregisteredmodels(instance::MLFlow, filter::String, max_results::Int64, + order_by::String, page_token::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `filter`: String filter condition. See [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-registeredmodels). +- `max_results`: Maximum number of models desired. +- `order_by`: List of columns for ordering search results, which can include model name + and last updated timestamp with an optional “DESC” or “ASC” annotation, where “ASC” is + the default. Tiebreaks are done by model name ASC. +- `page_token`: Pagination token to go to the next page based on a previous search query. + +# Returns +- Vector of [`RegisteredModel`](@ref) that were found in the [`MLFlow`](@ref) instance. +- The next page token if there are more results. +""" +function searchregisteredmodels(instance::MLFlow; filter::String="", + max_results::Int64=100, order_by::Array{String}=String[], + page_token::String="")::Tuple{Array{RegisteredModel},Union{String,Nothing}} + parameters = (; max_results, page_token, filter) + + if order_by |> !isempty + parameters = (; order_by, parameters...) + end + + result = mlfget(instance, "registered-models/search"; parameters...) + + registered_models = get(result, "registered_models", []) |> (x -> [RegisteredModel(y) for y in x]) + next_page_token = get(result, "next_page_token", nothing) + + return registered_models, next_page_token +end + +""" + setregisteredmodeltag(instance::MLFlow, name::String, key::String, value::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Unique name of the model. +- `key:` Name of the [`Tag`](@ref). +- `value:` String value of the [`Tag`](@ref) being logged. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function setregisteredmodeltag(instance::MLFlow, name::String, key::String, value::String)::Bool + mlfpost(instance, "registered-models/set-tag"; name=name, key=key, value=value) + return true +end + +""" + deleteregisteredmodeltag(instance::MLFlow, name::String, key::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref) that the tag was logged under. +- `key:` Name of the [`Tag`](@ref). + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteregisteredmodeltag(instance::MLFlow, name::String, key::String)::Bool + mlfdelete(instance, "registered-models/delete-tag"; name=name, key=key) + return true +end + +""" + deleteregisteredmodelalias(instance::MLFlow, name::String, alias::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `alias:` Name of the alias. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteregisteredmodelalias(instance::MLFlow, name::String, alias::String)::Bool + mlfdelete(instance, "registered-models/alias"; name=name, alias=alias) + return true +end + +""" + setregisteredmodelalias(instance::MLFlow, name::String, alias::String, version::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` Name of the [`RegisteredModel`](@ref). +- `alias:` Name of the alias. +- `version:` [`ModelVersion`](@ref) number. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function setregisteredmodelalias(instance::MLFlow, name::String, alias::String, + version::String)::Bool + mlfpost(instance, "registered-models/alias"; name=name, alias=alias, version=version) + return true +end + +""" + createregisteredmodelpermission(instance::MLFlow, name::String, username::String, + permission::Permission) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` [`RegisteredModel`](@ref) name. +- `username:` [`User`](@ref) username. +- `permission:` [`Permission`](@ref) to grant. + +# Returns +An instance of type [`RegisteredModelPermission`](@ref). +""" +function createregisteredmodelpermission(instance::MLFlow, name::String, username::String, + permission::Permission)::RegisteredModelPermission + result = mlfpost(instance, "registered-models/permissions/create"; name=name, + username=username, permission=permission) + return result["registered_model_permission"] |> RegisteredModelPermission +end + +""" + getregisteredmodelpermission(instance::MLFlow, name::String, username::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` [`RegisteredModel`](@ref) name. +- `username:` [`User`](@ref) username. + +# Returns +An instance of type [`RegisteredModelPermission`](@ref). +""" +function getregisteredmodelpermission(instance::MLFlow, name::String, + username::String)::RegisteredModelPermission + result = mlfget(instance, "registered-models/permissions/get"; name=name, + username=username) + return result["registered_model_permission"] |> RegisteredModelPermission +end + +""" + updateregisteredmodelpermission(instance::MLFlow, name::String, username::String, + permission::Permission) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` [`RegisteredModel`](@ref) name. +- `username:` [`User`](@ref) username. +- `permission:` New [`Permission`](@ref) to grant. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function updateregisteredmodelpermission(instance::MLFlow, name::String, username::String, + permission::Permission)::Bool + mlfpatch(instance, "registered-models/permissions/update"; name=name, username=username, + permission=permission) + return true +end + +""" + deleteregisteredmodelpermission(instance::MLFlow, name::String, username::String) + +# Arguments +- `instance:` [`MLFlow`](@ref) configuration. +- `name:` [`RegisteredModel`](@ref) name. +- `username:` [`User`](@ref) username. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteregisteredmodelpermission(instance::MLFlow, name::String, + username::String)::Bool + mlfdelete(instance, "registered-models/permissions/delete"; name=name, + username=username) + return true +end diff --git a/src/services/run.jl b/src/services/run.jl new file mode 100644 index 0000000..c2d5b5c --- /dev/null +++ b/src/services/run.jl @@ -0,0 +1,219 @@ +""" + createrun(instance::MLFlow, experiment_id::String; + run_name::Union{String, Missing}=missing, + start_time::Union{Int64, Missing}=missing, + tags::Union{Dict{<:Any}, Array{<:Any}}=[]) + +Create a new [`Run`](@ref) within an [`Experiment`](@ref). A [`Run`](@ref) is usually a +single execution of a machine learning or data ETL pipeline. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_id`: ID of the associated [`Experiment`](@ref). +- `run_name`: Name of the [`Run`](@ref). +- `start_time`: Unix timestamp in milliseconds of when the [`Run`](@ref) started. +- `tags`: Additional metadata for [`Run`](@ref). + +# Returns +An instance of type [`Run`](@ref). +""" +function createrun(instance::MLFlow, experiment_id::String; + run_name::Union{String,Missing}=missing, start_time::Union{Int64,Missing}=missing, + tags::MLFlowUpsertData{Tag}=Tag[])::Run + result = mlfpost(instance, "runs/create"; experiment_id=experiment_id, + run_name=run_name, start_time=start_time, tags=parse(Tag, tags)) + return result["run"] |> Run +end +createrun(instance::MLFlow, experiment_id::Integer; + run_name::Union{String,Missing}=missing, start_time::Union{Integer,Missing}=missing, + tags::MLFlowUpsertData{Tag}=Tag[])::Run = + createrun(instance, string(experiment_id); run_name=run_name, start_time=start_time, + tags=tags) +createrun(instance::MLFlow, experiment::Experiment; + run_name::Union{String,Missing}=missing, start_time::Union{Integer,Missing}=missing, + tags::MLFlowUpsertData{Tag}=Tag[])::Run = + createrun(instance, string(experiment.experiment_id); run_name=run_name, + start_time=start_time, tags=tags) + +""" + deleterun(instance::MLFlow, run_id::String) + deleterun(instance::MLFlow, run::Run) + +Mark a [`Run`](@ref) for deletion. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) to delete. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleterun(instance::MLFlow, run_id::String)::Bool + mlfpost(instance, "runs/delete"; run_id=run_id) + return true +end +deleterun(instance::MLFlow, run::Run)::Bool = + deleterun(instance, run.info.run_id) + +""" + restorerun(instance::MLFlow, run_id::String) + restorerun(instance::MLFlow, run::Run) + +Restore a deleted [`Run`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) to restore. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function restorerun(instance::MLFlow, run_id::String)::Bool + mlfpost(instance, "runs/restore"; run_id=run_id) + return true +end +restorerun(instance::MLFlow, run::Run)::Bool = + restorerun(instance, run.info.run_id) + +""" + getrun(instance::MLFlow, run_id::String) + +Get metadata, metrics, params, and tags for a [`Run`](@ref). In the case where multiple +metrics with the same key are logged for a [`Run`](@ref), return only the value with the +latest timestamp. If there are multiple values with the latest timestamp, return the +maximum of these values. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) to fetch. + +# Returns +An instance of type [`Run`](@ref). +""" +function getrun(instance::MLFlow, run_id::String)::Run + result = mlfget(instance, "runs/get"; run_id=run_id) + return result["run"] |> Run +end + +""" + setruntag(instance::MLFlow, run_id::String, key::String, value::String) + setruntag(instance::MLFlow, run::Run, key::String, value::String) + setruntag(instance::MLFlow, run::Run, tag::Tag) + +Set a [`Tag`](@ref) on a [`Run`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) under which to log the [`Tag`](@ref). +- `key`: Name of the [`Tag`](@ref). +- `value`: String value of the [`Tag`](@ref) being logged. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function setruntag(instance::MLFlow, run_id::String, key::String, value::String) + :Bool + mlfpost(instance, "runs/set-tag"; run_id=run_id, key=key, value=value) + return true +end +setruntag(instance::MLFlow, run::Run, key::String, value::String)::Bool = + setruntag(instance, run.info.run_id, key, value) +setruntag(instance::MLFlow, run::Run, tag::Tag)::Bool = + setruntag(instance, run.info.run_id, tag.key, tag.value) + +""" + deleteruntag(instance::MLFlow, run_id::String, key::String) + deleteruntag(instance::MLFlow, run::Run, key::String) + deleteruntag(instance::MLFlow, run::Run, tag::Tag) + +Delete a [`Tag`](@ref) on a [`Run`](@ref). + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) that the [`Tag`](@ref) was logged under. +- `key`: Name of the [`Tag`](@ref). + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteruntag(instance::MLFlow, run_id::String, key::String)::Bool + mlfpost(instance, "runs/delete-tag"; run_id=run_id, key=key) + return true +end +deleteruntag(instance::MLFlow, run::Run, key::String)::Bool = + deleteruntag(instance, run.info.run_id, key) +deleteruntag(instance::MLFlow, run::Run, tag::Tag)::Bool = + deleteruntag(instance, run.info.run_id, tag.key) + +""" + searchruns(instance::MLFlow; experiment_ids::Array{String}=String[], filter::String="", + run_view_type::ViewType=ACTIVE_ONLY, max_results::Int=1000, + order_by::Array{String}=String[], page_token::String="") + +Search for runs that satisfy expressions. Search expressions can use [`Metric`](@ref) and +[`Param`](@ref) keys. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `experiment_ids`: List of [`Experiment`](@ref) IDs to search over. +- `filter`: A filter expression over params, metrics, and tags, that allows returning a + subset of runs. See [MLFlow documentation](https://mlflow.org/docs/latest/rest-api.html#search-runs). +- `run_view_type`: Whether to display only active, only deleted, or all runs. Defaults to + only active runs. +- `max_results`: Maximum number of runs desired. +- `order_by`: List of columns to be ordered by, including attributes, params, metrics, and + tags with an optional “DESC” or “ASC” annotation, where “ASC” is the default. +- `page_token`: Token indicating the page of runs to fetch. + +# Returns +- Vector of [`Run`](@ref) that were found in the specified experiments. +- The next page token if there are more results. +""" +function searchruns(instance::MLFlow; experiment_ids::Array{String}=String[], + filter::String="", run_view_type::ViewType=ACTIVE_ONLY, max_results::Int=1000, + order_by::Array{String}=String[], + page_token::String="")::Tuple{Array{Run},Union{String,Nothing}} + parameters = (; experiment_ids, filter, :run_view_type => run_view_type |> Integer, + max_results, page_token) + + if order_by |> !isempty + parameters = (; order_by, parameters...) + end + + result = mlfpost(instance, "runs/search"; parameters...) + + runs = get(result, "runs", []) |> (x -> [Run(y) for y in x]) + next_page_token = get(result, "next_page_token", nothing) + + return runs, next_page_token +end + +""" + updaterun(instance::MLFlow, run_id::String; status::Union{RunStatus, Missing}=missing, + end_time::Union{Int64, Missing}=missing, run_name::Union{String, Missing}=missing) + updaterun(instance::MLFlow, run::Run; status::Union{RunStatus, Missing}=missing, + end_time::Union{Int64, Missing}=missing, run_name::Union{String, Missing}=missing) + +Update [`Run`](@ref) metadata. + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `run_id`: ID of the [`Run`](@ref) to update. +- `status`: Updated status of the [`Run`](@ref). +- `end_time`: Unix timestamp in milliseconds of when the [`Run`](@ref) ended. +- `run_name`: Updated name of the [`Run`](@ref). + +# Returns +- An instance of type [`RunInfo`](@ref) with the updated metadata. +""" +function updaterun(instance::MLFlow, run_id::String; + status::Union{RunStatus,Missing}=missing, end_time::Union{Int64,Missing}=missing, + run_name::Union{String,Missing})::RunInfo + result = mlfpost(instance, "runs/update"; run_id=run_id, status=(status |> Integer), + end_time=end_time, run_name=run_name) + return result["run_info"] |> RunInfo +end +updaterun(instance::MLFlow, run::Run; status::Union{RunStatus,Missing}=missing, + end_time::Union{Int64,Missing}=missing, run_name::Union{String,Missing})::RunInfo = + updaterun(instance, run.info.run_id; status=status, end_time=end_time, + run_name=run_name) diff --git a/src/services/user.jl b/src/services/user.jl new file mode 100644 index 0000000..e8dd415 --- /dev/null +++ b/src/services/user.jl @@ -0,0 +1,77 @@ +""" + createuser(instance::MLFlow, username::String, password::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `username`: Username. +- `password`: Password. + +# Returns +An [`User`](@ref) object. +""" +function createuser(instance::MLFlow, username::String, password::String)::User + result = mlfpost(instance, "users/create"; username=username, password=password) + return result["user"] |> User +end + +""" + getuser(instance::MLFlow, username::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `username`: Username. + +# Returns +An [`User`](@ref) object. +""" +function getuser(instance::MLFlow, username::String)::User + result = mlfget(instance, "users/get"; username=username) + return result["user"] |> User +end + +""" + updateuserpassword(instance::MLFlow, username::String, password::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `username`: Username. +- `password`: New password. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function updateuserpassword(instance::MLFlow, username::String, password::String)::Bool + mlfpatch(instance, "users/update-password"; username=username, password=password) + return true +end + +""" + updateuseradmin(instance::MLFlow, username::String, is_admin::Bool) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `username`: Username. +- `is_admin`: New admin status. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function updateuseradmin(instance::MLFlow, username::String, is_admin::Bool)::Bool + mlfpatch(instance, "users/update-admin"; username=username, is_admin=is_admin) + return true +end + +""" + deleteuser(instance::MLFlow, username::String) + +# Arguments +- `instance`: [`MLFlow`](@ref) configuration. +- `username`: Username. + +# Returns +`true` if successful. Otherwise, raises exception. +""" +function deleteuser(instance::MLFlow, username::String)::Bool + mlfdelete(instance, "users/delete"; username=username) + return true +end diff --git a/src/types/artifact.jl b/src/types/artifact.jl index 1b7b984..85e2070 100644 --- a/src/types/artifact.jl +++ b/src/types/artifact.jl @@ -1,31 +1,14 @@ """ - MLFlowArtifactFileInfo - -Metadata of a single artifact file -- result of [`listartifacts`](@ref). + FileInfo # Fields -- `filepath::String`: File path, including the root artifact directory of a run. -- `filesize::Int64`: Size in bytes. -""" -struct MLFlowArtifactFileInfo - filepath::String - filesize::Int64 -end -Base.show(io::IO, t::MLFlowArtifactFileInfo) = show(io, ShowCase(t, new_lines=true)) -get_path(mlfafi::MLFlowArtifactFileInfo) = mlfafi.filepath -get_size(mlfafi::MLFlowArtifactFileInfo) = mlfafi.filesize - -""" - MLFlowArtifactDirInfo - -Metadata of a single artifact directory -- result of [`listartifacts`](@ref). - -# Fields -- `dirpath::String`: Directory path, including the root artifact directory of a run. -""" -struct MLFlowArtifactDirInfo - dirpath::String +- `path::String`: Path relative to the root artifact directory run. +- `is_dir::Bool`: Whether the path is a directory. +- `file_size::Int64`: Size in bytes. Unset for directories. +""" +struct FileInfo + path::String + is_dir::Bool + file_size::Int64 end -Base.show(io::IO, t::MLFlowArtifactDirInfo) = show(io, ShowCase(t, new_lines=true)) -get_path(mlfadi::MLFlowArtifactDirInfo) = mlfadi.dirpath -get_size(mlfadi::MLFlowArtifactDirInfo) = 0 +Base.show(io::IO, t::FileInfo) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/types/dataset.jl b/src/types/dataset.jl new file mode 100644 index 0000000..5ff967a --- /dev/null +++ b/src/types/dataset.jl @@ -0,0 +1,43 @@ +""" + Dataset + +Represents a reference to data used for training, testing, or evaluation during the model +development process. + +# Fields +- `name::String`: The name of the dataset. +- `digest::String`: The digest of the dataset. +- `source_type::String`: The type of the dataset source. +- `source::String`: Source information for the dataset. +- `schema::String`: The schema of the dataset. This field is optional. +- `profile::String`: The profile of the dataset. This field is optional. +""" +struct Dataset + name::String + digest::String + source_type::String + source::String + schema::Union{String,Nothing} + profile::Union{String,Nothing} +end +Dataset(data::Dict{String,Any}) = Dataset(data["name"], data["digest"], + data["source_type"], data["source"], get(data, "schema", nothing), + get(data, "profile", nothing)) +Base.show(io::IO, t::Dataset) = show(io, ShowCase(t, new_lines=true)) + +""" + DatasetInput + +Represents a dataset and input tags. + +# Fields +- `tags::Array{Tag}`: A list of tags for the dataset input. +- `dataset::Dataset`: The dataset being used as a run input. +""" +struct DatasetInput + tags::Array{Tag} + dataset::Dataset +end +DatasetInput(data::Dict{String,Any}) = DatasetInput( + [Tag(tag) for tag in get(data, "tags", [])], Dataset(data["dataset"])) +Base.show(io::IO, t::DatasetInput) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/types/enums.jl b/src/types/enums.jl new file mode 100644 index 0000000..4edfb6e --- /dev/null +++ b/src/types/enums.jl @@ -0,0 +1,71 @@ +""" + ModelVersionStatus + +# Members +- `PENDING_REGISTRATION`: Request to register a new model version is pending as server + performs background tasks. +- `FAILED_REGISTRATION`: Request to register a new model version has failed. +- `READY`: Model version is ready for use. +""" +@enum ModelVersionStatus begin + PENDING_REGISTRATION = 1 + FAILED_REGISTRATION = 2 + READY = 3 +end +ModelVersionStatus(status::String) = Dict(value => key for (key, value) in ModelVersionStatus |> Base.Enums.namemap)[status|>Symbol] |> ModelVersionStatus + +""" + RunStatus + +Status of a run. + +# Members +- `RUNNING`: Run has been initiated. +- `SCHEDULED`: Run is scheduled to run at a later time. +- `FINISHED`: Run has completed. +- `FAILED`: Run execution failed. +- `KILLED`: Run killed by user. +""" +@enum RunStatus begin + RUNNING = 1 + SCHEDULED = 2 + FINISHED = 3 + FAILED = 4 + KILLED = 5 +end +RunStatus(status::String) = Dict(value => key for (key, value) in RunStatus |> Base.Enums.namemap)[status|>Symbol] |> RunStatus + +""" + ViewType + +View type for ListExperiments query. + +# Members +- `ACTIVE_ONLY`: Default. Return only active experiments. +- `DELETED_ONLY`: Return only deleted experiments. +- `ALL`: Get all experiments. +""" +@enum ViewType begin + ACTIVE_ONLY = 1 + DELETED_ONLY = 2 + ALL = 3 +end + +""" + Permission + +Permission of a user to an experiment or a registered model. + +# Members +- `READ`: Can read. +- `EDIT`: Can read and update. +- `MANAGE`: Can read, update, delete and manage. +- `NO_PERMISSIONS`: No permissions. +""" +@enum Permission begin + READ = 1 + EDIT = 2 + MANAGE = 3 + NO_PERMISSIONS = 4 +end +Permission(permission::String) = Dict(value => key for (key, value) in Permission |> Base.Enums.namemap)[permission|>Symbol] |> Permission diff --git a/src/types/experiment.jl b/src/types/experiment.jl index 7c4921c..26ddcf4 100644 --- a/src/types/experiment.jl +++ b/src/types/experiment.jl @@ -1,34 +1,43 @@ """ - MLFlowExperiment - -Represents an MLFlow experiment. + Experiment # Fields -- `name::String`: experiment name. -- `lifecycle_stage::String`: life cycle stage, one of ["active", "deleted"] -- `experiment_id::Integer`: experiment identifier. -- `tags::Any`: list of tags. -- `artifact_location::String`: where are experiment artifacts stored. - -# Constructors - -- `MLFlowExperiment(name, lifecycle_stage, experiment_id, tags, artifact_location)` -- `MLFlowExperiment(exp::Dict{String,Any})` - +- `experiment_id::Integer`: Unique identifier for the experiment. +- `name::String`: Human readable name that identifies the experiment. +- `artifact_location::String`: Location where artifacts for the experiment are stored. +- `lifecycle_stage::String`: Current life cycle stage of the experiment: “active” or + “deleted”. Deleted experiments are not returned by APIs. +- `last_update_time::Int64`: Last update time. +- `creation_time::Int64`: Creation time. +- `tags::Array{Tag}`: Additional metadata key-value pairs. """ -struct MLFlowExperiment +struct Experiment + experiment_id::String name::String - lifecycle_stage::String - experiment_id::Integer - tags::Any artifact_location::String + lifecycle_stage::String + last_update_time::Int64 + creation_time::Int64 + tags::Array{Tag} end -function MLFlowExperiment(exp::Dict{String,Any}) - name = get(exp, "name", missing) - lifecycle_stage = get(exp, "lifecycle_stage", missing) - experiment_id = parse(Int, get(exp, "experiment_id", missing)) - tags = get(exp, "tags", missing) - artifact_location = get(exp, "artifact_location", missing) - MLFlowExperiment(name, lifecycle_stage, experiment_id, tags, artifact_location) +Experiment(data::Dict{String,Any}) = Experiment(data["experiment_id"], data["name"], + data["artifact_location"], data["lifecycle_stage"], data["last_update_time"], + data["creation_time"], [Tag(tag) for tag in get(data, "tags", [])]) +Base.show(io::IO, t::Experiment) = show(io, ShowCase(t, new_lines=true)) + +""" + ExperimentPermission + +# Fields +- `experiment_id::String`: [`Experiment`](@ref) id. +- `user_id::String`: [`User`](@ref) id. +- `permission::Permission`: [`Permission`](@ref) granted. +""" +struct ExperimentPermission + experiment_id::String + user_id::String + permission::Permission end -Base.show(io::IO, t::MLFlowExperiment) = show(io, ShowCase(t, new_lines=true)) +ExperimentPermission(data::Dict{String,Any}) = ExperimentPermission(data["experiment_id"], + data["user_id"] |> string, Permission(data["permission"])) +Base.show(io::IO, t::ExperimentPermission) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/types/mlflow.jl b/src/types/mlflow.jl index 326a0ff..111232e 100644 --- a/src/types/mlflow.jl +++ b/src/types/mlflow.jl @@ -6,13 +6,8 @@ Base type which defines location and version for MLFlow API service. # Fields - `apiroot::String`: API root URL, e.g. `http://localhost:5000/api` - `apiversion::Union{Integer, AbstractFloat}`: used API version, e.g. `2.0` -- `headers::Dict`: HTTP headers to be provided with the REST API requests (useful for authetication tokens) -Default is `false`, using the REST API endpoint. - -# Constructors - -- `MLFlow(apiroot; apiversion=2.0,headers=Dict())` -- `MLFlow()` - defaults to `MLFlow(ENV["MLFLOW_TRACKING_URI"])` or `MLFlow("http://localhost:5000/api")` +- `headers::Dict`: HTTP headers to be provided with the REST API requests (useful for + authetication tokens) Default is `false`, using the REST API endpoint. # Examples @@ -28,15 +23,15 @@ mlf = MLFlow(remote_url, headers=Dict("Authorization" => "Bearer string, Permission(data["permission"])) +Base.show(io::IO, t::RegisteredModelPermission) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/types/run.jl b/src/types/run.jl index 13920a9..92ff484 100644 --- a/src/types/run.jl +++ b/src/types/run.jl @@ -1,215 +1,125 @@ """ - MLFlowRunStatus + Metric <: LoggingData -Represents the status of an MLFlow Run. +Metric associated with a run, represented as a key-value pair. # Fields -- `status::String`: one of RUNNING/SCHEDULED/FINISHED/FAILED/KILLED - -# Constructors - -- `MLFlowRunStatus(status::String)` +- `key::String`: Key identifying this metric. +- `value::Float64`: Value associated with this metric. +- `timestamp::Int64`: The timestamp at which this metric was recorded. +- `step::Union{Int64, Nothing}`: Step at which to log the metric. """ -struct MLFlowRunStatus - status::String - function MLFlowRunStatus(status::String) - acceptable_statuses = ["RUNNING", "SCHEDULED", "FINISHED", "FAILED", "KILLED"] - status ∈ acceptable_statuses || error("Invalid status $status - choose one of $acceptable_statuses") - new(status) - end +struct Metric <: LoggingData + key::String + value::Float64 + timestamp::Int64 + step::Union{Int64,Nothing} end -Base.show(io::IO, t::MLFlowRunStatus) = show(io, ShowCase(t, new_lines=true)) +Metric(data::Dict{String,Any}) = Metric(data["key"], data["value"], data["timestamp"], + data["step"]) +Base.show(io::IO, t::Metric) = show(io, ShowCase(t, new_lines=true)) """ - MLFlowRunInfo + Param <: LoggingData -Represents run metadata. +Param associated with a run. # Fields -- `run_id::String`: run identifier. -- `experiment_id::Integer`: experiment identifier. -- `status::MLFlowRunStatus`: run status. -- `run_name::String`: run name. -- `start_time::Union{Int64,Missing}`: when was the run started, UNIX time in milliseconds. -- `end_time::Union{Int64,Missing}`: when did the run end, UNIX time in milliseconds. -- `artifact_uri::String`: where are artifacts from this run stored. -- `lifecycle_stage::String`: one of `active` or `deleted`. - -# Constructors - -- `MLFlowRunInfo(run_id, experiment_id, status, run_name, start_time, end_time, artifact_uri, lifecycle_stage)` -- `MLFlowRunInfo(info::Dict{String,Any})` +- `key::String`: Key identifying this param. +- `value::String`: Value associated with this param. """ -struct MLFlowRunInfo - run_id::String - experiment_id::Integer - status::MLFlowRunStatus - run_name::String - start_time::Union{Int64,Missing} - end_time::Union{Int64,Missing} - artifact_uri::String - lifecycle_stage::String -end -function MLFlowRunInfo(info::Dict{String,Any}) - run_id = get(info, "run_id", missing) - experiment_id = get(info, "experiment_id", missing) - status = get(info, "status", missing) - run_name = get(info, "run_name", missing) - start_time = get(info, "start_time", missing) - end_time = get(info, "end_time", missing) - artifact_uri = get(info, "artifact_uri", "") - lifecycle_stage = get(info, "lifecycle_stage", "") - - experiment_id = ismissing(experiment_id) ? experiment_id : parse(Int64, experiment_id) - status = ismissing(status) ? status : MLFlowRunStatus(status) - - # support for mlflow 1.21.0 - if !ismissing(start_time) && !(typeof(start_time) <: Int) - start_time = parse(Int64, start_time) - end - if !ismissing(end_time) && !(typeof(end_time) <: Int) - end_time = parse(Int64, end_time) - end - MLFlowRunInfo(run_id, experiment_id, status, run_name, start_time, end_time, artifact_uri, lifecycle_stage) +struct Param <: LoggingData + key::String + value::String end -Base.show(io::IO, t::MLFlowRunInfo) = show(io, ShowCase(t, new_lines=true)) -get_run_id(runinfo::MLFlowRunInfo) = runinfo.run_id +Param(data::Dict{String,Any}) = Param(data["key"], data["value"]) +Base.show(io::IO, t::Param) = show(io, ShowCase(t, new_lines=true)) """ - MLFlowRunDataMetric + RunInfo -Represents a metric. +Metadata of a single run. # Fields -- `key::String`: metric identifier. -- `value::Float64`: metric value. -- `step::Int64`: step. -- `timestamp::Int64`: timestamp in UNIX time in milliseconds. - -# Constructors - -- `MLFlowRunDataMetric(d::Dict{String,Any})` - +- `run_id::String`: Unique identifier for the run. +- `run_name::String`: The name of the run. +- `experiment_id::String`: The experiment ID. +- `status::RunStatus`: Current status of the run. +- `start_time::Int64`: Unix timestamp of when the run started in milliseconds. +- `end_time::Int64`: Unix timestamp of when the run ended in milliseconds. +- `artifact_uri::String`: URI of the directory where artifacts should be uploaded. This can + be a local path (starting with “/”), or a distributed file system (DFS) path, + like s3://bucket/directory or dbfs:/my/directory. If not set, the local ./mlruns + directory is chosen. +- `lifecycle_stage::String`: Current life cycle stage of the experiment: "active" or + "deleted". """ -struct MLFlowRunDataMetric - key::String - value::Float64 - step::Int64 - timestamp::Int64 -end -function MLFlowRunDataMetric(d::Dict{String,Any}) - key = d["key"] - value = d["value"] - if typeof(d["step"]) <: Int - step = d["step"] - else - step = parse(Int64, d["step"]) - end - if typeof(d["timestamp"]) <: Int - timestamp = d["timestamp"] - else - timestamp = parse(Int64, d["timestamp"]) - end - MLFlowRunDataMetric(key, value, step, timestamp) +struct RunInfo + run_id::String + run_name::String + experiment_id::String + status::RunStatus + start_time::Int64 + end_time::Union{Int64,Nothing} + artifact_uri::String + lifecycle_stage::String end -Base.show(io::IO, t::MLFlowRunDataMetric) = show(io, ShowCase(t, new_lines=true)) +RunInfo(data::Dict{String,Any}) = RunInfo(data["run_id"], data["run_name"], + data["experiment_id"], RunStatus(data["status"]), data["start_time"], + get(data, "end_time", nothing), data["artifact_uri"], data["lifecycle_stage"]) +Base.show(io::IO, t::RunInfo) = show(io, ShowCase(t, new_lines=true)) """ - MLFlowRunDataParam + RunInputs -Represents a parameter. +Run data (metrics, params, and tags). # Fields -- `key::String`: parameter identifier. -- `value::String`: parameter value. - -# Constructors -- `MLFlowRunDataParam(d::Dict{String,String})` - +- `metrics::Array{Metric}`: Run metrics. +- `params::Array{Param}`: Run parameters. +- `tags::Array{Tag}`: Additional metadata key-value pairs. """ -struct MLFlowRunDataParam - key::String - value::String +struct RunData + metrics::Array{Metric} + params::Array{Param} + tags::Array{Tag} end -function MLFlowRunDataParam(d::Dict{String,String}) - key = d["key"] - value = d["value"] - MLFlowRunDataParam(key, value) -end -Base.show(io::IO, t::MLFlowRunDataParam) = show(io, ShowCase(t, new_lines=true)) +RunData(data::Dict{String,Any}) = RunData( + [Metric(metric) for metric in get(data, "metrics", [])], + [Param(param) for param in get(data, "params", [])], + [Tag(tag) for tag in get(data, "tags", [])]) +Base.show(io::IO, t::RunData) = show(io, ShowCase(t, new_lines=true)) """ - MLFlowRunData + RunInputs -Represents run data. +Run inputs. # Fields -- `metrics::Dict{String,MLFlowRunDataMetric}`: run metrics. -- `params::Dict{String,MLFlowRunDataParam}`: run parameters. -- `tags`: list of run tags. - -# Constructors - -- `MLFlowRunData(data::Dict{String,Any})` - +- `dataset_inputs::Array{DatasetInput}`: Dataset inputs to the Run. """ -struct MLFlowRunData - metrics::Dict{String,MLFlowRunDataMetric} - params::Union{Dict{String,MLFlowRunDataParam},Missing} - tags +struct RunInputs + dataset_inputs::Array{DatasetInput} end -function MLFlowRunData(data::Dict{String,Any}) - metrics = Dict{String,MLFlowRunDataMetric}() - if haskey(data, "metrics") - for metric in data["metrics"] - new_metric = MLFlowRunDataMetric(metric) - metrics[new_metric.key] = new_metric - end - end - params = Dict{String,MLFlowRunDataParam}() - if haskey(data, "params") - for param in data["params"] - new_param = MLFlowRunDataParam(param["key"], param["value"]) - params[new_param.key] = new_param - end - end - tags = haskey(data, "tags") ? data["tags"] : missing - MLFlowRunData(metrics, params, tags) -end -Base.show(io::IO, t::MLFlowRunData) = show(io, ShowCase(t, new_lines=true)) -get_params(rundata::MLFlowRunData) = rundata.params +RunInputs(data::Dict{String,Any}) = RunInputs( + [DatasetInput(dataset_input) for dataset_input in get(data, "dataset_inputs", [])]) +Base.show(io::IO, t::RunInputs) = show(io, ShowCase(t, new_lines=true)) """ - MLFlowRun + Run -Represents an MLFlow run. +A single run. # Fields -- `info::MLFlowRunInfo`: Run metadata. -- `data::MLFlowRunData`: Run data. - -# Constructors - -- `MLFlowRun(rundata::MLFlowRunData)` -- `MLFlowRun(runinfo::MLFlowRunInfo)` -- `MLFlowRun(info::Dict{String,Any})` -- `MLFlowRun(info::Dict{String,Any}, data::Dict{String,Any})` - +- `info::RunInfo`: Metadata of the run. +- `data::RunData`: Run data (metrics, params, and tags). +- `inputs::RunInputs`: Run inputs. """ -struct MLFlowRun - info::Union{MLFlowRunInfo,Missing} - data::Union{MLFlowRunData,Missing} +struct Run + info::RunInfo + data::RunData + inputs::RunInputs end -MLFlowRun(rundata::MLFlowRunData) = - MLFlowRun(missing, rundata) -MLFlowRun(runinfo::MLFlowRunInfo) = - MLFlowRun(runinfo, missing) -MLFlowRun(info::Dict{String,Any}) = - MLFlowRun(MLFlowRunInfo(info), missing) -MLFlowRun(info::Dict{String,Any}, data::Dict{String,Any}) = - MLFlowRun(MLFlowRunInfo(info), MLFlowRunData(data)) -Base.show(io::IO, t::MLFlowRun) = show(io, ShowCase(t, new_lines=true)) -get_info(run::MLFlowRun) = run.info -get_data(run::MLFlowRun) = run.data -get_run_id(run::MLFlowRun) = get_run_id(run.info) -get_params(run::MLFlowRun) = get_params(run.data) +Run(data::Dict{String,Any}) = Run(RunInfo(data["info"]), RunData(data["data"]), + RunInputs(data["inputs"])) +Base.show(io::IO, t::Run) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/types/tag.jl b/src/types/tag.jl new file mode 100644 index 0000000..5eae9af --- /dev/null +++ b/src/types/tag.jl @@ -0,0 +1,15 @@ +""" + Tag <: LoggingData + +Generic tag type for MLFlow entities. + +# Fields +- `key::String`: The tag key. +- `value::String`: The tag value. +""" +struct Tag <: LoggingData + key::String + value::String +end +Tag(data::Dict{String,Any})::Tag = Tag(data["key"], data["value"] |> string) +Base.show(io::IO, t::Tag) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/types/user.jl b/src/types/user.jl new file mode 100644 index 0000000..038ffae --- /dev/null +++ b/src/types/user.jl @@ -0,0 +1,23 @@ +""" + User + +# Fields +- `id::String`: User ID. +- `username::String`: Username. +- `is_admin::Bool`: Whether the user is an admin. +- `experiment_permissions::Array{ExperimentPermission}`: All experiment permissions + explicitly granted to the user. +- `registered_model_permissions::Array{RegisteredModelPermission}`: All registered model + explicitly granted to the user. +""" +struct User + id::String + username::String + is_admin::Bool + experiment_permissions::Array{ExperimentPermission} + registered_model_permissions::Array{RegisteredModelPermission} +end +User(data::Dict{String,Any}) = User(data["id"] |> string, data["username"], data["is_admin"], + [ExperimentPermission(permission) for permission in get(data, "experiment_permissions", [])], + [RegisteredModelPermission(permission) for permission in get(data, "registered_model_permissions", [])]) +Base.show(io::IO, t::User) = show(io, ShowCase(t, new_lines=true)) diff --git a/src/utils.jl b/src/utils.jl index bc8e225..7543b4c 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,70 +1,93 @@ -""" - uri(mlf::MLFlow, endpoint="", query=missing) +const NumberOrString = Union{Number,String} +const MLFlowUpsertData{T} = Union{Array{T},Array{<:Dict{String,<:Any}}, + Dict{String,<:NumberOrString},Array{<:Pair{String,<:NumberOrString}}, + Array{<:Tuple{String,<:NumberOrString}}} -Retrieves an URI based on `mlf`, `endpoint`, and, optionally, `query`. +function dict_to_T_array(::Type{T}, + dict::Dict{String,<:NumberOrString}) where {T<:LoggingData} + entities = T[] + for (key, value) in dict + if T <: Metric + push!(entities, Metric(key, Float64(value), round(Int, now() |> datetime2unix), + nothing)) + else + push!(entities, T(key, value |> string)) + end + end -# Examples -```@example -MLFlowClient.uri(mlf, "experiments/get", Dict(:experiment_id=>10)) -``` -""" -function uri(mlf::MLFlow, endpoint="", query=missing) - u = URI("$(mlf.apiroot)/$(mlf.apiversion)/mlflow/$(endpoint)") - !ismissing(query) && return URI(u; query=query) - u + return entities end -""" - headers(mlf::MLFlow,custom_headers::AbstractDict) +function pairarray_to_T_array(::Type{T}, pair_array::Array{<:Pair}) where {T<:LoggingData} + entities = T[] + for pair in pair_array + key = pair.first |> string + if T <: Metric + value = pair.second + push!(entities, Metric(key, Float64(value), round(Int, now() |> datetime2unix), + nothing)) + else + value = pair.second |> string + push!(entities, T(key, value)) + end + end -Retrieves HTTP headers based on `mlf` and merges with user-provided `custom_headers` - -# Examples -```@example -headers(mlf,Dict("Content-Type"=>"application/json")) -``` -""" -headers(mlf::MLFlow, custom_headers::AbstractDict) = merge(mlf.headers, custom_headers) - -""" - generatefilterfromentity_type(filter_params::AbstractDict{K,V}, entity_type::String) where {K,V} - -Generates a `filter` string from `filter_params` dictionary and `entity_type`. - -# Arguments -- `filter_params`: dictionary to use for filter generation. -- `entity_type`: entity type to use for filter generation. + return entities +end -# Returns -A string that can be passed as `filter` to [`searchruns`](@ref). +function tuplearray_to_T_array(::Type{T}, + tuple_array::Array{<:Tuple{String,<:NumberOrString}}) where {T<:LoggingData} + entities = T[] + for tuple in tuple_array + if length(tuple) != 2 + error("Tuple must have exactly two elements (format: (key, value))") + end -# Examples + key = tuple |> first |> string + if T <: Metric + value = tuple |> last + push!(entities, Metric(key, Float64(value), round(Int, now() |> datetime2unix), + nothing)) + else + value = tuple |> last |> string + push!(entities, T(key, value)) + end + end -```@example -generatefilterfromentity_type(Dict("paramkey1" => "paramvalue1", "paramkey2" => "paramvalue2"), "param") -``` -""" -function generatefilterfromentity_type(filter_params::AbstractDict{K,V}, entity_type::String) where {K,V} - length(filter_params) > 0 || return "" - # NOTE: may have issues with escaping. - filters = ["$(entity_type).\"$(k)\" = \"$(v)\"" for (k, v) ∈ filter_params] - join(filters, " and ") + return entities end -""" - generatefilterfromparams(filter_params::AbstractDict{K,V}) where {K,V} - -Generates a `filter` string from `filter_params` dictionary and `param` entity type. -""" -generatefilterfromparams(filter_params::AbstractDict{K,V}) where {K,V} = generatefilterfromentity_type(filter_params, "param") -""" - generatefilterfrommattributes(filter_attributes::AbstractDict{K,V}) where {K,V} +function dictarray_to_T_array(::Type{T}, + dict_array::Array{<:Dict{String,<:Any}}) where {T<:LoggingData} + entities = T[] + for dict in dict_array + key = dict["key"] |> string + if T <: Metric + value = Float64(dict["value"]) + if haskey(dict, "timestamp") + timestamp = dict["timestamp"] + else + timestamp = round(Int, now() |> datetime2unix) + end + push!(entities, Metric(key, value, timestamp, nothing)) + else + value = dict["value"] |> string + push!(entities, T(key, value)) + end + end -Generates a `filter` string from `filter_attributes` dictionary and `attribute` entity type. -""" -generatefilterfromattributes(filter_attributes::AbstractDict{K,V}) where {K,V} = generatefilterfromentity_type(filter_attributes, "attribute") + return entities +end -const MLFLOW_ERROR_CODES = (; - RESOURCE_ALREADY_EXISTS = "RESOURCE_ALREADY_EXISTS", - RESOURCE_DOES_NOT_EXIST = "RESOURCE_DOES_NOT_EXIST", -) +function parse(::Type{T}, entities::MLFlowUpsertData{T}) where {T<:LoggingData} + if entities isa Dict{String,<:NumberOrString} + return dict_to_T_array(T, entities) + elseif entities isa Array{<:Dict{String,<:Any}} + return dictarray_to_T_array(T, entities) + elseif entities isa Array{<:Pair{String,<:NumberOrString}} + return pairarray_to_T_array(T, entities) + elseif entities isa Array{<:Tuple{String,<:NumberOrString}} + return tuplearray_to_T_array(T, entities) + end + return entities +end diff --git a/test/base.jl b/test/base.jl index 2561ed0..8206d5c 100644 --- a/test/base.jl +++ b/test/base.jl @@ -1,7 +1,8 @@ -using MLFlowClient using Test -using UUIDs using Dates +using UUIDs +using Base64 +using MLFlowClient function mlflow_server_is_running(mlf::MLFlow) try @@ -16,7 +17,8 @@ end # skips test if mlflow is not available on default location, ENV["MLFLOW_TRACKING_URI"] macro ensuremlf() e = quote - mlf = MLFlow() + encoded_credentials = Base64.base64encode("admin:password") + mlf = MLFlow(headers=Dict("Authorization" => "Basic $(encoded_credentials)")) mlflow_server_is_running(mlf) || return nothing end eval(e) diff --git a/test/runtests.jl b/test/runtests.jl index 3f35682..6ccb44f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,7 +4,11 @@ end include("base.jl") -include("test_functional.jl") -include("test_experiments.jl") -include("test_runs.jl") -include("test_loggers.jl") +include("services/run.jl") +include("services/misc.jl") +include("services/logger.jl") +include("services/artifact.jl") +include("services/experiment.jl") +include("services/registered_model.jl") +include("services/model_version.jl") +include("services/user.jl") diff --git a/test/services/artifact.jl b/test/services/artifact.jl new file mode 100644 index 0000000..5024da8 --- /dev/null +++ b/test/services/artifact.jl @@ -0,0 +1,25 @@ +@testset verbose = true "list artifacts" begin + # TODO: Add more specific tests after implementing the complete artifact service + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment_id) + + @testset "using run id" begin + root_uri, files, next_page_token = listartifacts(mlf, run.info.run_id) + + @test run.info.artifact_uri == root_uri + @test isempty(files) + @test isnothing(next_page_token) + end + + @testset "using run" begin + root_uri, files, next_page_token = listartifacts(mlf, run) + + @test run.info.artifact_uri == root_uri + @test isempty(files) + @test isnothing(next_page_token) + end + + deleteexperiment(mlf, experiment_id) +end diff --git a/test/services/experiment.jl b/test/services/experiment.jl new file mode 100644 index 0000000..e713e92 --- /dev/null +++ b/test/services/experiment.jl @@ -0,0 +1,379 @@ +@testset verbose = true "create experiment" begin + @ensuremlf + + experiment_name = UUIDs.uuid4() |> string + + @testset "base" begin + experiment_id = createexperiment(mlf, experiment_name) + @test isa(experiment_id, String) + end + + @testset "name exists" begin + experiment = getexperimentbyname(mlf, experiment_name) + @test_throws ErrorException createexperiment(mlf, experiment.name) + deleteexperiment(mlf, experiment.experiment_id) + end + + @testset "with tags as array of tags" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string; + tags=[Tag("test_key", "test_value")]) + deleteexperiment(mlf, experiment_id) + end + + @testset "with tags as array of pairs" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string; + tags=["test_key" => "test_value"]) + deleteexperiment(mlf, experiment_id) + end + + @testset "with tags as array of dicts" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string; + tags=[Dict("key" => "test_key", "value" => "test_value")]) + deleteexperiment(mlf, experiment_id) + end + + @testset "with tags as dict" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string; + tags=Dict("test_key" => "test_value")) + deleteexperiment(mlf, experiment_id) + end +end + +@testset verbose = true "get experiment" begin + @ensuremlf + experiment_name = UUIDs.uuid4() |> string + artifact_location = "test_location" + tags = [Tag("test_key", "test_value")] + experiment_id = createexperiment(mlf, experiment_name; + artifact_location=artifact_location, tags=tags) + + @testset "using string id" begin + experiment = getexperiment(mlf, experiment_id) + @test isa(experiment, Experiment) + @test experiment.experiment_id == experiment_id + @test experiment.name == experiment_name + @test occursin(artifact_location, experiment.artifact_location) + @test experiment.tags |> !isempty + @test (experiment.tags |> first).key == (tags |> first).key + @test (experiment.tags |> first).value == (tags |> first).value + end + + @testset "using integer id" begin + experiment = getexperiment(mlf, parse(Int, experiment_id)) + @test isa(experiment, Experiment) + end + + @testset "using name" begin + experiment = getexperimentbyname(mlf, experiment_name) + @test isa(experiment, Experiment) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "delete experiment" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "using string id" begin + @test deleteexperiment(mlf, experiment_id) + restoreexperiment(mlf, experiment_id) + end + + @testset "using integer id" begin + @test deleteexperiment(mlf, parse(Int, experiment_id)) + restoreexperiment(mlf, experiment_id) + end + + @testset "using Experiment" begin + experiment = getexperiment(mlf, experiment_id) + @test deleteexperiment(mlf, experiment) + restoreexperiment(mlf, experiment_id) + end + + @testset "delete already deleted" begin + deleteexperiment(mlf, experiment_id) + @test_throws ErrorException deleteexperiment(mlf, experiment_id) + end +end + +@testset verbose = true "restore experiment" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "using string id" begin + deleteexperiment(mlf, experiment_id) + @test restoreexperiment(mlf, experiment_id) + end + + @testset "using integer id" begin + deleteexperiment(mlf, experiment_id) + @test restoreexperiment(mlf, parse(Int, experiment_id)) + end + + @testset "using Experiment" begin + experiment = getexperiment(mlf, experiment_id) + deleteexperiment(mlf, experiment_id) + @test restoreexperiment(mlf, experiment) + end + + @testset "restore not found" begin + @test_throws ErrorException restoreexperiment(mlf, 123) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "update experiment" begin + @ensuremlf + experiment_name = UUIDs.uuid4() |> string + experiment_id = createexperiment(mlf, experiment_name) + + @testset "update name with string id" begin + new_name = UUIDs.uuid4() |> string + updateexperiment(mlf, experiment_id, new_name) + experiment = getexperiment(mlf, experiment_id) + @test experiment.name == new_name + end + + @testset "update name with integer id" begin + new_name = UUIDs.uuid4() |> string + updateexperiment(mlf, parse(Int, experiment_id), new_name) + experiment = getexperiment(mlf, experiment_id) + @test experiment.name == new_name + end + + @testset "update name with Experiment" begin + new_name = UUIDs.uuid4() |> string + experiment = getexperiment(mlf, experiment_id) + updateexperiment(mlf, experiment, new_name) + experiment = getexperiment(mlf, experiment_id) + @test experiment.name == new_name + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "search experiments" begin + @ensuremlf + + experiment_ids = [ + createexperiment(mlf, UUIDs.uuid4() |> string), + createexperiment(mlf, UUIDs.uuid4() |> string), + createexperiment(mlf, UUIDs.uuid4() |> string)] + + @testset "default search" begin + experiments, next_page_token = searchexperiments(mlf) + + @test length(experiments) == 4 # four because of the default experiment + @test next_page_token |> isnothing + end + + @testset "with pagination" begin + experiments, next_page_token = searchexperiments(mlf; max_results=1) + + @test length(experiments) == 1 + @test next_page_token |> !isnothing + @test next_page_token isa String + end + + experiment_ids .|> (id -> deleteexperiment(mlf, id)) +end + +@testset verbose = true "set experiment tag" begin + @ensuremlf + + @testset "set tag with string id" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + setexperimenttag(mlf, experiment_id, "test_key", "test_value") + experiment = getexperiment(mlf, experiment_id) + @test experiment.tags |> !isempty + @test (experiment.tags |> first).key == "test_key" + @test (experiment.tags |> first).value == "test_value" + deleteexperiment(mlf, experiment_id) + end + + @testset "set tag with integer id" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + setexperimenttag(mlf, parse(Int, experiment_id), "test_key", "test_value") + experiment = getexperiment(mlf, experiment_id) + @test experiment.tags |> !isempty + @test (experiment.tags |> first).key == "test_key" + @test (experiment.tags |> first).value == "test_value" + deleteexperiment(mlf, experiment_id) + end + + @testset "set tag with Experiment" begin + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + experiment = getexperiment(mlf, experiment_id) + setexperimenttag(mlf, experiment, "test_key", "test_value") + experiment = getexperiment(mlf, experiment_id) + @test experiment.tags |> !isempty + @test (experiment.tags |> first).key == "test_key" + @test (experiment.tags |> first).value == "test_value" + deleteexperiment(mlf, experiment_id) + end +end + +@testset verbose = true "create experiment permission" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + permission = Permission("READ") + + @testset "with string experiment id" begin + user = createuser(mlf, "missy", "gala") + experiment_permission = + createexperimentpermission(mlf, experiment_id, user.username, permission) + + @test experiment_permission isa ExperimentPermission + @test experiment_permission.experiment_id == experiment_id + @test experiment_permission.user_id == user.id + @test experiment_permission.permission == permission + deleteexperimentpermission(mlf, experiment_id, user.username) + deleteuser(mlf, user.username) + end + + @testset "with integer experiment id" begin + user = createuser(mlf, "missy", "gala") + experiment_permission = + createexperimentpermission(mlf, parse(Int, experiment_id), user.username, permission) + + @test experiment_permission isa ExperimentPermission + @test experiment_permission.experiment_id == experiment_id + @test experiment_permission.user_id == user.id + @test experiment_permission.permission == permission + deleteexperimentpermission(mlf, experiment_id, user.username) + deleteuser(mlf, user.username) + end + + @testset "with Experiment" begin + experiment = getexperiment(mlf, experiment_id) + user = createuser(mlf, "missy", "gala") + experiment_permission = + createexperimentpermission(mlf, experiment, user.username, permission) + + @test experiment_permission isa ExperimentPermission + @test experiment_permission.experiment_id == experiment_id + @test experiment_permission.user_id == user.id + @test experiment_permission.permission == permission + deleteexperimentpermission(mlf, experiment_id, user.username) + deleteuser(mlf, user.username) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "get experiment permission" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + permission = Permission("READ") + user = createuser(mlf, "missy", "gala") + + @testset "with string experiment id" begin + createexperimentpermission(mlf, experiment_id, user.username, permission) + experiment_permission = getexperimentpermission(mlf, experiment_id, user.username) + + @test experiment_permission isa ExperimentPermission + @test experiment_permission.experiment_id == experiment_id + @test experiment_permission.user_id == user.id + @test experiment_permission.permission == permission + deleteexperimentpermission(mlf, experiment_id, user.username) + end + + @testset "with integer experiment id" begin + createexperimentpermission(mlf, parse(Int, experiment_id), user.username, permission) + experiment_permission = getexperimentpermission(mlf, parse(Int, experiment_id), user.username) + + @test experiment_permission isa ExperimentPermission + @test experiment_permission.experiment_id == experiment_id + @test experiment_permission.user_id == user.id + @test experiment_permission.permission == permission + deleteexperimentpermission(mlf, experiment_id, user.username) + end + + @testset "with Experiment" begin + experiment = getexperiment(mlf, experiment_id) + createexperimentpermission(mlf, experiment, user.username, permission) + experiment_permission = getexperimentpermission(mlf, experiment, user.username) + + @test experiment_permission isa ExperimentPermission + @test experiment_permission.experiment_id == experiment_id + @test experiment_permission.user_id == user.id + @test experiment_permission.permission == permission + deleteexperimentpermission(mlf, experiment_id, user.username) + end + + deleteuser(mlf, user.username) + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "update experiment permission" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + permission = Permission("READ") + user = createuser(mlf, "missy", "gala") + + @testset "with string experiment id" begin + createexperimentpermission(mlf, experiment_id, user.username, permission) + updateexperimentpermission(mlf, experiment_id, user.username, Permission("EDIT")) + experiment_permission = getexperimentpermission(mlf, experiment_id, user.username) + + @test experiment_permission.permission == Permission("EDIT") + deleteexperimentpermission(mlf, experiment_id, user.username) + end + + @testset "with integer experiment id" begin + createexperimentpermission(mlf, parse(Int, experiment_id), user.username, permission) + updateexperimentpermission(mlf, parse(Int, experiment_id), user.username, Permission("EDIT")) + experiment_permission = getexperimentpermission(mlf, parse(Int, experiment_id), user.username) + + @test experiment_permission.permission == Permission("EDIT") + deleteexperimentpermission(mlf, experiment_id, user.username) + end + + @testset "with Experiment" begin + experiment = getexperiment(mlf, experiment_id) + createexperimentpermission(mlf, experiment, user.username, permission) + updateexperimentpermission(mlf, experiment, user.username, Permission("EDIT")) + experiment_permission = getexperimentpermission(mlf, experiment, user.username) + + @test experiment_permission.permission == Permission("EDIT") + deleteexperimentpermission(mlf, experiment_id, user.username) + end + + deleteuser(mlf, user.username) + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "delete experiment permission" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + permission = Permission("READ") + user = createuser(mlf, "missy", "gala") + + @testset "with string experiment id" begin + createexperimentpermission(mlf, experiment_id, user.username, permission) + deleteexperimentpermission(mlf, experiment_id, user.username) + @test_throws ErrorException getexperimentpermission(mlf, experiment_id, user.username) + end + + @testset "with integer experiment id" begin + createexperimentpermission(mlf, parse(Int, experiment_id), user.username, permission) + deleteexperimentpermission(mlf, parse(Int, experiment_id), user.username) + @test_throws ErrorException getexperimentpermission(mlf, parse(Int, experiment_id), user.username) + end + + @testset "with Experiment" begin + experiment = getexperiment(mlf, experiment_id) + createexperimentpermission(mlf, experiment, user.username, permission) + deleteexperimentpermission(mlf, experiment, user.username) + @test_throws ErrorException getexperimentpermission(mlf, experiment, user.username) + end + + deleteuser(mlf, user.username) + deleteexperiment(mlf, experiment_id) +end diff --git a/test/services/logger.jl b/test/services/logger.jl new file mode 100644 index 0000000..32bad21 --- /dev/null +++ b/test/services/logger.jl @@ -0,0 +1,300 @@ +@testset verbose = true "log metric" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "with run id as string" begin + run = createrun(mlf, experiment_id) + logmetric(mlf, run.info.run_id, "missy", 0.9) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + + @test last_metric isa Metric + @test last_metric.key == "missy" + @test last_metric.value == 0.9 + deleterun(mlf, run) + end + + @testset "with run" begin + run = createrun(mlf, experiment_id) + logmetric(mlf, run, "gala", 0.1) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + + @test last_metric isa Metric + @test last_metric.key == "gala" + @test last_metric.value == 0.1 + deleterun(mlf, run) + end + + @testset "with run id as string and metric" begin + run = createrun(mlf, experiment_id) + logmetric(mlf, run.info.run_id, Metric("missy", 0.9, 123, 1)) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + + @test last_metric isa Metric + @test last_metric.key == "missy" + @test last_metric.value == 0.9 + @test last_metric.timestamp == 123 + @test last_metric.step == 1 + deleterun(mlf, run) + end + + @testset "with run and metric" begin + run = createrun(mlf, experiment_id) + logmetric(mlf, run, Metric("gala", 0.1, 123, 1)) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + + @test last_metric isa Metric + @test last_metric.key == "gala" + @test last_metric.value == 0.1 + @test last_metric.timestamp == 123 + @test last_metric.step == 1 + deleterun(mlf, run) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "log batch" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "with run id as string" begin + run = createrun(mlf, experiment_id) + logbatch(mlf, run.info.run_id; metrics=[("gala", 0.1)]) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + + @test last_metric isa Metric + @test last_metric.key == "gala" + @test last_metric.value == 0.1 + deleterun(mlf, run) + end + + @testset "with run" begin + run = createrun(mlf, experiment_id) + logbatch(mlf, run; metrics=[("missy", 0.9)]) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + + @test last_metric isa Metric + @test last_metric.key == "missy" + @test last_metric.value == 0.9 + deleterun(mlf, run) + end + + @testset "with metrics, params and tags as dict" begin + run = createrun(mlf, experiment_id) + logbatch(mlf, run; metrics=Dict("ana" => 0.5), + params=Dict("test_param" => "0.9"), + tags=Dict("test_tag" => "gala")) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + last_param = run.data.params |> last + last_tag = run.data.tags[ + findall(x -> !occursin("mlflow.runName", x.key), run.data.tags)[1]] + + @test last_metric isa Metric + @test last_metric.key == "ana" + @test last_metric.value == 0.5 + + @test last_param isa Param + @test last_param.key == "test_param" + @test last_param.value == "0.9" + + @test last_tag isa Tag + @test last_tag.key == "test_tag" + @test last_tag.value == "gala" + + deleterun(mlf, run) + end + + @testset "with metrics, params and tags as pair array" begin + run = createrun(mlf, experiment_id) + logbatch(mlf, run; metrics=["ana" => 0.5], + params=["test_param" => "0.9"], tags=["test_tag" => "gala"]) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + last_param = run.data.params |> last + last_tag = run.data.tags[ + findall(x -> !occursin("mlflow.runName", x.key), run.data.tags)[1]] + + @test last_metric isa Metric + @test last_metric.key == "ana" + @test last_metric.value == 0.5 + + @test last_param isa Param + @test last_param.key == "test_param" + @test last_param.value == "0.9" + + @test last_tag isa Tag + @test last_tag.key == "test_tag" + @test last_tag.value == "gala" + + deleterun(mlf, run) + end + + @testset "with metrics, params and tags as tuple array" begin + run = createrun(mlf, experiment_id) + logbatch(mlf, run; metrics=[("ana", 0.5)], + params=[("test_param", "0.9")], tags=[("test_tag", "gala")]) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + last_param = run.data.params |> last + last_tag = run.data.tags[ + findall(x -> !occursin("mlflow.runName", x.key), run.data.tags)[1]] + + @test last_metric isa Metric + @test last_metric.key == "ana" + @test last_metric.value == 0.5 + + @test last_param isa Param + @test last_param.key == "test_param" + @test last_param.value == "0.9" + + @test last_tag isa Tag + @test last_tag.key == "test_tag" + @test last_tag.value == "gala" + + deleterun(mlf, run) + end + + @testset "with metrics, params and tags as dict array" begin + run = createrun(mlf, experiment_id) + logbatch(mlf, run; + metrics=[Dict("key" => "ana", "value" => 0.5, "timestamp" => 123)], + params=[Dict("key" => "test_param", "value" => "0.9")], + tags=[Dict("key" => "test_tag", "value" => "gala")]) + + run = refresh(mlf, run) + last_metric = run.data.metrics |> last + last_param = run.data.params |> last + last_tag = run.data.tags[ + findall(x -> !occursin("mlflow.runName", x.key), run.data.tags)[1]] + + @test last_metric isa Metric + @test last_metric.key == "ana" + @test last_metric.value == 0.5 + @test last_metric.timestamp == 123 + + @test last_param isa Param + @test last_param.key == "test_param" + @test last_param.value == "0.9" + + @test last_tag isa Tag + @test last_tag.key == "test_tag" + @test last_tag.value == "gala" + + deleterun(mlf, run) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "log inputs" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "with run id as string" begin + run = createrun(mlf, experiment_id) + inputs = [DatasetInput([Tag("tag_key", "tag_value")], + Dataset("dataset_name", "dataset_digest", "dataset_source_type", + "dataset_source", nothing, nothing))] + loginputs(mlf, run.info.run_id, inputs) + + run = refresh(mlf, run) + + @test run.inputs.dataset_inputs |> length == 1 + + dataset_input = run.inputs.dataset_inputs |> first + dataset_input_tag = dataset_input.tags |> first + + @test dataset_input_tag isa Tag + @test dataset_input_tag.key == "tag_key" + @test dataset_input_tag.value == "tag_value" + + @test dataset_input.dataset isa Dataset + @test dataset_input.dataset.name == "dataset_name" + @test dataset_input.dataset.digest == "dataset_digest" + @test dataset_input.dataset.source_type == "dataset_source_type" + @test dataset_input.dataset.source == "dataset_source" + @test dataset_input.dataset.schema |> isnothing + @test dataset_input.dataset.profile |> isnothing + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "log param" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "with run id as string" begin + run = createrun(mlf, experiment_id) + logparam(mlf, run.info.run_id, "missy", "0.9") + + run = refresh(mlf, run) + last_param = run.data.params |> last + + @test last_param isa Param + @test last_param.key == "missy" + @test last_param.value == "0.9" + deleterun(mlf, run) + end + + @testset "with run" begin + run = createrun(mlf, experiment_id) + logparam(mlf, run, "gala", "0.1") + + run = refresh(mlf, run) + last_param = run.data.params |> last + + @test last_param isa Param + @test last_param.key == "gala" + @test last_param.value == "0.1" + deleterun(mlf, run) + end + + @testset "with run id as string and param" begin + run = createrun(mlf, experiment_id) + logparam(mlf, run.info.run_id, Param("missy", "0.9")) + + run = refresh(mlf, run) + last_param = run.data.params |> last + + @test last_param isa Param + @test last_param.key == "missy" + @test last_param.value == "0.9" + deleterun(mlf, run) + end + + @testset "with run and param" begin + run = createrun(mlf, experiment_id) + logparam(mlf, run, Param("gala", "0.1")) + + run = refresh(mlf, run) + last_param = run.data.params |> last + + @test last_param isa Param + @test last_param.key == "gala" + @test last_param.value == "0.1" + deleterun(mlf, run) + end + + deleteexperiment(mlf, experiment_id) +end diff --git a/test/services/misc.jl b/test/services/misc.jl new file mode 100644 index 0000000..d53a34c --- /dev/null +++ b/test/services/misc.jl @@ -0,0 +1,18 @@ +@testset verbose = true "get metric history" begin + @ensuremlf + + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment_id) + for i in 1:20 + logmetric(mlf, run, "missy", i |> Float64) + end + + @testset "default search" begin + metrics, next_page_token = getmetrichistory(mlf, run, "missy") + + @test length(metrics) == 20 + @test next_page_token |> isnothing + end + + deleteexperiment(mlf, experiment_id) +end diff --git a/test/services/model_version.jl b/test/services/model_version.jl new file mode 100644 index 0000000..fbf9a20 --- /dev/null +++ b/test/services/model_version.jl @@ -0,0 +1,233 @@ +@testset verbose = true "get latest model versions" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + createmodelversion(mlf, "missy", run.info.artifact_uri) + createmodelversion(mlf, "missy", run.info.artifact_uri) + + model_versions = getlatestmodelversions(mlf, "missy") + + @test model_versions isa Array{ModelVersion} + @test length(model_versions) == 1 + @test (model_versions |> first).name == "missy" + @test (model_versions |> first).source == run.info.artifact_uri + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "create model version" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + @testset "base" begin + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + + @test model_version isa ModelVersion + @test model_version.name == "missy" + @test model_version.source == run.info.artifact_uri + end + + @testset "with all params" begin + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri; + run_id=run.info.run_id, tags=[Tag("test_key", "test_value")], + run_link="run.link", description="test_description") + end + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "get model version" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + retrieved_model_version = getmodelversion(mlf, "missy", model_version.version) + + @test retrieved_model_version isa ModelVersion + @test retrieved_model_version.name == model_version.name + @test retrieved_model_version.version == model_version.version + @test retrieved_model_version.source == model_version.source + @test retrieved_model_version.run_id == model_version.run_id + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "update model version" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + updated_model_version = updatemodelversion(mlf, "missy", model_version.version; + description="test_description") + + @test updated_model_version isa ModelVersion + @test updated_model_version.name == model_version.name + @test updated_model_version.version == model_version.version + @test updated_model_version.source == model_version.source + @test updated_model_version.run_id == model_version.run_id + @test updated_model_version.description != model_version.description + @test updated_model_version.description == "test_description" + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "delete model version" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + deletemodelversion(mlf, "missy", model_version.version) + + @test_throws ErrorException getmodelversion(mlf, "missy", model_version.version) + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "search model versions" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + createregisteredmodel(mlf, "gala") + + createmodelversion(mlf, "missy", run.info.artifact_uri) + createmodelversion(mlf, "gala", run.info.artifact_uri) + + @testset "default search" begin + model_versions, next_page_token = searchmodelversions(mlf) + + @test length(model_versions) == 2 # four because of the default experiment + @test next_page_token |> isnothing + end + + @testset "with pagination" begin + experiments, next_page_token = searchexperiments(mlf; max_results=1) + + @test length(experiments) == 1 + @test next_page_token |> !isnothing + @test next_page_token isa String + end + + deleteregisteredmodel(mlf, "missy") + deleteregisteredmodel(mlf, "gala") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "get download uri for model version artifacts" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + download_uri = getdownloaduriformodelversionartifacts(mlf, model_version.name, + model_version.version) + + @test download_uri isa String + @test download_uri == run.info.artifact_uri + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "transition model version stage" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + updated_model_version = transitionmodelversionstage(mlf, model_version.name, + model_version.version, "Production", true) + + @test updated_model_version isa ModelVersion + @test updated_model_version.name == model_version.name + @test updated_model_version.version == model_version.version + @test updated_model_version.source == model_version.source + @test updated_model_version.run_id == model_version.run_id + @test updated_model_version.current_stage == "Production" + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "set model version tag" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + setmodelversiontag(mlf, model_version.name, model_version.version, "test_key", + "test_value") + retrieved_model_version = getmodelversion(mlf, "missy", model_version.version) + + @test retrieved_model_version.tags |> length == 1 + @test (retrieved_model_version.tags |> first).key == "test_key" + @test (retrieved_model_version.tags |> first).value == "test_value" + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "delete model version tag" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + setmodelversiontag(mlf, model_version.name, model_version.version, "test_key", + "test_value") + deletemodelversiontag(mlf, model_version.name, model_version.version, "test_key") + retrieved_model_version = getmodelversion(mlf, "missy", model_version.version) + + @test isempty(retrieved_model_version.tags) + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "get model version by alias" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + registered_model = createregisteredmodel(mlf, "missy") + + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + setregisteredmodelalias(mlf, registered_model.name, "gala", model_version.version) + + retrieved_model_version = getmodelversionbyalias(mlf, "missy", "gala") + + @assert retrieved_model_version.aliases |> !isempty + @assert (retrieved_model_version.aliases |> first) == "gala" + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end diff --git a/test/services/registered_model.jl b/test/services/registered_model.jl new file mode 100644 index 0000000..3140099 --- /dev/null +++ b/test/services/registered_model.jl @@ -0,0 +1,245 @@ +@testset verbose = true "create registered model" begin + @ensuremlf + + @testset "base" begin + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + + @test registered_model isa RegisteredModel + @test registered_model.name == "missy" + @test registered_model.description == "gala" + end + + @testset "name exists" begin + registered_model = getregisteredmodel(mlf, "missy") + @test_throws ErrorException createregisteredmodel(mlf, registered_model.name) + deleteregisteredmodel(mlf, "missy") + end + + @testset "with tags as array of tags" begin + createregisteredmodel(mlf, "missy"; tags=[Tag("test_key", "test_value")]) + deleteregisteredmodel(mlf, "missy") + end + + @testset "with tags as array of pairs" begin + createregisteredmodel(mlf, "missy"; tags=["test_key" => "test_value"]) + deleteregisteredmodel(mlf, "missy") + end + + @testset "with tags as array of dicts" begin + createregisteredmodel(mlf, "missy"; + tags=[Dict("key" => "test_key", "value" => "test_value")]) + deleteregisteredmodel(mlf, "missy") + end + + @testset "with tags as dict" begin + createregisteredmodel(mlf, "missy"; tags=Dict("test_key" => "test_value")) + deleteregisteredmodel(mlf, "missy") + end +end + +@testset verbose = true "get registered model" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + retrieved_registered_model = getregisteredmodel(mlf, registered_model.name) + + @test retrieved_registered_model isa RegisteredModel + @test retrieved_registered_model.name == registered_model.name + @test retrieved_registered_model.description == registered_model.description + + deleteregisteredmodel(mlf, "missy") +end + +@testset verbose = true "rename registered model" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + renamed_registered_model = renameregisteredmodel(mlf, registered_model.name, "gala") + + @test renamed_registered_model isa RegisteredModel + @test renamed_registered_model.name == "gala" + @test renamed_registered_model.description == registered_model.description + + deleteregisteredmodel(mlf, "gala") +end + +@testset verbose = true "update registered model" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + updated_registered_model = updateregisteredmodel(mlf, registered_model.name; + description="ana") + + @test updated_registered_model isa RegisteredModel + @test updated_registered_model.name == registered_model.name + @test updated_registered_model.description == "ana" + + deleteregisteredmodel(mlf, "missy") +end + +@testset verbose = true "delete registered model" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + deleteregisteredmodel(mlf, "missy") + + @test_throws ErrorException getregisteredmodel(mlf, "missy") +end + +@testset verbose = true "search registered models" begin + @ensuremlf + + createregisteredmodel(mlf, "missy"; description="gala") + createregisteredmodel(mlf, "gala"; description="missy") + + @testset "default search" begin + registered_models, next_page_token = searchregisteredmodels(mlf) + + @test length(registered_models) == 2 # four because of the default experiment + @test next_page_token |> isnothing + end + + @testset "with pagination" begin + registered_models, next_page_token = searchregisteredmodels(mlf; max_results=1) + + @test length(registered_models) == 1 + @test next_page_token |> !isnothing + @test next_page_token isa String + end + + deleteregisteredmodel(mlf, "missy") + deleteregisteredmodel(mlf, "gala") +end + +@testset verbose = true "set registered model tag" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + setregisteredmodeltag(mlf, registered_model.name, "test_key", "test_value") + + retrieved_registered_model = getregisteredmodel(mlf, registered_model.name) + @test retrieved_registered_model.tags |> !isempty + @test (retrieved_registered_model.tags |> first).key == "test_key" + @test (retrieved_registered_model.tags |> first).value == "test_value" + + deleteregisteredmodel(mlf, "missy") +end + +@testset verbose = true "delete registered model tag" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + setregisteredmodeltag(mlf, registered_model.name, "test_key", "test_value") + deleteregisteredmodeltag(mlf, registered_model.name, "test_key") + + retrieved_registered_model = getregisteredmodel(mlf, registered_model.name) + @test retrieved_registered_model.tags |> isempty + + deleteregisteredmodel(mlf, "missy") +end + +@testset verbose = true "delete registered model alias" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + + setregisteredmodelalias(mlf, registered_model.name, "gala", model_version.version) + deleteregisteredmodelalias(mlf, registered_model.name, "gala") + + retrieved_registered_model = getregisteredmodel(mlf, registered_model.name) + @test retrieved_registered_model.aliases |> isempty + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "delete registered model alias" begin + @ensuremlf + + experiment = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment) + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + model_version = createmodelversion(mlf, "missy", run.info.artifact_uri) + + setregisteredmodelalias(mlf, registered_model.name, "gala", model_version.version) + setregisteredmodelalias(mlf, registered_model.name, "missy", model_version.version) + + retrieved_registered_model = getregisteredmodel(mlf, registered_model.name) + @test retrieved_registered_model.aliases |> !isempty + @test length(retrieved_registered_model.aliases) == 2 + + deleteregisteredmodel(mlf, "missy") + deleteexperiment(mlf, experiment) +end + +@testset verbose = true "create registered model permission" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + user = createuser(mlf, "missy", "gala") + permission = createregisteredmodelpermission(mlf, registered_model.name, user.username, Permission("READ")) + + @test permission isa RegisteredModelPermission + @test permission.name == registered_model.name + @test permission.user_id == user.id + @test permission.permission == Permission("READ") + + deleteregisteredmodelpermission(mlf, registered_model.name, user.username) + deleteuser(mlf, user.username) + deleteregisteredmodel(mlf, "missy") +end + +@testset verbose = true "get registered model permission" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + user = createuser(mlf, "missy", "gala") + permission = createregisteredmodelpermission(mlf, registered_model.name, user.username, Permission("READ")) + retrieved_permission = getregisteredmodelpermission(mlf, registered_model.name, user.username) + + @test retrieved_permission isa RegisteredModelPermission + @test retrieved_permission.name == registered_model.name + @test retrieved_permission.user_id == user.id + @test retrieved_permission.permission == Permission("READ") + + deleteregisteredmodelpermission(mlf, registered_model.name, user.username) + deleteuser(mlf, user.username) + deleteregisteredmodel(mlf, "missy") +end + +@testset verbose = true "update registered model permission" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + user = createuser(mlf, "missy", "gala") + permission = createregisteredmodelpermission(mlf, registered_model.name, user.username, Permission("READ")) + updateregisteredmodelpermission(mlf, registered_model.name, user.username, Permission("MANAGE")) + retrieved_permission = getregisteredmodelpermission(mlf, registered_model.name, user.username) + + @test retrieved_permission isa RegisteredModelPermission + @test retrieved_permission.name == registered_model.name + @test retrieved_permission.user_id == user.id + @test retrieved_permission.permission == Permission("MANAGE") + + deleteregisteredmodelpermission(mlf, registered_model.name, user.username) + deleteuser(mlf, user.username) + deleteregisteredmodel(mlf, "missy") +end +# +@testset verbose = true "delete registered model permission" begin + @ensuremlf + + registered_model = createregisteredmodel(mlf, "missy"; description="gala") + user = createuser(mlf, "missy", "gala") + permission = createregisteredmodelpermission(mlf, registered_model.name, user.username, Permission("READ")) + deleteregisteredmodelpermission(mlf, registered_model.name, user.username) + + @test_throws ErrorException getregisteredmodelpermission(mlf, registered_model.name, user.username) + deleteuser(mlf, user.username) + deleteregisteredmodel(mlf, "missy") +end diff --git a/test/services/run.jl b/test/services/run.jl new file mode 100644 index 0000000..fb7af1b --- /dev/null +++ b/test/services/run.jl @@ -0,0 +1,219 @@ +@testset verbose = true "create run" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "base" begin + run = createrun(mlf, experiment_id) + + @test run.info isa RunInfo + @test run.data isa RunData + @test run.inputs isa RunInputs + @test run.info.experiment_id == experiment_id + end + + @testset "with experiment id as string" begin + run = createrun(mlf, experiment_id) + + @test run.info.experiment_id == experiment_id + end + + @testset "with experiment id as integer" begin + run = createrun(mlf, parse(Int, experiment_id)) + + @test run.info.experiment_id == experiment_id + end + + @testset "with experiment" begin + experiment = getexperiment(mlf, experiment_id) + run = createrun(mlf, experiment) + + @test run.info.experiment_id == experiment_id + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "delete run" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment_id) + + @testset "using string id" begin + @test deleterun(mlf, run.info.run_id) + restorerun(mlf, run.info.run_id) + end + + @testset "using Run" begin + @test deleterun(mlf, run) + restorerun(mlf, run.info.run_id) + end + + @testset "delete already deleted" begin + deleterun(mlf, run.info.run_id) + @test deleterun(mlf, run.info.run_id) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "restore run" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment_id) + + @testset "using string id" begin + deleterun(mlf, run.info.run_id) + @test restorerun(mlf, run.info.run_id) + end + + @testset "using Run" begin + deleterun(mlf, run) + @test restorerun(mlf, run) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "get run" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment_id) + + @testset "using id" begin + retrieved_run = getrun(mlf, run.info.run_id) + + @test retrieved_run.info isa RunInfo + @test retrieved_run.data isa RunData + @test retrieved_run.inputs isa RunInputs + @test retrieved_run.info.experiment_id == experiment_id + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "set run tag" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "set tag with run string id" begin + run = createrun(mlf, experiment_id) + setruntag(mlf, run.info.run_id, "tag", "value") + + run = refresh(mlf, run) + + @test run.data.tags |> !isempty + last_tag = run.data.tags[ + findall(x -> !occursin("mlflow.runName", x.key), run.data.tags)[1]] + @test last_tag.key == "tag" + @test last_tag.value == "value" + + deleterun(mlf, run) + end + + @testset "set tag with run" begin + run = createrun(mlf, experiment_id) + setruntag(mlf, run, "tag", "value") + + run = refresh(mlf, run) + + @test run.data.tags |> !isempty + last_tag = run.data.tags[ + findall(x -> !occursin("mlflow.runName", x.key), run.data.tags)[1]] + @test last_tag.key == "tag" + @test last_tag.value == "value" + + deleterun(mlf, run) + end + + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "delete run tag" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + + @testset "delete tag with run string id" begin + run = createrun(mlf, experiment_id) + setruntag(mlf, run.info.run_id, "tag", "value") + deleteruntag(mlf, run.info.run_id, "tag") + + run = refresh(mlf, run) + + @test (run.data.tags |> length) == 1 # The default tag + deleterun(mlf, run) + end + + @testset "delete tag with run string id" begin + run = createrun(mlf, experiment_id) + setruntag(mlf, run, "tag", "value") + deleteruntag(mlf, run, "tag") + + run = refresh(mlf, run) + + @test (run.data.tags |> length) == 1 # The default tag + deleterun(mlf, run) + end + deleteexperiment(mlf, experiment_id) +end + +@testset verbose = true "search runs" begin + @ensuremlf + + experiment_ids = [ + createexperiment(mlf, UUIDs.uuid4() |> string), + createexperiment(mlf, UUIDs.uuid4() |> string), + ] + for experiment_id in experiment_ids + createrun(mlf, experiment_id) + end + + @testset "default search" begin + runs, next_page_token = searchruns(mlf; experiment_ids=experiment_ids) + + @test length(runs) == 2 + @test next_page_token |> isnothing + end + + @testset "with pagination" begin + runs, next_page_token = searchruns(mlf; experiment_ids=experiment_ids, + max_results=1) + + @test length(runs) == 1 + @test next_page_token |> !isnothing + @test next_page_token isa String + end + + experiment_ids .|> (id -> deleteexperiment(mlf, id)) +end + +@testset verbose = true "update run" begin + @ensuremlf + experiment_id = createexperiment(mlf, UUIDs.uuid4() |> string) + run = createrun(mlf, experiment_id) + + @testset "update with string id" begin + status = MLFlowClient.FINISHED + end_time = 123 + run_name = "missy" + + run_info = updaterun(mlf, run.info.run_id; status=status, end_time=end_time, run_name=run_name) + + @test run_info.status == status + @test run_info.end_time == end_time + @test run_info.run_name == run_name + end + + @testset "update with Run" begin + status = MLFlowClient.FAILED + end_time = 456 + run_name = "gala" + + run_info = updaterun(mlf, run.info.run_id; status=status, end_time=end_time, run_name=run_name) + + @test run_info.status == status + @test run_info.end_time == end_time + @test run_info.run_name == run_name + end + + deleteexperiment(mlf, experiment_id) +end diff --git a/test/services/user.jl b/test/services/user.jl new file mode 100644 index 0000000..7e81ea8 --- /dev/null +++ b/test/services/user.jl @@ -0,0 +1,62 @@ +@testset verbose = true "create user" begin + @ensuremlf + + user = createuser(mlf, "missy", "gala") + + @test user isa User + @test user.username == "missy" + @test user.is_admin == false + + deleteuser(mlf, user.username) +end + +@testset verbose = true "get user" begin + @ensuremlf + + user = createuser(mlf, "missy", "gala") + + retrieved_user = getuser(mlf, "missy") + + @test retrieved_user isa User + @test retrieved_user.username == "missy" + @test retrieved_user.is_admin == false + + deleteuser(mlf, retrieved_user.username) +end + +@testset verbose = true "update user password" begin + @ensuremlf + + getmlfinstance(encoded_credentials::String) = + MLFlow(headers=Dict("Authorization" => "Basic $(encoded_credentials)")) + + user = createuser(mlf, "missy", "gala") + encoded_credentials = Base64.base64encode("$(user.username):gala") + + updateuserpassword(getmlfinstance(encoded_credentials), "missy", "ana") + encoded_credentials = Base64.base64encode("$(user.username):ana") + + @test_nowarn searchexperiments(getmlfinstance(encoded_credentials)) + deleteuser(mlf, user.username) +end + +@testset verbose = true "update user admin" begin + @ensuremlf + + user = createuser(mlf, "missy", "gala") + updateuseradmin(mlf, "missy", true) + + retrieved_user = getuser(mlf, "missy") + @test retrieved_user.is_admin == true + + deleteuser(mlf, retrieved_user.username) +end + +@testset verbose = true "delete user" begin + @ensuremlf + + user = createuser(mlf, "missy", "gala") + deleteuser(mlf, "missy") + + @test_throws ErrorException getuser(mlf, "missy") +end diff --git a/test/test_experiments.jl b/test/test_experiments.jl deleted file mode 100644 index cc697f4..0000000 --- a/test/test_experiments.jl +++ /dev/null @@ -1,104 +0,0 @@ -@testset "createexperiment" begin - @ensuremlf - exp = createexperiment(mlf) - - @test isa(exp, MLFlowExperiment) - @test_throws ErrorException createexperiment(mlf; name=exp.name) - - deleteexperiment(mlf, exp) -end - -@testset verbose = true "getexperiment" begin - @ensuremlf - exp = createexperiment(mlf) - experiment = getexperiment(mlf, exp.experiment_id) - - @testset "getexperiment_by_experiment_id" begin - @test isa(experiment, MLFlowExperiment) - @test experiment.experiment_id == exp.experiment_id - end - - @testset "getexperiment_by_experiment_name" begin - experiment_by_name = getexperiment(mlf, exp.name) - @test isa(experiment_by_name, MLFlowExperiment) - @test experiment_by_name.experiment_id == exp.experiment_id - end - - @testset "getexperiment_not_found" begin - @test isa(getexperiment(mlf, 123), Missing) - end - deleteexperiment(mlf, exp) -end - -@testset "getorcreateexperiment" begin - @ensuremlf - expname = "getorcreate" - artifact_location = "test$(expname)" - e = getorcreateexperiment(mlf, expname; artifact_location=artifact_location) - @test isa(e, MLFlowExperiment) - - ee = getorcreateexperiment(mlf, expname) - @test isa(ee, MLFlowExperiment) - @test e === ee - @test occursin(artifact_location, e.artifact_location) - deleteexperiment(mlf, ee) -end - -@testset "deleteexperiment" begin - @ensuremlf - exp = createexperiment(mlf) - deleteexperiment(mlf, exp) - - experiments = searchexperiments(mlf) - @test length(experiments) == 1 # 1 for the default experiment -end - -@testset "restoreexperiment" begin - @ensuremlf - exp = createexperiment(mlf) - deleteexperiment(mlf, exp) - - experiments = searchexperiments(mlf) - @test length(experiments) == 1 # 1 for the default experiment - - restoreexperiment(mlf, exp) - experiments = searchexperiments(mlf) - @test length(experiments) == 2 # the restored experiment and the default one - - deleteexperiment(mlf, exp) -end - -@testset verbose = true "searchexperiments" begin - @ensuremlf - n_experiments = 3 - for i in 2:n_experiments - createexperiment(mlf) - end - createexperiment(mlf; name="test") - experiments = searchexperiments(mlf) - - @testset "searchexperiments_get_all" begin - @test length(experiments) == (n_experiments + 1) # Adding one for the default experiment - end - - @testset "searchexperiments_by_filter" begin - experiments_by_filter = searchexperiments(mlf; filter="name=\"test\"") - @test length(experiments_by_filter) == 1 - @test experiments_by_filter[1].name == "test" - end - - @testset "searchexperiments_by_filter_attributes" begin - experiments_by_filter = searchexperiments(mlf; filter_attributes=Dict("name" => "test")) - @test length(experiments_by_filter) == 1 - @test experiments_by_filter[1].name == "test" - end - - @testset "searchexperiments_filter_exception" begin - @test_throws ErrorException searchexperiments(mlf; filter="test", filter_attributes=Dict("test" => "test")) - end - - popfirst!(experiments) # removing the default experiment (it can't be deleted) - for e in experiments - deleteexperiment(mlf, e) - end -end diff --git a/test/test_functional.jl b/test/test_functional.jl deleted file mode 100644 index 400ba06..0000000 --- a/test/test_functional.jl +++ /dev/null @@ -1,122 +0,0 @@ -@testset "MLFlow" begin - mlf = MLFlow() - @test mlf.apiroot == ENV["MLFLOW_TRACKING_URI"] - @test mlf.apiversion == 2.0 - @test mlf.headers == Dict() - mlf = MLFlow("https://localhost:5001/api", apiversion=3.0) - @test mlf.apiroot == "https://localhost:5001/api" - @test mlf.apiversion == 3.0 - @test mlf.headers == Dict() - let custom_headers = Dict("Authorization" => "Bearer EMPTY") - mlf = MLFlow("https://localhost:5001/api", apiversion=3.0, headers=custom_headers) - @test mlf.apiroot == "https://localhost:5001/api" - @test mlf.apiversion == 3.0 - @test mlf.headers == custom_headers - end -end - -# test that sensitive fields are not displayed by show() -@testset "MLFLow/show" begin - let io = IOBuffer(), - secret_token = "SECRET" - - custom_headers = Dict("Authorization" => "Bearer $secret_token") - mlf = MLFlow("https://localhost:5001/api", apiversion=3.0, headers=custom_headers) - @test mlf.apiroot == "https://localhost:5001/api" - @test mlf.apiversion == 3.0 - @test mlf.headers == custom_headers - show(io, mlf) - show_output = String(take!(io)) - @test !(occursin(secret_token, show_output)) - end -end - -@testset "utils" begin - using MLFlowClient: uri, headers - using URIs: URI - - let apiroot = "http://localhost:5001/api", apiversion = 2.0, endpoint = "experiments/get" - mlf = MLFlow(apiroot; apiversion=apiversion) - apiuri = uri(mlf, endpoint) - @test apiuri == URI("$apiroot/$apiversion/mlflow/$endpoint") - end - let apiroot = "http://localhost:5001/api", auth_headers = Dict("Authorization" => "Bearer 123456"), - custom_headers = Dict("Content-Type" => "application/json") - - mlf = MLFlow(apiroot; headers=auth_headers) - apiheaders = headers(mlf, custom_headers) - @test apiheaders == Dict("Authorization" => "Bearer 123456", "Content-Type" => "application/json") - end -end - -@testset "artifacts" begin - @ensuremlf - exp = createexperiment(mlf) - @test isa(exp, MLFlowExperiment) - exprun = createrun(mlf, exp) - @test isa(exprun, MLFlowRun) - # only run the below if artifact_uri is a local directory - # i.e. when running mlflow server as a separate process next to the testset - # when running mlflow in a container, the below tests will be skipped - # this is what happens in github actions - mlflow runs in a container, the artifact_uri is not immediately available, and tests are skipped - artifact_uri = exprun.info.artifact_uri - if isdir(artifact_uri) - @test_throws SystemError logartifact(mlf, exprun, "/etc/shadow") - - tmpfiletoupload = "sometempfilename.txt" - f = open(tmpfiletoupload, "w") - write(f, "samplecontents") - close(f) - artifactpath = logartifact(mlf, exprun, tmpfiletoupload) - @test isfile(artifactpath) - rm(tmpfiletoupload) - artifactpath = logartifact(mlf, exprun, "randbytes.bin", b"some rand bytes here") - @test isfile(artifactpath) - - mkdir(joinpath(artifact_uri, "newdir")) - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "randbytesindir.bin"), b"bytes here") - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "randbytesindir2.bin"), b"bytes here") - mkdir(joinpath(artifact_uri, "newdir", "new2")) - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "new2", "randbytesindir.bin"), b"bytes here") - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "new2", "randbytesindir2.bin"), b"bytes here") - mkdir(joinpath(artifact_uri, "newdir", "new2", "new3")) - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "new2", "new3", "randbytesindir.bin"), b"bytes here") - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "new2", "new3", "randbytesindir2.bin"), b"bytes here") - mkdir(joinpath(artifact_uri, "newdir", "new2", "new3", "new4")) - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "new2", "new3", "new4", "randbytesindir.bin"), b"bytes here") - artifactpath = logartifact(mlf, exprun, joinpath("newdir", "new2", "new3", "new4", "randbytesindir2.bin"), b"bytes here") - - # artifact tree should now look like this: - # - # ├── newdir - # │   ├── new2 - # │   │   ├── new3 - # │   │   │   ├── new4 - # │   │   │   │   ├── randbytesindir2.bin - # │   │   │   │   └── randbytesindir.bin - # │   │   │   ├── randbytesindir2.bin - # │   │   │   └── randbytesindir.bin - # │   │   ├── randbytesindir2.bin - # │   │   └── randbytesindir.bin - # │   ├── randbytesindir2.bin - # │   └── randbytesindir.bin - # ├── randbytes.bin - # └── sometempfilename.txt - - # 4 directories, 10 files - - artifactlist = listartifacts(mlf, exprun) - @test sort(basename.(get_path.(artifactlist))) == ["newdir", "randbytes.bin", "sometempfilename.txt"] - @test sort(get_size.(artifactlist)) == [0, 14, 20] - - ald2 = listartifacts(mlf, exprun, maxdepth=2) - @test length(ald2) == 6 - @test sort(basename.(get_path.(ald2))) == ["new2", "newdir", "randbytes.bin", "randbytesindir.bin", "randbytesindir2.bin", "sometempfilename.txt"] - aldrecursion = listartifacts(mlf, exprun, maxdepth=-1) - @test length(aldrecursion) == 14 # 4 directories, 10 files - @test sum(typeof.(aldrecursion) .== MLFlowArtifactDirInfo) == 4 # 4 directories - @test sum(typeof.(aldrecursion) .== MLFlowArtifactFileInfo) == 10 # 10 files - end - deleterun(mlf, exprun) - deleteexperiment(mlf, exp) -end diff --git a/test/test_loggers.jl b/test/test_loggers.jl deleted file mode 100644 index bf3ea00..0000000 --- a/test/test_loggers.jl +++ /dev/null @@ -1,224 +0,0 @@ -@testset verbose = true "settag" begin - @ensuremlf - expname = "settag-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id) - - @testset "settag_by_run_id_and_key_value" begin - settag(mlf, r.info.run_id, "run_id_key_value", "test") - retrieved_run = searchruns(mlf, e; filter="tags.run_id_key_value = 'test'") - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "settag_by_run_info_and_key_value" begin - settag(mlf, r.info, "run_id_key_value", "test") - retrieved_run = searchruns(mlf, e; filter="tags.run_id_key_value = 'test'") - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "settag_by_run_and_key_value" begin - settag(mlf, r, "run_id_key_value", "test") - retrieved_run = searchruns(mlf, e; filter="tags.run_id_key_value = 'test'") - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "settag_by_union_and_dict_key_value" begin - settag(mlf, r, Dict("run_id_key_value" => "test")) - retrieved_run = searchruns(mlf, e; filter="tags.run_id_key_value = 'test'") - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - deleteexperiment(mlf, e) -end - -@testset verbose = true "logparam" begin - @ensuremlf - expname = "logparam-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id) - - @testset "logparam_by_run_id_and_key_value" begin - logparam(mlf, r.info.run_id, "run_id_key_value", "test") - retrieved_run = searchruns(mlf, e; filter_params=Dict("run_id_key_value" => "test")) - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "logparam_by_run_info_and_key_value" begin - logparam(mlf, r.info, "run_id_key_value", "test") - retrieved_run = searchruns(mlf, e; filter_params=Dict("run_id_key_value" => "test")) - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "logparam_by_run_and_key_value" begin - logparam(mlf, r, "run_id_key_value", "test") - retrieved_run = searchruns(mlf, e; filter_params=Dict("run_id_key_value" => "test")) - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "logparam_by_union_and_dict_key_value" begin - logparam(mlf, r, Dict("run_id_key_value" => "test")) - retrieved_run = searchruns(mlf, e; filter_params=Dict("run_id_key_value" => "test")) - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - deleteexperiment(mlf, e) -end - -@testset verbose = true "logmetric" begin - @ensuremlf - expname = "logmetric-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id) - - @testset "logmetric_by_run_id_and_key_value" begin - logmetric(mlf, r.info.run_id, "run_id_key_value", 1) - retrieved_run = searchruns(mlf, e) - @test length(retrieved_run) == 1 - @test isa(retrieved_run[1].data.metrics["run_id_key_value"], MLFlowRunDataMetric) - @test retrieved_run[1].data.metrics["run_id_key_value"].value == 1 - end - - @testset "logmetric_by_run_info_and_key_value" begin - logmetric(mlf, r.info, "run_id_key_value", 1) - retrieved_run = searchruns(mlf, e) - @test length(retrieved_run) == 1 - @test isa(retrieved_run[1].data.metrics["run_id_key_value"], MLFlowRunDataMetric) - @test retrieved_run[1].data.metrics["run_id_key_value"].value == 1 - end - - @testset "logmetric_by_run_and_key_value" begin - logmetric(mlf, r, "run_id_key_value", 1) - retrieved_run = searchruns(mlf, e) - @test length(retrieved_run) == 1 - @test isa(retrieved_run[1].data.metrics["run_id_key_value"], MLFlowRunDataMetric) - @test retrieved_run[1].data.metrics["run_id_key_value"].value == 1 - end - - @testset "logmetric_by_union_and_key_arrayvalue" begin - logmetric(mlf, r, "run_id_key_value", [1, 2, 3]) - retrieved_run = searchruns(mlf, e) - @test length(retrieved_run) == 1 - @test isa(retrieved_run[1].data.metrics["run_id_key_value"], MLFlowRunDataMetric) - @test retrieved_run[1].data.metrics["run_id_key_value"].value == 3 - end - - deleteexperiment(mlf, e) -end - -@testset verbose = true "logartifact" begin - @ensuremlf - expname = "logartifact-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname; artifact_location="/tmp/mlflow") - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id) - artifact_uri = r.info.artifact_uri - - tmpfile = "/tmp/mlflowclient-tempfile.txt" - open(tmpfile, "w") do f - write(f, "test") - end - - @testset "logartifact_by_run_and_filenameanddata" begin - artifact = logartifact(mlf, r, tmpfile, "testing") - @test isfile(artifact) - end - - @testset "logartifact_by_run_id_and_file" begin - artifact = logartifact(mlf, r.info.run_id, tmpfile) - @test isfile(artifact) - end - - @testset "logartifact_by_run_and_file" begin - artifact = logartifact(mlf, r, tmpfile) - @test isfile(artifact) - end - - @testset "logartifact_by_run_info_and_file" begin - artifact = logartifact(mlf, r.info, tmpfile) - @test isfile(artifact) - end - - @testset "logartifact_using_IOBuffer" begin - io = IOBuffer() - write(io, "testing IOBuffer") - seekstart(io) - artifact = logartifact(mlf, r, tmpfile, io) - @test isfile(artifact) - end - - @testset "logartifact_error" begin - @test_broken logartifact(mlf, r, "/etc/shadow") - end - - deleteexperiment(mlf, e) -end - -@testset verbose=true "logbatch" begin - @ensuremlf - expname = "logbatch-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id) - - @testset "logbatch_by_types" begin - param_array = [MLFlowRunDataParam("test_param_type", "test")] - metric_array = [MLFlowRunDataMetric("test_metric", 5, 3, 1)] - logbatch(mlf, r.info.run_id; params=param_array, metrics=metric_array) - - retrieved_run = searchruns(mlf, e; - filter_params=Dict("test_param_type" => "test")) - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - @testset "logbatch_by_dicts" begin - param_dict_array = [Dict("key"=>"test_param_dict", "value"=>"test")] - metric_dict_array = [ - Dict("key"=>"test_metric", "value"=>5, "step"=>3, "timestamp"=>1)] - logbatch(mlf, r.info.run_id; - params=param_dict_array, metrics=metric_dict_array) - - retrieved_run = searchruns(mlf, e; - filter_params=Dict("test_param_dict" => "test")) - @test length(retrieved_run) == 1 - @test retrieved_run[1].info.run_id == r.info.run_id - end - - deleteexperiment(mlf, e) -end - -@testset verbose=true "listartifacts" begin - @ensuremlf - expname = "listartifacts-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id) - - @testset "listartifacts_by_run_id" begin - artifacts = listartifacts(mlf, r.info.run_id) - @test length(artifacts) == 0 - end - - @testset "listartifacts_by_run" begin - artifacts = listartifacts(mlf, r) - @test length(artifacts) == 0 - end - - @testset "listartifacts_by_run_info" begin - artifacts = listartifacts(mlf, r.info) - @test length(artifacts) == 0 - end - - deleteexperiment(mlf, e) -end diff --git a/test/test_runs.jl b/test/test_runs.jl deleted file mode 100644 index 06adae1..0000000 --- a/test/test_runs.jl +++ /dev/null @@ -1,172 +0,0 @@ -@testset verbose = true "createrun" begin - @ensuremlf - expname = "createrun-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - - function runtests(run) - @test isa(run, MLFlowRun) - @test run.info.run_name == runname - end - - @testset "createrun_by_experiment_id" begin - r = createrun(mlf, e.experiment_id; run_name=runname) - runtests(r) - end - - @testset "createrun_by_experiment_type" begin - r = createrun(mlf, e; run_name=runname) - runtests(r) - end - - deleteexperiment(mlf, e) -end - -@testset "getrun" begin - @ensuremlf - expname = "getrun-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id; run_name=runname) - - retrieved_r = getrun(mlf, r.info.run_id) - - @test isa(retrieved_r, MLFlowRun) - @test retrieved_r.info.run_id == r.info.run_id - deleteexperiment(mlf, e) -end - -@testset verbose = true "updaterun" begin - @ensuremlf - expname = "updaterun-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - runname = "run-$(UUIDs.uuid4())" - r = createrun(mlf, e.experiment_id; run_name=runname) - - new_runname = "new_updaterun-$(UUIDs.uuid4())" - new_status = "FINISHED" - new_status_using_type = MLFlowRunStatus("FINISHED") - - function runtests(run_updated) - @test isa(run_updated, MLFlowRun) - @test run_updated.info.run_name != r.info.run_name - @test run_updated.info.status.status != r.info.status - @test run_updated.info.run_name == new_runname - @test run_updated.info.status.status == new_status - end - - @testset "updaterun_by_run_id" begin - r_updated = updaterun(mlf, r.info.run_id, new_status; run_name=new_runname) - runtests(r_updated) - end - @testset "updaterun_by_run_info" begin - r_updated = updaterun(mlf, r.info, new_status; run_name=new_runname) - runtests(r_updated) - end - @testset "updaterun_byrun" begin - r_updated = updaterun(mlf, r, new_status; run_name=new_runname) - runtests(r_updated) - end - - @testset "updaterun_by_run_info_and_defined_status" begin - r_updated = updaterun(mlf, r.info, new_status_using_type; run_name=new_runname) - runtests(r_updated) - end - @testset "updaterun_by_run_and_defined_status" begin - r_updated = updaterun(mlf, r, new_status_using_type; run_name=new_runname) - runtests(r_updated) - end - - deleteexperiment(mlf, e) -end - -@testset verbose = true "deleterun" begin - @ensuremlf - expname = "deleterun-$(UUIDs.uuid4())" - e = getorcreateexperiment(mlf, expname) - - function runtests(run) - @test deleterun(mlf, run) - end - - @testset "deleterun_by_run_info" begin - r = createrun(mlf, e.experiment_id) - runtests(r.info) - end - - @testset "deleterun_by_run" begin - r = createrun(mlf, e.experiment_id) - runtests(r) - end - - deleteexperiment(mlf, e) -end - -@testset verbose = true "searchruns" begin - @ensuremlf - getexpname() = "searchruns-$(UUIDs.uuid4())" - e1 = getorcreateexperiment(mlf, getexpname()) - e2 = getorcreateexperiment(mlf, getexpname()) - - run_array1 = MLFlowRun[] - run_array2 = MLFlowRun[] - run_status = ["FINISHED", "FINISHED", "FAILED"] - failed_runs = 0 - - function addruns!(run_array, experiment, run_status) - for status in run_status - run = createrun(mlf, experiment.experiment_id) - run = updaterun(mlf, run, status) - if status == "FAILED" - logparam(mlf, run, "test", "failed") - failed_runs += 1 - else - logparam(mlf, run, "test", "test") - end - push!(run_array, run) - end - end - - addruns!(run_array1, e1, run_status) - addruns!(run_array2, e2, run_status) - - @testset "searchruns_by_experiment_id" begin - runs = searchruns(mlf, e1.experiment_id) - @test runs |> length == run_array1 |> length - end - - @testset "searchruns_by_experiment" begin - runs = searchruns(mlf, e1) - @test runs |> length == run_array1 |> length - end - - @testset "searchruns_by_experiments_array" begin - runs = searchruns(mlf, [e1, e2]) - @test runs |> length == (run_array1 |> length) + (run_array2 |> length) - end - - @testset "searchruns_by_filter" begin - runs = searchruns(mlf, [e1, e2]; filter="param.test = \"failed\"") - @test failed_runs == runs |> length - end - - @testset "searchruns_by_filter_params" begin - runs = searchruns(mlf, [e1, e2]; filter_params=Dict("test" => "failed")) - @test failed_runs == runs |> length - end - - @testset "searchruns_filter_exception" begin - @test_throws ErrorException searchruns(mlf, [e1, e2]; filter="test", filter_params=Dict("test" => "test")) - end - - @testset "runs_get_methods" begin - runs = searchruns(mlf, [e1, e2]; filter_params=Dict("test" => "failed")) - @test get_info(runs[1]) == runs[1].info - @test get_data(runs[1]) == runs[1].data - @test get_run_id(runs[1]) == runs[1].info.run_id - @test get_params(runs[1]) == runs[1].data.params - end - - deleteexperiment(mlf, e1) - deleteexperiment(mlf, e2) -end