UPSTREAM: <carry>: add artifact storage api for backend

This change introduces a new api for artifact data retrieval for mlpipeline api server, to be consumed by UI and end users. Two new endpoints are introduced, allowing for 1) collection retrieval and 2) individual artifact metadata retrieval.

When an individual artifact resource is retrieved, users/ui can specify a share_url option that provides a pre-signed url that with a very short expiry date that can be used as an artifact download link directly from the originating object store provider. This is currently only implemented for s3-compliant object store backends.

Author: HumairAK

backend/api/v2beta1/artifacts.proto

@@ -0,0 +1,140 @@
+// Copyright 2024 The Kubeflow Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package kubeflow.pipelines.backend.api.v2beta1;
+
+import "google/api/annotations.proto";
+import "google/api/httpbody.proto";
+import "google/rpc/status.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/kubeflow/pipelines/backend/api/v2beta1/go_client";
+
+service ArtifactService {
+  // Finds all artifacts within the specified namespace.
+  // Namespace field is required. In multi-user mode, the caller
+  // is required to have RBAC verb "list" on the "artifacts"
+  // resource for the specified namespace.
+  rpc ListArtifacts(ListArtifactRequest) returns (ListArtifactResponse) {
+    option (google.api.http) = {
+      get: "/apis/v2beta1/artifacts"
+    };
+  }
+
+  // Finds a specific Artifact by ID.
+  rpc GetArtifact(GetArtifactRequest) returns (Artifact) {
+    option (google.api.http) = {
+      get: "/apis/v2beta1/artifacts/{artifact_id}"
+    };
+  }
+}
+
+message GetArtifactRequest {
+  // Required. The ID of the artifact to be retrieved.
+  string artifact_id = 1;
+
+  enum ArtifactView {
+    // Not specified, equivalent to BASIC.
+    ARTIFACT_VIEW_UNSPECIFIED = 0;
+
+    // Server responses excludes download_url
+    BASIC = 1;
+
+    // Server responses include download_url
+    DOWNLOAD = 2;
+  }
+
+  // Optional. Set to "DOWNLOAD" to included a signed URL with
+  // an expiry (default 15 seconds, unless configured other wise).
+  // This URL can be used to download the Artifact directly from
+  // the Artifact's storage provider. Set to "BASIC" to exclude
+  // the download_url from server responses, thus preventing the
+  // creation of any signed url.
+  // Defaults to BASIC.
+  ArtifactView view = 2;
+}
+
+// Passed onto MLMD ListOperationOptions
+// https://github.com/kubeflow/pipelines/blob/master/third_party/ml-metadata/ml_metadata/proto/metadata_store.proto#L868
+message ListArtifactRequest {
+  // Optional.
+  // Max number of resources to return in the result. A value of zero or less
+  // will result in the default (20).
+  // The API implementation also enforces an upper-bound of 100, and picks the
+  // minimum between this value and the one specified here.
+  // [default = 20]
+  int32 max_result_size = 1;
+
+  enum Field {
+    FIELD_UNSPECIFIED = 0;
+    CREATE_TIME = 1;
+    LAST_UPDATE_TIME = 2;
+    ID = 3;
+  }
+
+  // Optional. Ordering field. [default = ID]
+  Field order_by_field = 2;
+  // Optional. Can be either "asc" (ascending) or "dsc" (descending). [default = asc]
+  string order_by = 3;
+
+  // Optional. The next_page_token value returned from a previous List request, if any.
+  string next_page_token = 4;
+
+  // Required. Namespace of the Artifact's context.
+  string namespace = 5;
+}
+
+message ListArtifactResponse {
+  // List of retrieved artifacts.
+  repeated Artifact artifacts = 1;
+
+  // Token to retrieve the next page of results, or empty if there are none
+  string next_page_token = 2;
+}
+
+message Artifact {
+  // Unique Artifact ID. Generated by MLMD.
+  string artifact_id = 1;
+  // Storage Provider to which this Artifact is located (e.g. S3, Minio, etc.).
+  string storage_provider = 2;
+  // The path location of this Artifact within the storage provider.
+  // For example an object located at s3://my-bucket/path/a/b/c will
+  // result in "path/a/b/c".
+  string storage_path = 3;
+  // The Artifact URI
+  string uri = 4;
+  // Optional Output. Specifies a signed-url that can be used to
+  // download this Artifact directly from its store.
+  string download_url  = 5;
+  // The namespace associated with this Artifact. This is determined
+  // by the namespace of the parent PipelineRun that created this Artifact.
+  string namespace = 6;
+  // The MLMD type of the artifact (e.g. system.Dataset)
+  string artifact_type = 7;
+  // The size of the artifact in bytes.
+  // If the artifact does not exist in object store (e.g. Metrics)
+  // then this is omitted.
+  int64 artifact_size = 8;
+  // Creation time of the artifact.
+  google.protobuf.Timestamp created_at = 9;
+  // Last update time of the artifact.
+  google.protobuf.Timestamp last_updated_at = 10;
+
+  // In case any error happens retrieving an artifact field, only artifact ID
+  // and the error message is returned. Client has the flexibility of choosing
+  // how to handle the error. This is especially useful when calling ListArtifacts.
+  google.rpc.Status error = 11;
+}